
    _j>              	       ^   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Zd dl
mc mZ d dlZd dlmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZmZ d dlmZ d dlm Z m!Z! d dl"m#Z# d d	l$m%Z% d
Z&dZ'dZ(dZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2de3deee4         eeee4         f         fdZ5dedede3ddfdZ6deee4ej7        f                  fdZ8de4de4de9fdZ:d eee4ej7        f                  d!ee4ee4         f         de4ddfd"Z;d eee4ej7        f                  d!ee4ee4         f         dee4e4f         fd#Z<d$e4de4fd%Z=d&e4de>fd'Z?d(e4d)ee4e4f         de>fd*Z@d+e4de4fd,ZAd-ejB        d.e4dee4         fd/ZCd0e4de4de9fd1ZDd(e4de4dee4         fd2ZEd3ee4         de4fd4ZFde4dee4ee4         f         fd5ZGde4ddfd6ZHd7 ZIded8e4de4ddfd9ZJejK        L                    e d:;          d>d<            ZMejK        L                    e d:;          ejK        jN        d>d=                        ZOdS )?    N)DictListOptionalTuplecast)	ClientAPI)
Collection)
Embeddings	Metadatas)MULTI_REGION_ENABLED)_create_isolated_database_mcmr_create_mcmr_clients)wait_for_version_increase)create_batches)chromachroma2zgarbage-collector-0zhttp://localhost:9000)zchroma-storagezchroma-storage2miniominio123z	us-east-1         <   g      >@round_indexreturnc                 \     fdt          t                    D             } fdt          t                    D             } fdt          t                    D             } fdt          t                    D             }|t          t          |          t          t          |          |fS )Nc           	      H    g | ]}d  d| dt          j                     S )zround-z-record--)uuiduuid4.0record_indexr   s     _/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/chromadb/test/property/test_add_gc.py
<listcomp>z&_records_for_round.<locals>.<listcomp>.   sH        	DCClCCTZ\\CC      c                 N    g | ]!}t                    t          |          d g"S )      ?)floatr    s     r#   r$   z&_records_for_round.<locals>.<listcomp>2   s@        
{		U<00#6  r%   c                     g | ]}|d S ))roundrecord r    s     r#   r$   z&_records_for_round.<locals>.<listcomp>6   s1        66  r%   c                      g | ]
}d  d| S )zround z record r,   r    s     r#   r$   z&_records_for_round.<locals>.<listcomp>:   s9        	544l44  r%   )rangeRECORDS_PER_ROUNDr   r
   r   )r   ids
embeddings	metadatas	documentss   `    r#   _records_for_roundr4   +   s       !"344  C   !"344  J   !"344  I   !"344  I Z,,d9i.H.H)SSr%   client
collectionc                 p    t          |          \  }}}}t          | ||||          D ]} |j        |  d S )N)apir0   r1   r2   r3   )r4   r   add)r5   r6   r   r0   r1   r2   r3   batchs           r#   
_add_roundr;   A   sc    ,>{,K,K)CY	     	
 r%   c            
      
   g } t           D ]x}	 t          j        ddd|t          ddgt          j        t          j        dd          }n$# t
          $ r t          j        d	           Y nw xY w|                     ||f           y| S )
Nkubectllogsz-nz--tail=0z--followT   )stdoutstderrtextbufsizez3kubectl is required to watch garbage collector logs)	GC_NAMESPACES
subprocessPopenGC_POD_NAMEPIPEFileNotFoundErrorpytestskipappend)watchers	namespaceprocs      r#   _start_gc_log_watchersrP   M   s    H" + +		O# "!  DD ! 	O 	O 	OKMNNNNN	OD)****Os   9AA('A(outputcollection_uuidc                     d}	 |                      d|          }|dk    rdS || ||dz            v rdS |t          d          z   }B)Nr   TzHard deleting collectionsFi   )findlen)rQ   rR   offsethard_delete_indexs       r#   _hard_delete_log_foundrY   f   sl    FF"KK(CVLL""5f%69JT9Q%QRRR4"S)D%E%EEFr%   rM   captured_stdoutc           	         t          j                    }	 | D ]3\  }}|j        |                    |j        t           j        |           4t          j                    t          z   }t          j                    |k     r>|                                r)t          dt          d|t          j                    z
                      }|                    |          D ]\  }}	t          t          |j                  }|j                                        }
|
dk    r|                    |j                   Y||                             |
           t'          d                    ||                   |          r |                                 d S t          j                    |k     r|                                )|                                 d S # |                                 w xY w)Nr'   g        timeout )	selectorsDefaultSelectorr@   register
EVENT_READtime	monotonicGC_HARD_DELETE_TIMEOUT_SECONDSget_mapminmaxselectr   strdatafileobjreadline
unregisterrL   rY   joinclose)rM   rZ   rR   selectorrN   rO   deadliner]   key_lines              r#   _wait_for_gc_hard_delete_logrv   q   s   
 (**H' 	L 	LOIt{"dk9+?KKKK>##&DDn))h.>.>.@.@)#s34>3C3C(CDDEEG"//'/:: 
 
Q ch//	{++--2::''444	*11$777)GGOI677    n))h.>.>.@.@) 	s   E$G -G G(c                 L   i }| D ]\  }}|                                  | D ]\  }}	 |                    d          \  }}n@# t          j        $ r. |                                 |                                \  }}Y nw xY w|r||                             |           |||<   |S )N   r\   )	terminatecommunicaterE   TimeoutExpiredkillrL   )rM   rZ   captured_stderrrt   rO   rN   r@   rA   s           r#   _stop_gc_log_watchersr~      s     O  4# , ,	4	0!--a-88NFFF( 	0 	0 	0IIKKK!--//NFFFF	0  	6I&--f555%+	""s   A:A>=A>valuec                 D    t           j                            | d          S )Nz-_.~safe)urllibparsequote)r   s    r#   
_aws_quoter      s    <e&111r%   
date_stampc                    t          j        dt                               d          |                     d          t          j                                                  }t          j        |t                              d          t          j                                                  }t          j        |dt          j                                                  }t          j        |dt          j                                                  S )NAWS4utf-8s   s3s   aws4_request)hmacnewMINIO_SECRET_KEYencodehashlibsha256digestMINIO_REGION)r   date_key
region_keyservice_keys       r#   _signing_keyr      s    x!!!((11'""  fhh	 
 ,%%g.. fhh  (:ugn==DDFFK8K'.AAHHJJJr%   bucketqueryc                    t           j                            t                    }t          j                            t          j        j                  }|                    d          }|                    d          }t          j
        d                                          }d|  }t           j                            |d          }d                    d t          |                                          D                       }	d	|j         d
| d| d}
d}d                    d||	|
||g          }| dt"           d}d                    d||t          j
        |                    d                                                    g          }t'          j        t+          |          |                    d          t          j
                                                  }dt,           d| d| d| }t           j                            |j        |j        |d|	df          }t           j                            ||||dd          }	 t           j                            |t8                    5 }|                                cd d d            S # 1 swxY w Y   d S # t           j        j        $ rf}|                                                     dd          }tC          j"        dt           d|  d|j#         d|j$         d|
           Y d }~d S d }~wt           j        j%        $ r-}tC          j"        dt           d|  d |            Y d }~d S d }~ww xY w)!Nz%Y%m%dT%H%M%SZz%Y%m%dr%   /z/-_.~r   &c              3   ^   K   | ](\  }}t          |           d t          |           V  )dS )=N)r   )r!   rs   r   s      r#   	<genexpr>z$_minio_signed_get.<locals>.<genexpr>   sS        5?S%:c??00Z..00     r%   zhost:z
x-amz-content-sha256:z
x-amz-date:
z$host;x-amz-content-sha256;x-amz-dateGETz/s3/aws4_requestzAWS4-HMAC-SHA256r   zAWS4-HMAC-SHA256 Credential=z, SignedHeaders=z, Signature=r^   )Authorizationzx-amz-content-sha256z
x-amz-date)headersmethodr\   replace)errorsz)Failed to list MinIO objects from S3 API z: HTTP  z; body=z"Failed to connect to MinIO S3 API z: )&r   r   urlparseMINIO_S3_ENDPOINTdatetimenowtimezoneutcstrftimer   r   	hexdigestr   ro   sorteditemsnetlocr   r   r   r   r   MINIO_ACCESS_KEY
urlunparseschemerequestRequesturlopen!MINIO_OBJECT_LIST_TIMEOUT_SECONDSreaderror	HTTPErrordecoderJ   failcodereasonURLError)r   r   endpointr   amz_dater   payload_hashpathcanonical_uricanonical_querycanonical_headerssigned_headerscanonical_requestcredential_scopestring_to_sign	signatureauthorizationurlr   responseebodys                         r#   _minio_signed_getr      sq   |$$%677H



 1 5
6
6C||,--Hh''J>#&&0022Lv<<DL&&t'&::Mhh  CI%++--CXCX    O	# 	# 	# ,	# 	#	# 	# 	# 
 <N			
	 	 %EE|EEEYYN,33G<<==GGII		
 N Z  g&&  ikk	 	A&	A 	A)9	A 	A'	A 	A5>	A 	A  ,
!
!OO	
	 	C n$$*$0"
 

  %  G
^##> $ 
 
 	#==??	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# <! 
 
 
vvxxwy99U U U#)U U23&U U;<8U ULPU U	
 	
 	
 	
 	
 	
 	
 	
 	
 <  
 
 
1 1 1#)1 1-.1 1	
 	
 	
 	
 	
 	
 	
 	
 	

sI   &J! 3JJ! JJ! JJ! !M5ALM-"MMtagc                 :    |                      dd          d         S )N}r?   rT   )rsplit)r   s    r#   _xml_local_namer   
  s    ::c1b!!r%   elementnamec                 R    | D ]#}t          |j                  |k    r	|j        c S $d S N)r   r   rB   )r   r   childs      r#   _child_textr     s>      59%%--: .4r%   rs   c                     d| d| v S )Nz/collection/z/versionfiles/r,   )rs   rR   s     r#   _is_collection_version_file_keyr     s    9/999S@@r%   c                    g }d }	 ddd}|||d<   t          | |          }t          j        |          }|                                D ]V}t	          |j                  dk    rt          |d          }|)||v r%t          ||          s|                    |           Wt          |d          d	k    }	|	sn't          |d
          }|t          j
        d           t          |          S )NT21000)z	list-typezmax-keyszcontinuation-tokenContentsKeyIsTruncatedtrueNextContinuationTokenzFMinIO returned a truncated object listing without a continuation token)r   ET
fromstringiterr   r   r   r   rL   rJ   r   r   )
r   rR   keyscontinuation_tokenr   r   rootr   rs   is_truncateds
             r#    _list_minio_files_for_collectionr     s#   D(,!v66)*<E&' //}T""yy{{ 		! 		!Gw{++z99gu--C#s**7_MM + C   "4776A 	(/FGG%KX  18 $<<r%   pathsc                 |    d}| d |         }t          |           |k    rdndt          |           |z
   d}| | S )N2   r^   z	 ... and z morerV   )r   sample_sizesamplesuffixs       r#   _format_minio_file_sampler   ;  s[    K<K< F%jjK''-XUk9Q-X-X-X  fr%   c           
          t          j                    t          z   }i }	  fdt          D             }t	          |                                          r|S t          j                    |k    rWd |                                D             }t          j        d  d| dd |                                D                         t          j	        d           )	NTc                 2    i | ]}|t          |          S r,   r   r!   r   rR   s     r#   
<dictcomp>z8_wait_for_minio_files_for_collection.<locals>.<dictcomp>H  6     
 
 
 4V_MM
 
 
r%   c                     g | ]	\  }}||
S r,   r,   r!   r   r   s      r#   r$   z8_wait_for_minio_files_for_collection.<locals>.<listcomp>O  s1       (655  r%   z;Expected MinIO to contain non-version files for collection zC in every test bucket before deletion, but these buckets had none: z. Found counts: c                 4    i | ]\  }}|t          |          S r,   r   r   s      r#   r   z8_wait_for_minio_files_for_collection.<locals>.<dictcomp>W  s$    UUUMFEVSZZUUUr%   r?   )
rc   rd   #MINIO_OBJECT_APPEAR_TIMEOUT_SECONDSMINIO_BUCKETSallvaluesr   rJ   r   sleep)rR   rr   paths_by_bucketmissing_bucketss   `   r#   $_wait_for_minio_files_for_collectionr  D  s#   ~"EEH,.O
 
 
 
'
 
 
 %%''(( 	#"">x'' ,;,A,A,C,C  O KY"Y Y/>Y Y VU_=R=R=T=TUUU	Y Y   	
1%r%   c                     t          j                    t          z   }i }	  fdt          D             }t	          |                                          sd S t          j                    |k    rud |                                D             }d |                                D             }t          j        d  dd |                                D              d|            t          j	        d	           )
NTc                 2    i | ]}|t          |          S r,   r   r   s     r#   r   z1_wait_for_minio_files_deleted.<locals>.<dictcomp>`  r   r%   c                     i | ]
\  }}|||S r,   r,   r   s      r#   r   z1_wait_for_minio_files_deleted.<locals>.<dictcomp>g  s3       "/&%e  r%   c                 4    i | ]\  }}|t          |          S r,   )r   r   s      r#   r   z1_wait_for_minio_files_deleted.<locals>.<dictcomp>j  s7       !FE 1%88  r%   z0Expected non-version MinIO files for collection zI to be deleted from every test bucket, but found files in these buckets: c                 4    i | ]\  }}|t          |          S r,   r   r   s      r#   r   z1_wait_for_minio_files_deleted.<locals>.<dictcomp>r  s$    OOOMFEVSZZOOOr%   z. Samples: r?   )
rc   rd   #MINIO_OBJECT_DELETE_TIMEOUT_SECONDSr  anyr  r   rJ   r   r  )rR   rr   r  	remainingsampless   `    r#   _wait_for_minio_files_deletedr  \  sC   ~"EEH,.O
 
 
 
'
 
 
 ?))++,, 	F>x'' 3B3H3H3J3J  I %.__%6%6  G K&"& & POY__=N=NOOO& & $	& &   	
1-r%   c                 2    || t          |          z           S r   r   )r   round_arounds     r#   round_robinr  x  s    c,&7&7788r%   collection_namec                    t                      }d t          D             }i }	 |                     |           t          ||           t	          ||          }n# t	          ||          }w xY wd |                                D             }fd|                                D             }|                                D ]'\  }}	t          | dt          |	           d           (|                                D ]#\  }}
|
rt          | d|
dd                      $|s,J d d	d
 |                                D                          d S )Nc                     i | ]}|g S r,   r,   )r!   rN   s     r#   r   z@_delete_collection_and_assert_gc_hard_delete.<locals>.<dictcomp>  s    DDDy"DDDr%   c                 @    i | ]\  }}|d                      |          S )r^   )ro   )r!   rN   liness      r#   r   z@_delete_collection_and_assert_gc_hard_delete.<locals>.<dictcomp>  s5       &6i	2775>>  r%   c                 :    g | ]\  }}t          |          |S r,   )rY   )r!   rN   r@   rR   s      r#   r$   z@_delete_collection_and_assert_gc_hard_delete.<locals>.<listcomp>  s=       Iv!&/::  r%   z GC stdout captured z bytesz GC stderr: iz:Expected garbage collector logs to hard delete collection z. Captured stdout tails: c                 *    i | ]\  }}||d d         S )i0Nr,   )r!   rN   r@   s      r#   r   z@_delete_collection_and_assert_gc_hard_delete.<locals>.<dictcomp>  s'    \\\+<9fYuvv\\\r%   )rP   rD   delete_collectionrv   r~   r   printrV   )r5   r  rR   rM   rZ   r}   stdout_by_namespacematching_namespacesrN   r@   rA   s     `        r#   ,_delete_collection_and_assert_gc_hard_deleter!  |  s    &''HDDmDDDO&(OK  111$XPPP//JJ//JJJJJJ :I:O:O:Q:Q     !4!:!:!<!<   16688 E E	6CCFCCCDDDD,2244 > >	6 	>Y<<F566N<<===  	`	` 	`\\@S@Y@Y@[@[\\\	` 	`    s   &A A-z;MCMR GC coverage requires a multi-region Kubernetes cluster)r   c                     t                      \  } }t          | |d           dt          j                    j         }|                     |          }|                    |           t          | |t          |j	                             d S )Ntilt-spanningtest_add_gc_empty_r   )
r   r   r   r   hexcreate_collectionget_collectionr!  rj   id)client1client2r  r6   s       r#   .test_add_gc_hard_deletes_empty_mcmr_collectionr,    s    
 ,--GW"7G_EEE=4:<<+;==O***@@J0000#jm"4"4    r%   c            
      v   t                      \  } }t          | |d           | |g}dt          j                    j         }|                     |          }|                    |          }t          |j                  }||g}t          t          |                                d                   }t          t          |                                d                   }	t          t                    D ]U}
t          |
|          }t          |
|          }t          |||
           t!          | ||          }t!          |||	          }	Vt#          |          }|                                D ]+\  }}t'          d| dt)          |           d| d           ,t+          | ||           t-          |           d S )	Nr#  test_add_gc_r%  versionzMinIO bucket z contained z files for collection z before deletion)r   r   r   r   r&  r'  r(  rj   r)  r   int	get_modelr.   COMPACTION_ROUNDSr  r;   r   r  r   r  rV   r!  r  )r*  r+  clientsr  coll1coll2rR   collectionscurrent_version1current_version2r   writer_clientwriter_collection#minio_files_before_delete_by_bucketr   r   s                   r#   (test_add_gc_hard_deletes_mcmr_collectionr<    s    ,--GW"7G_EEE G7TZ\\%577O%%?%;;E"""88E%(mmO%.KC!2!29!=>>C!2!29!=>>.// 

 

#K99'[AA="3[AAA4_&6
 
 5_&6
 
 +O+ +' =BBDD 
 
1F 1 1s5zz 1 11 1 1	
 	
 	
 	

 1/   "/22222r%   )r   N)Pr   r   r   r_   rE   rc   urllib.errorr   urllib.parseurllib.requestxml.etree.ElementTreeetreeElementTreer   r   typingr   r   r   r   r   rJ   chromadb.apir   chromadb.api.models.Collectionr	   chromadb.api.typesr
   r   chromadb.test.conftestr   $chromadb.test.property.test_add_mcmrr   r   -chromadb.test.utils.wait_for_version_increaser   chromadb.utils.batch_utilsr   rD   rG   r   r  r   r   r   r2  r/   re   r  r  r   r0  rj   r4   r;   rF   rP   boolrY   rv   r~   r   bytesr   r   r   Elementr   r   r   r   r  r  r  r!  markskipifr,  skip_single_regionr<  r,   r%   r#   <module>rQ     s                           " " " " " " " " "  4 4 4 4 4 4 4 4 4 4 4 4 4 4  " " " " " " 5 5 5 5 5 5 4 4 4 4 4 4 4 4 7 7 7 7 7 7        T S S S S S 5 5 5 5 5 5 &#+ 5    !$ &( #&( #$( !TT
49j)T#Y67T T T T,	y 	j 	s 	t 	 	 	 	U3
0@+@%A B    2F3 F F F F F F5j../0#tCy.)  
	   <5j../0#tCy.) 
#s(^   (2c 2c 2 2 2 2
KS 
KU 
K 
K 
K 
KS
c S
$sCx. S
U S
 S
 S
 S
l" " " " " " 3 8C=    A As At A A A AS 3 4PS9    DT#Y 3    # $sDQTI~BV    03 4    89 9 9(+>A	   D H   
 
 
	 
 H    &3 &3 &3  	 
&3 &3 &3r%   