_j* BUddlZddlZddlZddlmZddlmZmZddlm Z ddl m Z ddl m Z mZddlmZddlmZdd lmZmZdd lmZmZmZdd lmZmZddlZddlZdd lmZm Z m!Z!dd l"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlZ>m?Z?m@Z@mAZAddlBmCZCe GddZDe GddeDZEe GddeDZFeGejHZIeeJd<GddeIZKGd d!ejHeK"ZLGd#d$ejMjNZOGd%d&ejMjNZPd'eQd(eQfd)ZRd*eQd(eQfd+ZSd,eQd-eTdzd(efd.ZUd/eLd,eQd0eQd1eVd2eQf d3ZWd4ZXeLZYdS)5N)deque)IterableIterator) ExitStack)deepcopy) dataclassfield)datetime)chain)Path PurePosixPath)AnyNoReturnUnion)quoteunquote)_DEFAULT_CALLBACK NoOpCallback TqdmCallback) apply_config) isfilelike) constants)CommitOperationCopyCommitOperationDelete)BucketNotFoundErrorEntryNotFoundErrorHfHubHTTPErrorRepositoryNotFoundErrorRevisionNotFoundError) hf_hub_urlhttp_get)SPECIAL_REFS_REVISION_REGEX BucketFile BucketFolderHfApiLastCommitInfoRepoFile RepoFolder)HFValidationErrorhf_raise_for_status http_backoffhttp_stream_backoff parse_hf_uri)md5c4eZdZUdZeed<eed<defdZdS)HfFileSystemResolvedPathz_Top level Data structure containing information about a resolved Hugging Face file system path.rootpathreturncJ|jd|jdS)N/)r2r3rstripselfs [/var/www/FlaskApp/flask-venv/lib/python3.11/site-packages/huggingface_hub/hf_file_system.py unresolvez"HfFileSystemResolvedPath.unresolve+s()))di))00555N)__name__ __module__ __qualname____doc__str__annotations__r;r<r:r1r1$sIii III III63666666r<r1ceZdZUdZeed<eed<eed<eed<edZeed<edZeed <ed d Z ed zed <d Z d S)"HfFileSystemResolvedRepositoryPathzLData structure containing information about a resolved path in a repository. repo_typerepo_idrevision path_in_repoFinitr2r3N)defaultrepr _raw_revisionc tj|jd|jz}|jr|d|j|_n;|jtjkr|dt|j|_n||_|j |_ dS)N@) rREPO_TYPES_URL_PREFIXESgetrFrGrNr2rHDEFAULT_REVISION safe_revisionrIr3)r9 repo_paths r: __post_init__z0HfFileSystemResolvedRepositoryPath.__post_init__=s599$."MMPTP\\   "$;;t'9;;DII ]i8 8 8$EE}T]'C'CEEDII!DI% r<) r=r>r?r@rArBr r2r3rNrWrCr<r:rErE/sVVNNN LLLMMM5!!!D#!!!5!!!D#!!!!&d ? ? ?M3:???&&&&&r<rEcFeZdZUdZeed<edZeed<dZdS)HfFileSystemResolvedBucketPathzHData structure containing information about a resolved path in a bucket. bucket_idFrJr2c$d|jz|_dS)Nzbuckets/)rZr2r8s r:rWz,HfFileSystemResolvedBucketPath.__post_init__Os/ r<N) r=r>r?r@rArBr r2rWrCr<r:rYrYHsQRRNNN5!!!D#!!!00000r<rY _cached_basec(eZdZdZfdZdZxZS)_Cacheda Metaclass for caching HfFileSystem instances according to the args. This creates an additional reference to the filesystem, which prevents the filesystem from being garbage collected when all *user* references go away. A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* be made for a filesystem instance to be garbage collected. This is a slightly modified version of `fsspec.spec._Cached` to improve it. In particular in `_tokenize` the pid isn't taken into account for the `fs_token` used to identify cached instances. The `fs_token` logic is also robust to defaults values and the order of the args. Finally new instances reuse the states from sister instances in the main thread. cHtj|i|i|_dSN)super__init___cache)clsargskwargs __class__s r:rbz_Cached.__init__gs,$)&))) r<ctt||}|dd}|j|tjg|Ri|}|j|tjjg|Ri|}|s$|jr||jvr||_ |j|Stj |g|Ri|}|sZ|jrS||jvrJ|j| }| D]\}} t||| ||_||_||_|jr|s||_ ||j|<|S)Nskip_instance_cacheF)rpop _tokenize threading get_ident main_threadidentcachablerc_latesttype__call___get_instance_stateitemssetattr _fs_token_ storage_argsstorage_options) rdrerfskipfs_tokenfs_token_main_threadobjinstance_stateattr state_values r:rsz_Cached.__call__osc6**zz/77 3=i&9&;&;MdMMMfMM,s}S)2G2I2I2OaRVaaaZ`aa   SZ)?)?"CK:h' '-5d555f55C 4CL 4-ASZ-O-O!$,@!A!U!U!W!W)7)=)=)?)?44%D+C{3333%CN#C "(C | +D +& '* 8$Jr<)r=r>r?r@rbrs __classcell__rgs@r:r^r^WsQ  r<r^ceZdZdZdZdZddddddedzdeezdzdedzd edzffd Z e d ed efd Z dedededzd e ee dzffdZded e ee dzffdZ d9dededzd eezfdZd9dedzd dfdZ d:dedededzdedzd edf dZd9dededzd dfdZ d;dedededzdedzd df dZ ddededzd)ed!ed"ededzd eeeeeeeffzfd*Z$d9d+ed,ededzd dfd-Z%ded e&fd.Z'd?ded"ededzd eeeffd/Z(d0Z)d1Z*d2Z+ded efd3Z,e-dfd@fd4 Z.e/d5Z0d6Z1d7Z2d8Z3xZ4S)A HfFileSystemaz Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. The file system supports paths for the `hf://` protocol, which follows those URL schemes: * Models, Datasets and Spaces repositories: ``` hf://[@]/ hf://datasets/[@]/ hf://spaces/[@]/ ``` * Buckets (generic storage): ``` hf://buckets// ``` Note: when using the [`HfFileSystem`] directly, passing the `hf://` protocol prefix is optional in paths. Args: endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . token (`bool` or `str`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. block_size (`int`, *optional*): Block size for reading and writing files. expand_info (`bool`, *optional*): Whether to expand the information of the files. **storage_options (`dict`, *optional*): Additional options for the filesystem. See [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.__init__). Usage: ```python >>> from huggingface_hub import hffs >>> # List files >>> hffs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> hffs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with hffs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with hffs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` Specify a token for authentication: ```python >>> from huggingface_hub import HfFileSystem >>> hffs = HfFileSystem(token=token) ``` rPhfN)endpointtoken block_size expand_inforrrrctj|i||p tj|_||_t |||_||_||_ i|_ i|_ i|_ dS)N)rr) rarbrENDPOINTrrr&_apirr_repo_and_revision_exists_cache_bucket_exists_cachedircache)r9rrrrreryrgs r:rbzHfFileSystem.__init__s{ $2/222 6I$6  85999 $&rt,NP!:< r<threading_identr4cBdp tjd<dd<fdtD|||f}t t |}|S)zDeterministic token for cachingrrc"i|] }|| SrCrC).0keyrfs r: z*HfFileSystem._tokenize..s===s#vc{===r<)rSrrsortedr/rAencode hexdigest)rdrrerf tokenize_argshs ` r:rkzHfFileSystem._tokenizes$ZZ 33Iy7Iz **W--w====fVnn===otV< M""))++ , ,{{}}r<rFrGrHc|||f|jvr |j|||tjd|j|||f<d|j||df<nj#t t f$r(}d|f|j|||f<d|f|j||df<Yd}~n6d}~wt$r&}d|f|j|||f<d|j||df<Yd}~nd}~wwxYw|j|||fS)N)rHrFtimeoutTNF)rr repo_inforHF_HUB_ETAG_TIMEOUTrr*r )r9rFrGrHes r:_repo_and_revision_existz%HfFileSystem._repo_and_revision_existsT w )1U U U ^ ##h)YMj$Xb4i(5STS]4i$5OPP,->? \ \ \W\^_W_4i(5STSXZ[S[4i$5OPPPPPP( ^ ^ ^W\^_W_4i(5STS]4i$5OPPPPPP ^ 3Y4RSSs#(AB8"B B8B33B8rZc||jvrI |j|d|j|<n##t$r}d|f|j|<Yd}~nd}~wwxYw|j|S)NrF)rr bucket_infor)r9rZrs r:_bucket_existszHfFileSystem._bucket_exists s D5 5 5 B %%i0008B))44' @ @ @7>>??   a!%!4!4VY!?!? M3  1%dC0001FIFL_``` `  FO$?FOW_D_D_ :>C9KyOm9m9m 3 2 21 5 5sw  1 7 7Q ? ? GXGJhX^^C-C-C-C]egi\j * l'./?'@'@ $'833 M4LMMnvMMM,0+H+HV]_g+h+h ( #S* 1%dC0005 Wh L\  ? &8+;H'+'D'DV[RXR[]e'f'f$& - !$ , , ,$( $;;6?69=C9KyOm9m9m 3 2 21 5 5sw ,223::1=LNTNas<(IC0C,D,Dq,H*I(IJJgsL'3889S1 KHf.AQ]    r<cV|s4|j|jdS||}|}|r2|j|d||}|2|jst|trW|j|j |j dfd|j|j |j |j fddS|j |jddSdS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N)rclearrrr;rj_parentr3 isinstancerErFrGrHrrZ)r9r3 resolved_paths r:invalidate_cachezHfFileSystem.invalidate_cachecsX Q M   ! ! !  0 6 6 8 8 8 8 8 --d33M **,,D * !!$---||D)) * !% Qm-OPPQ8<<&0-2GNPT8<<&0-2GI_`bf-11-2I4PPPPP Q Qr<rbmode)HfFileSystemFileHfFileSystemStreamFilec ||n|j}|||d<d|vrtd|dkrt||f||d|St||f||d|S)Nraz/Appending to remote files is not yet supported.r)rrH)rrrr)r9r3rrrHrfs r:_openzHfFileSystem._opens$.#9ZZt  !#-F< $;;%&WXX X ??)$]4(]]V\]] ]#D$WTHWWPVWW Wr<c |||}t|tr(|j|j|jgn`|j|j|j |j |j |j | d| d||dS)NrHdeletecommit_messagecommit_description)rIrGrrFrHrrr3)rrrYrbatch_bucket_filesrZr3 delete_filerIrGrrFrHrSrr;)r9r3rHrfrs r:_rmzHfFileSystem._rms))$)BB m%C D D  I ( ()@-J\I] ( ^ ^ ^ ^ I ! !*7%-j'1&/%zz*:;;#)::.B#C#C "    =#:#:#<#<=====r<F recursivemaxdepthc f||}||||}t|tr0fd|D}j|j|nfd|D} d| D} d|d} | |rd nd z } | |d |dnd z } j|j|j j | |j | d | | d |d S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). > [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. r)rrrHcng|]1}||j2SrCisdirrr3rr3r9s r: z#HfFileSystem.rm..s<]]]tDJJW[L\L\]d''--2]]]r<rcng|]1}||j2SrCrrs r:rz#HfFileSystem.rm..s?ddddSWS]S]^bScScdT..t449dddr<c.g|]}t|S))rI)r)rrIs r:rz#HfFileSystem.rm..s$mmm|/\JJJmmmr<zDelete  z recursively rPNz up to depth rr)rGrFr operationsrHrrr)r expand_pathrrYrrrZ create_commitrGrFrrHrSrr;) r9r3rrrHrfrpathsr paths_in_reporrs ` r:rmzHfFileSystem.rms~6))$)BB   XX` aa m%C D D ]]]]u]]]F I ( ()@ ( P P P PddddedddMmm_lmmmJ.t...N  Annr AN H [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `list[Union[str, dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). r)rrHNc,g|]}|dk|SnamerCror3s r:rz#HfFileSystem.ls..s'777QvY$%6%61%6%6%6r<rcg|] }|d SrrCrrs r:rz#HfFileSystem.ls..s":":":1V9":":":r<)rr;_ls_treerr3rrr)r9r3rrrHrfrouts ` r:lszHfFileSystem.lss>4))$)BB &&(( 2$-SgSSFSSCC! 2 2 2 % 2%dD111 #dmDLL$6$6eRZee^dee%    7777c777C3xx1}}%dD111 2:ss":":c":":"::s5A!C "$BC  BC B4C  C c ||njjnd}||}|}|j}|r|nd}g} |jvr[|sXj|} | | g} |rdt fd| D} | r| \} ||krt| djvr| | dnIj| d} | | | fd| D| g}|r#t|trfd| D}|r| s|rN|rKtj | |z}|d s||ks|t| |vr|d n||/t%|dd }||z}fd | D} t)jD]5}|d zrj|d6jd| |d ||| nt|t0r#|j|j | }n4j|j|j |||j|j}|D]F}|d z|j z}t|t@r*||j!d|j"|j#|j$|j%|j&d}ngt|tNr||j!d|j$|j(|j)d}n4t|tTr|dd|j+|j%d}n |dd|j)d}|d}j,|g||t%|dd ||kr| |H| S)NFrrc0g|]}|ddk|fSrr directoryrCr path_infodepths r:rz)HfFileSystem._ls_tree..s/oooIiX^N_cnNnNneY'NnNnNnr<rc6g|]}|ddkdz|fS)rrrrrCrs r:rz)HfFileSystem._ls_tree..)s<!"!"!"(1'0'8K'G'G&+QY $:'G'G'Gr<cVg|]%}|d |d&S) last_commitNr)r)rrr9s r:rz)HfFileSystem._ls_tree..3s3$f$f$fQ}M]MeT\\!F)%<%.Ds6UUUQQvY-A-A+PSBS-T-TUqUUUr<T)rrrHrr)prefixr)rexpandrHrFfilersizerrblob_idlfsxet_hashrsecurity)rrrrrmtime uploaded_atrrrrrrtree_idr)rrrrr )-rrr;r2rextendrpopleftappendrrEosr3 commonprefixendswithr r7rrrlistrrjrrY_list_bucket_tree_with_foldersrZrlist_repo_treerGrHrFr(rrrrrrr$rr r)r  setdefault)r9r3rrrHrrr root_pathrcached_path_infosdirs_not_in_dircache dirs_to_visitdir_infodirs_not_expanded common_prefixcommon_path_depth cached_pathtreer cache_pathcache_path_info parent_pathrrs` @@r:rzHfFileSystem._ls_treesz'2KKTM]Mi9I9Iot ))$)BB &&((!& (/88a 4=  $ d 3  JJ( ) ) )#%   %oooo9Jooo!! $&3&;&;&=&=OE8'5H+<+<#F+4=@@0778HIIII04 hv>N0O-JJ'8999)00!"!"!"!"5F!"!"!"$ !#  gz-9[\\ g$f$f$f$fc$f$f$f! 2   HY !# 4 45IL]5] ^ ^ %--c225$ 11$.BDU(V(VVV"((---m44 '(3CIIKK(@(F(Fs(K(K% 11HUUUU#UUU#' #6#6==K"--kC.?@@= ))+t<<< !!+t444 MM#"+ $!)$/!) "   -)GHH ::!+(-'; y//!)!&'&*3+5 0") 0) 0 &_y~= i22" * ) &#,#4(}$-$6'0'<$-$6 ' 'OO :66 * ) &$-$6!*'0'< ''OO :66 * ! +#,#4'0'< ''OO!+ ! +'0'< ''O #ll?6+BCC  ((b99@@QQQ"3t99;;/55c::#u'8'8JJ/// r<rc\|j|||}i}|rd|dznd}g}|D]}|||st t |jjd| dz D]} tdt| |j } | j|vr || | || j<O| j 0|| j} | j | j | j kr | j | _ |std|d |d |S) z>8>v>>>r<withdirsc  ||dkrtd|||}|} |j|fd||d| |s d D n"|j|fi|} | dd kr| g zn d D nE#t $r8 |j|fd |i|dd kr|ii ni n#t $ri YnwxYwYnwxYwt } |s| S fd | DS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[list[str], dict[str, dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rT)rrrc*g|]}|ddk|SrrCrs r:rz%HfFileSystem.find.. s&BBBQ6k)A)Aq)A)A)Ar<rrrc i|] }|d| SrrCrs r:rz%HfFileSystem.find..s---A1V9a---r<rHrc"i|] }|| SrCrC)rrrs r:rz%HfFileSystem.find..s666D#d)666r<)rrr;rinforFileNotFoundErrorr) r9r3rr2rrrHrfrrnamesrs @r:findzHfFileSystem.finds@  HqLL:;; ;))$)BB &&(( .$-cgPXcc\bccC UBB#BBB&DId55f55 +4V+< +K+KykC''QT-----CC!"    49T??H???G6QQ*CCC$     "s  7L6666666 6s6B C !CC CCCCCpath1path2c |||}|||}t|tst|trtd|j|jko|j|jk}|rd|d|}|j|j|j|j| d|| ddt|j |j |jg n| |d |j5} | } d d d n #1swxYwYd|d|}|j| |j |j|j|j|j| d|| d || || d S)a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rz)Copy from/to buckets is not available yetzCopy z to rrrP)src_path_in_reporI src_revision)rGrFrHrrrrNpath_or_fileobjrIrGrrFrHrrr)rrrYrrFrGrrrHrSrrIopenread upload_filerrr;) r9r;r<rHrfresolved_path1resolved_path2 same_reporfcontents r:cp_filezHfFileSystem.cp_files{ **58*DD**58*DD n&D E E S :J J  S&&QRR R  $(@ @ u^E[_m_uEu   7U7777N I # #&.(2'0%zz*:NKK#)::.BB#G#G')7)D%3%@%3%< $    5$1HII #Q&&(( # # # # # # # # # # # # # # #7U7777N I ! ! '+8&.j(2'0%zz*:NKK#)::.B#C#C "    >#;#;#=#=>>> >#;#;#=#=>>>>>s'EE E c |j|fii|ddi}d|vr(|dtd|d|djSd|vr|dr|dSd|vr|dr|dStd |d |d ) ak Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last modified time of the file. rTrNz9'modified' is not implemented for repository paths like 'r$rr z'Cannot determined 'modified' for path 'z ' (info: ))r7rdate)r9r3rfr7s r:modifiedzHfFileSystem.modifiedQstyAA!@F!@M4!@!@AA D M"*)*mfj*m*m*mnnn &+ + __g_= d " "tM': " & &%&fPT&f&f_c&f&f&fgg gr<c  ||}||d|j|jnd}|jsddd t |t rd d<t |t r\|rZ|j|j |j |j d }i dt|j |j|j d  nt |t ra|}||| fd|j|D} | st)d| d nd |}|s||jvr||||jvr3fd|j|D} | st)d| d |s |r r d|j|j |j||j |j } | st)d| d} t |j |j |j d|j} t | t0r5| dz| jz| jd| j| j| j| j| jd n| dz| jzdd| j| jd |s fddD J S)aS Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance > (or `HfApi.get_bucket_paths_info()` or `HfApi.bucket_info()` for buckets) Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rrNFrr)rrrrr)rFrH)oidtitlerM)r r)rc,g|]}|dk|SrrCrs r:rz%HfFileSystem.info..s'OOO!QvY$=N=NA=N=N=Nr<c,g|]}|dk|SrrCrs r:rz%HfFileSystem.info..s'SSSa6dARARARARARr<)rrHrFrP)rIrNr6rrr c"i|] }|| SrCrC)rkrs r:rz%HfFileSystem.info..sGGG1c!fGGGr<) rr;rSrr3rrErlist_repo_commitsrGrFrHr' commit_idrR created_atrYrrrrget_paths_inforIrNr(rrrrrrr )r9r3rrHrfrrrr"out1 paths_inforrrs ` @r:r7zHfFileSystem.infojs,))$)BB &&((jj t/?/K4++QV  !N H#C -)KLL *%)M"-)KLL Q\ "i99!)]5LWdWm: ##1'19JQ\Qg$$$ 'E F F: H,,t,,K GGKG 1 1 1OOOOt}[9OOOD 2%dD111q'CCC,,t,,K %;dm#C#C $$$dm++SSSS4=#=SSS6)$5551g& H#+++#+#mBTB\!Y55!)!.&*3+5 6 "6)$555&qM >!+!)!*!#"/"=  )++ i22 )C). @ ) &#,#4(}$-$6'0'<$-$6  CC!*C). @ ! +#,#4'0'< C#HGGGG.FGGGC r<c |ddr|||j|fi|dS#t$rYdSwxYw)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFT)rSrr7OSError)r9r3rfs r:existszHfFileSystem.existssr  zz)U++ ,%%d+++ DId % %f % % %4   55 s9= A  A cd ||ddkS#t$rYdSwxYw)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rrrFr7r^r9r3s r:rzHfFileSystem.isdirsC 99T??6*k9 9   55  ! //cd ||ddkS#t$rYdSwxYw)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rrrFrarbs r:isfilezHfFileSystem.isfilesC 99T??6*f4 4   55 rccZ||}t|tr'|jd|jdt |j}n-t|j|j |j |j |j}| |r| ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. z /buckets/z /resolve/)rFrHrz/tree/r)rrrYrrZrr3r!rGrIrFrHrreplace)r9r3rurls r:rhzHfFileSystem.urls))$// m%C D D ]jj]-DjjuUbUgOhOhjjCC%*'1&/ C ::d   8++k8Q77C r<c |d}t|dhz }t|tt frt |dkrtj||f||d|St|r|}n-| |rtj |ddSt|ttfr3tj tj|dd}|t#|d}d}|} ||| } ||| d } ||  t-|| ||| d|jt|t r|jnd || |r|dSdS#|r|wwxYw) ad Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` or `HfApi.download_bucket_files` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. rHr)callbackoutfileT)exist_okNFwbrr)rh temp_filedisplayed_filename expected_size resume_sizeheaders _tqdm_bar)rSsetkeysrrrrraget_filerrrmakedirsrAr r3dirnamerBtellrr7set_sizer"rhr;r_build_hf_headerstqdmseekclose) r9rpathlpathrjrkrfrHunhandled_kwargs close_file initial_posresolve_remote_pathrprgs r:rvzHfFileSystem.get_file/sF$::j))v{{}}-- <(\<$@AA `SIYEZEZ]^E^E^$577#E5_8W__X^__ _ e   GG ZZ    K - - - -4 ec4[ ) ) ? K.. > > > > ?5$''GJllnn #///II %( ;;FC -((( HH0::<<==!#(+ 3355+5h +M+MW(--SW     LL % % %    z   s BH((Ic td)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.rr8s r: transactionzHfFileSystem.transactionms""LMMMr<c td)z@Begin write transaction for deferring files, non-context versionrrr8s r:start_transactionzHfFileSystem.start_transactionxs""LMMMr<cntt||j|j|ffSr`) make_instancerrrxryrtr8s r: __reduce__zHfFileSystem.__reduce__~s8 JJ     $ $ & &    r<czt|jt|jt|jdS)N)rrr)rrrrr8s r:rtz HfFileSystem._get_instance_states: ///78\/]/]$,T-F$G$G   r<r`)rNN)FNN)TFN)FFNNN)NFFFN)FNr4N)5r=r>r?r@ root_markerprotocolrAboolintrb classmethodrktuple ExceptionrrrErYrrrrrrrdictrrrrr$r%rrr/r1r:rJr rNr7r_rrerhrrvpropertyrrrrtrrs@r:rrs@@DKH $#'!%#' <<<*<czD < $J < D[ <<<<<<2     [ TT'*T69DjT tY%% &TTTT&44dI>>sTz>t>>>>& ## 0>0>0>0>* 0> * 0> 0>0>0>0>f]a);););!%);7;);ORUYz); cDcN" #););););\ ##'#KKKK K * K D[ K*KKKKZ&&&)&6:& *|+ ,&&&&P77(5d3iQUVYQZAZ;[2\777777 ???d ?S ??????& $#;7;7;7*;7 ;7  ;7  ;7*;7 cT#tCH~-. .;7;7;7;7z:?:?S:?:?d :?X\:?:?:?:?xhShxhhhh2llltlsTzl_cdgildl_mllll\2$$4/@< < < < < < < |NNXNNNN           r<r) metaclassceZdZddedededzffd ZfdZdeded efd Z dd Z dd e d dfdZ dfd Z d efdZxZS)rNfsr3rHc ( ||||_n>#t$r1}d|ddvrt|d|d}~wwxYwt j||jfi||dS)NrwrrPB. Make sure the repository and revision exist before writing data.)rrr8rSrarbr;)r9rr3rHrfrrgs r:rbzHfFileSystemFile.__init__s !#!I!ID      fjj,,,,']]]    T/99;;FFvFFF s A,AAcht|dsdStSNr)hasattrra__del__r9rgs r:rzHfFileSystemFile.__del__s.t_--  Fww   r<startendr4cdd|d|dz i|jj}|}t d||t j}t||jS)Nrangebytes=-rGETrrr) rrr{rhr,rHF_HUB_DOWNLOAD_TIMEOUTr+rI)r9rrrrrhrs r: _fetch_rangezHfFileSystemFile._fetch_rangesy /e//cAg// gl,,.. hhjj Wi>_ ` ` `Ayr<c<tjdd|_dS)Nzhffs-F)rr)tempfileNamedTemporaryFilernr8s r:_initiate_uploadz!HfFileSystemFile._initiate_uploads!4GERRRr<Ffinalc \|jd|j}|j||r[|jt |jtrC|j j |jj |jj |jjfgn|j j |jj |jj|jj|j j|jj|jj|jd|jdt/j|jj |j |jdSdS)Nr)addrrr@r)bufferr}rCrnwriter~rrrYrrrrZrr3rDrIrGrrFrHrfrSrremoverr;)r9rblocks r: _upload_chunkzHfFileSystemFile._upload_chunks    "" U###   N " " "$,.LMM  //&08KTM_Md7e6f0 (($(N$7!%!3!@ .6'-"0:!/8#';??3C#D#D'+{7K'L'L)    Idn) * * * G $ $'1133 %     %  r<rPc\|jdkr||dkrx|jdkrm|j|jdd5}|}|xjt |z c_|cdddS#1swxYwYt|S)zRead remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rNrPr)r)rlocrrBr3rCrra)r9lengthrHrrgs r:rCzHfFileSystemFile.reads 9  &.FbLLdhRSmmdi!<< ffhhCHH$                ww||F###s3BB Bc@|j|jSr`rrhr3r8s r:rhzHfFileSystemFile.urlw{{49%%%r<r`r)FrP)r=r>r?rrArbrrbytesrrrrrCrhrrs@r:rrs  < s cDj      !!!!! #CESSSS4D4 $ $ $ $ $ $&S&&&&&&&&r<rc eZdZ ddededededzd ed ef fd Zdd ed efdZddefdZdde e dede fdZ defdZ fdZ dZdZxZS)rrNrnonerr3rrHr cache_typec p|dkrtd||dkrtd|d|vrtd|d ||||_nB#t$r5}d|d d vrt|d |Yd}~nd}~wwxYw|jdd |_tj||jf|||d |d|_ |t|_ d|_ t|_dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got rz?HfFileSystemStreamFile only supports cache_type='none' but got rz;HfFileSystemStreamFile only supports reading but got mode='r$rrrPr)rrrrr)rrrr8rSr;detailsrarbresponser _exit_stack_stream_iterator bytearray_stream_buffer) r9rr3rrHrrrfrrgs r:rbzHfFileSystemStreamFile.__init__s ??fZdffgg g   k_ikkll l $;;b[_bbbcc c !#!I!ID      fjj,,,,']]]-,,,,  !% 2 < < > >MM  ",,.. 59j]g  kq   04  $;;8<'kks A'' B&1+B!!B&rwhencecb|dkr|dkrdS||jkr|dkrdStd)NrrzCannot seek streaming HF file)rr)r9rrs r:r}zHfFileSystemStreamFile.seeks? !88! F $(??v{{ F8999r<rPrch|j|d} |j|jdS||j|}|xjt |z c_|S#t $r<|j|j|r|d}YnwxYw)zRead the remote file. If the file is already open, we reuse the connection. Otherwise, open a new connection and read from it. If reading the stream fails, we retry with a new connection. NFTr<)r_open_connectionr_read_from_streamrrrr~)r9r retried_oncers r:rCzHfFileSystemStreamFile.reads =  ! ! # # #  $ $=(D,A,I3,,T-BFKKCHH$  $ $ $=,M'')))%%'''#  $ $sA*9A**AB0/B0iteratorr4c|dkrdS|dkrVt|j}|j|D]}||t |S|t |jkr(t |jd|}|jd|=|St|j}|j|D]|}|t |z }|t |kr||=||d||j||dt |S)zRead up to `length` bytes from stream buffer and stream. If length < 0, read until EOF. If EOF is reached before length, fewer bytes may be returned. rr<N)rrrr rr)r9rrbufchunkresultneeds r:rz(HfFileSystemStreamFile._read_from_stream!se Q;;3 A::D/00C   % % ' ' '! " " 5!!!!::  S,-- - -4.ww788F#GVG,M+,, !!###  ECHH$Dc%jj   5!!!! 5$<(((#**5<888Szzr<c@|j|jSr`rr8s r:rhzHfFileSystemStreamFile.urlCrr<ct|dsdS|jtSr)rrr~rarrs r:rzHfFileSystemStreamFile.__del__FsCt_--  F    ww   r<cZt|j|j|j|j|jjffSr`)reopenrr3r blocksizecacherr8s r:rz!HfFileSystemStreamFile.__reduce__Ms$DIt~tzWWWr<c|jd|_|}|jj}|jdkrd|jd|d<|j td||tj |_ t|j n0#t$r#}|j jdkr d|_ Yd}~dSd}~wwxYw|j |_dS) z%Open a connection to the remote file.NrrrRangerri)rrrrhrrr{rr enter_contextr-rrrr+r status_code iter_bytes)r9rhrrrs r:rz'HfFileSystemStreamFile._open_connectionPs !!### $hhjj',0022 8a<<3333GG (66 !9          . . . .   z%,, $    !% 8 8 : :s'B<< C)C$#C$$C))rNrr)rr)r=r>r?rrArrbr}rCrrrrhrrrrrs@r:rrs_ # !*!* !*!* !* * !*  !*!*!*!*!*!*!*F:::S::::$$3$$$$8  (5/ 3 PU    D&S&&&&!!!!!XXX;;;;;;;r<rrHr4cLtj|r|nt|Sr`)r#match safe_quoters r:rUrUos%28BB \88 S[H\H\\r<sc$t|dS)NrP)safe)r)rs r:rrss    r<r3rc|}t|tr|d}n5t|tr|d}nt|tr|d}t ||)Nz (repository not found)z (revision not found)z (invalid repository id))rrr r*r8)r3rmsgs r:rrws~ C#.//0... C. / /0,,, C* + +0/// C c)r<rrrrc4|||||S)Nr)rB)rr3rrrs r:rrs 774dzj7 Q QQr<cl||i|}|D]\}}t||||Sr`)rurv)rdrerfr~rrrs r:rrsP d f  B+1133''kD+&&&& Ir<)Zrrrl collectionsrcollections.abcrr contextlibrcopyr dataclassesrr r itertoolsr pathlibr r typingrrr urllib.parserrfsspechttpxfsspec.callbacksrrr fsspec.configr fsspec.utilsrrPr _commit_apirrerrorsrrrrr file_downloadr!r"hf_apir#r$r%r&r'r(r)utilsr*r+r,r-r.utils.insecure_hashlibr/r1rErYrrAbstractFileSystemr\rBr^rspecAbstractBufferedFilerrrArUrrrrrrhffsrCr<r:rs ........ (((((((('''''''''''''''''''''''''' JJJJJJJJJJ&&&&&&######CCCCCCCC0///////vvvvvvvvvvvvvvvvvvjjjjjjjjjjjjjj'''''' 6666666 6 &&&&&)A&& &0 00000%=00 0D233 c33322222l222j@ @ @ @ @ 6,@ @ @ @ F H&H&H&H&H&v{7H&H&H&VR;R;R;R;R;V[=R;R;R;j]C]C]]]]##**)d*:*x****R|R3RcRsRPSRRRR|~~r<