U Mf @s&dZddlmZddlZddlZddlmZddlZddlZddl Z ddl m Z m Z m Z mZmZmZddlZddlZddlmZddlZddlmZmZmZmZmZmZmZmZmZddl m!Z!m"Z#m$Z$m%Z%m&Z&ddl'Z'ddl(Z(dd l)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0dd l1m2Z2dd l3m4Z4dd l5m6Z6dd l7m8Z8ddl9m:Z:ddl;me>?de@dZAede*dZBejCGdddZDejCGdddeeZEdddddZFeddddd ZGedddd!d ZGd"d"dd#d ZGdd$d%d&d'ZHedxd)ddd*d+d,ZIedydddd*d-d,ZIdzd/dd"d*d0d,ZId1d2ZJd3ddd4d5ZKe6ed?ZLddd@dAdBZMdCdDdEdFdGdHZNd;dIdJdKdLZOe6eeZdZUdZded<ded<ded<ded<d Zd ed <d S) IOArgsz? Return value of io/common.py:_get_filepath_or_buffer. str | BaseBufferfilepath_or_bufferstrencodingmoder compressionFbool should_closeN)__name__ __module__ __qualname____doc____annotations__r1r7r74/tmp/pip-unpacked-wheel-eb6vo0j3/pandas/io/common.pyr)Ds r)c@szeZdZUdZded<ded<ejedZded<d Z d ed <d Z d ed <d dddZ ddddZ dd dddZ dS) IOHandlesau Return value of io/common.py:get_handle Can be used as a context manager. This is used to easily close created buffers and to handle corner cases when TextIOWrapper is inserted. handle: The file handle to be used. created_handles: All file handles that are created by get_handle is_wrapped: Whether a TextIOWrapper needs to be detached. z IO[AnyStr]handlerr/)default_factoryzlist[IO[bytes] | IO[str]]created_handlesFr0 is_wrappedis_mmapNonereturnc Csz|jr8t|jtst|j|j|j|jz|jD] }| q@Wnt t fk rhYnXg|_d|_dS)z Close all created buffers. Note: If a TextIOWrapper was inserted, it is flushed and detached to avoid closing the potentially user-created buffer. FN) r= isinstancer:r AssertionErrorflushdetachr<removecloseOSError ValueError)selfr:r7r7r8rGgs   zIOHandles.closezIOHandles[AnyStr]cCs|SNr7rJr7r7r8 __enter__{szIOHandles.__enter__r )argsrAcGs |dSrK)rG)rJrNr7r7r8__exit__~szIOHandles.__exit__N)r2r3r4r5r6 dataclassesfieldlistr<r=r>rGrMrOr7r7r7r8r9Qs   r9objectr0)urlrAcCst|tsdSt|jtkS)z Check to see if a URL has a valid protocol. Parameters ---------- url : str or unicode Returns ------- isurl : bool If `url` has a valid protocol return True otherwise False. F)rBr, parse_urlscheme _VALID_URLSrTr7r7r8is_urls rYr,)r+rAcCsdSrKr7r+r7r7r8 _expand_usersr[cCsdSrKr7rZr7r7r8r[szstr | BaseBufferTcCst|trtj|S|S)a] Return the argument with an initial component of ~ or ~user replaced by that user's home directory. Parameters ---------- filepath_or_buffer : object to be converted if possible Returns ------- expanded_filepath_or_buffer : an expanded filepath or the input if not expandable )rBr,ospath expanduserrZr7r7r8r[s  r?)headerrAcCst|trtddS)NzPassing a bool to header is invalid. Use header=None for no header or header=int or list-like of ints to specify the row(s) making up the column names)rBr0 TypeError)r_r7r7r8validate_header_args ra.r)r+convert_file_likerAcCsdSrKr7r+rbr7r7r8stringify_pathsrdcCsdSrKr7rcr7r7r8rdsFzFilePath | BaseBufferTcCs2|st|rtt|St|tjr*|}t|S)a Attempt to convert a path-like object to a string. Parameters ---------- filepath_or_buffer : object to be converted Returns ------- str_filepath_or_buffer : maybe a string version of the object Notes ----- Objects supporting the fspath protocol (python 3.6+) are coerced according to its __fspath__ method. Any other object is passed through unchanged, which includes bytes, strings, buffers, or anything else that's not even path-like. )r$rr'rBr\PathLike __fspath__r[rcr7r7r8rds    cOsddl}|jj||S)z` Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of the stdlib. rN)urllib.requestrequesturlopen)rNkwargsurllibr7r7r8risrizFilePath | BaseBuffercCs$t|to"tt|o"|d S)zR Returns true if the given URL looks like something fsspec can handle )zhttp://zhttps://)rBr,r0_RFC_3986_PATTERNmatch startswithrXr7r7r8 is_fsspec_urls    rocompression_optionsr+)rputf-8rrr)r+r-r/r.storage_optionsrAc Cst|}t|\}}t||}|rJt|drJd|krJtjdttdd}t||d}d|kr|dkr|d krt|d |t |}d |krd|kr|d7}t |t r&t |r&|pi}d dl }|jj||d }t|0} | jdd} | dkrddi}t| } W5QRXt| ||d|dSt|rPt |t s@t|drX|dd}|drp|dd}td} g} z&tdd dlm}m}||tg} Wntk rYnXz$| j |fd|i|pi }Wn^t!| k r<|dkrddi}nt|}d|d<| j |fd|i|p.i }YnXt|||d|dS|r^t"dt |t t#t$j$frtt%|||d|dSt|dst|dsdt&|}t"|t|||d|dS)a If the filepath_or_buffer is a url, translate and return the buffer. Otherwise passthrough. Parameters ---------- filepath_or_buffer : a url, filepath (str, py.path.local or pathlib.Path), or buffer {compression_options} .. versionchanged:: 1.4.0 Zstandard support. encoding : the encoding to use to decode bytes, default is 'utf-8' mode : str, optional storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc., if using a URL that will be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error will be raised if providing this argument with a local path or a file-like buffer. See the fsspec and backend storage implementation docs for the set of allowed keys and values .. versionadded:: 1.2.0 ..versionchange:: 1.2.0 Returns the dataclass IOArgs. writebzDcompression has no effect when passing a non-binary object as input.) stacklevelN)methodw)bz2xz)zutf-16zutf-32z( will not write the byte order mark for tr)headerszContent-EncodinggziprwT)r+r-r/r1r.zs3a://zs3://zs3n://fsspecZbotocore) ClientErrorNoCredentialsErrorr.Zanonz?storage_options passed with file object or non-fsspec file pathFreadz)Invalid file path or buffer object type: )'rdget_compression_methodinfer_compressionhasattrwarningswarnRuntimeWarningr#dictUnicodeWarningrBr,rYrgrhRequestrir|getrrr)rorCrnreplacer!Zbotocore.exceptionsrrPermissionError ImportErroropentuplerIbytesmmapr[type)r+r-r/r.rscompression_methodZ fsspec_moderkZreq_inforeqcontent_encodingreaderr~Zerr_types_to_retry_with_anonrrZfile_objmsgr7r7r8_get_filepath_or_buffers%              r)r]rAcCsddlm}td||S)z converts an absolute native path to a FILE URL. Parameters ---------- path : a path in native format Returns ------- a valid FILE URL r) pathname2urlzfile:)rgrr)r]rr7r7r8file_path_to_urls rz.gzz.bz2.zipz.xzz.zst)r}ryziprzzstdz"tuple[str | None, CompressionDict])r/rAc Cs`t|trPt|}z|d}WqXtk rL}ztd|W5d}~XYqXXni}|}||fS)a Simplifies a compression argument to a compression method string and a mapping containing additional arguments. Parameters ---------- compression : str or mapping If string, specifies the compression method. If mapping, value at key 'method' specifies compression method. Returns ------- tuple of ({compression method}, Optional[str] {compression arguments}, Dict[str, Any]) Raises ------ ValueError on mapping missing 'method' key rwz.If mapping, compression must have key 'method'N)rBrrpopKeyErrorrI)r/compression_argsrerrr7r7r8rs r str | None)r+r/rAcCs|dkr dS|dkrZt|dd}t|ts.dStD]\}}||r6|Sq6dS|tkrf|Sddgtt}d|d|}t|dS)a/ Get the compression method for filepath_or_buffer. If compression='infer', the inferred compression method is returned. Otherwise, the input compression method is returned unchanged, unless it's invalid, in which case an error is raised. Parameters ---------- filepath_or_buffer : str or file handle File path or object. {compression_options} .. versionchanged:: 1.4.0 Zstandard support. Returns ------- string or None Raises ------ ValueError on invalid compression specified. NZinferT)rbUnrecognized compression type: z Valid compression types are ) rdrBr,_compression_to_extensionitemslowerendswithsortedrI)r+r/ extensionZvalidrr7r7r8rs$   rz Path | strcCs&t|j}|s"td|ddS)z Check if parent directory of a file exists, raise OSError if it does not Parameters ---------- path: Path or str Path to check parent directory of z1Cannot save file into a non-existent directory: ''N)r parentis_dirrH)r]rr7r7r8check_parent_directory/s r)r-r/ memory_maperrorsrszLiteral[False]zIOHandles[bytes]) path_or_bufr.r-r/ris_textrrsrAcCsdSrKr7rr.r-r/rrrrsr7r7r8 get_handle>s r)r-r/rrrrsz Literal[True]zIOHandles[str]cCsdSrKr7rr7r7r8rMs rTz!IOHandles[str] | IOHandles[bytes]cCs|pd}t||r"d|kr"|d7}t|t|tr@t|t|||||d}|j} t| ||j |j ||j dt k\} }} t| t} t |j } | d}d|kr| rtt| |r|dkr|j dd|_ n |dkrd|j kr|j d7_ |d krJ| r.t| tsttjf| |j d | } ntjf| |j d | } n0|d krptj| fd |j i| } n |dkrt| |j f| } | j dkrz| | | } t| dkr| | } n,t| dkrtd|ntd| n|dkrt| |j } nl|dkrhtd}d|j kr>d|jf| i}nd|jf| i}|j| fd |j i|} nd|}t|t| trt| | nNt| tr|j rd|j krt| |j |j |dd} n t| |j } | | d}|s|j dkrt| t rt!| |j d} nP|rf|s0t| |j rft"t#| |j |dd} | | t|jtpb|j$ }d|j krt%| dst&dt'|jd| (|j$rt|jtrt| |jt)| | |||j dS) aw Get file handle for given path/buffer and mode. Parameters ---------- path_or_buf : str or file handle File path or object. mode : str Mode to open path_or_buf with. encoding : str or None Encoding to use. {compression_options} .. versionchanged:: 1.0.0 May now be a dict with key 'method' as compression mode and other keys as compression options if compression mode is 'zip'. .. versionchanged:: 1.1.0 Passing compression options as keys in dict is now supported for compression modes 'gzip', 'bz2', 'zstd' and 'zip'. .. versionchanged:: 1.4.0 Zstandard support. memory_map : bool, default False See parsers._parser_params for more information. is_text : bool, default True Whether the type of the content passed to the file/buffer is string or bytes. This is not the same as `"b" not in mode`. If a string content is passed to a binary file/buffer, a wrapper is inserted. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. storage_options: StorageOptions = None Passed to _get_filepath_or_buffer .. versionchanged:: 1.2.0 Returns the dataclass IOHandles rqru)r-r/r.rsrwrrrr{r&r})filenamer.)fileobjr.ryr.rrzZero files found in ZIP file z9Multiple files found in ZIP file. Only one file per ZIP: rz zstandardZdctxZcctxrr-rnewlineFrb)r-rz1Expected file path name or file-like object, got z type)r:r<r=r>r/)*_is_binary_modecodecslookuprBr, lookup_errorrr+_maybe_memory_mapr-r.r/rrrrrrCr}GzipFileryBZ2File _BytesZipFileappendnamelistlenrrIr r!ZstdDecompressorZZstdCompressorr_BytesIOWrapperr _IOWrapperr1rr`rreverser9)rr.r-r/rrrrsZioargsr:handlesZis_pathrZ zip_namesrZ open_argsrr=r7r7r8r\s6                        csfeZdZdZdddddfdd Zd d Zd d Zd dfdd ZfddZe ddZ Z S)ra  Wrapper for standard library class ZipFile and allow the returned file-like handle to accept byte strings via `write` method. BytesIO provides attributes of file-like object and ZipFile.writestr writes bytes strings into a member of the archive. Nz1FilePath | ReadBuffer[bytes] | WriteBuffer[bytes]r,r)filer. archive_namec sB|dd}||_d|_dtji}||tj||f|dS)Nrur&r/)rrmultiple_write_bufferzipfile ZIP_DEFLATEDupdatesuper__init__)rJrr.rrjZ kwargs_zip __class__r7r8rhs    z_BytesZipFile.__init__cCs<t|jtjtfr8t|j}|jdkr2|djS|jSdS)z If an explicit archive_name is not given, we still want the file inside the zip file not to be named something.zip, because that causes confusion (GH39465). rr&N) rBrr\rer,r suffix with_suffixname)rJrr7r7r8infer_filename|s    z_BytesZipFile.infer_filenamecCs2|jdkr"t|trtnt|_|j|dSrK)rrBrrrrt)rJdatar7r7r8rts z_BytesZipFile.writer?r@c sR|jdks|jjrdS|jp&|p&d}|jt||jW5QRXdS)Nr)rclosedrrrwritestrgetvalue)rJrrr7r8rDs z_BytesZipFile.flushcs|tdSrK)rDrrGrLrr7r8rGsz_BytesZipFile.closecCs |jdkSrK)fprLr7r7r8rsz_BytesZipFile.closed)N) r2r3r4r5rrrtrDrGpropertyr __classcell__r7r7rr8r^s   rc@sbeZdZdZddddddd d Zdd d d ZddddZddddddZddddZdS) _MMapWrappera Wrapper for the Python's mmap class so that it can be properly read in by Python's csv.reader class. Parameters ---------- f : file object File object to be mapped onto memory. Must support the 'fileno' method or have an equivalent attribute rqstrictTr r,r0)fr-rdecodecCsn||_||_t||d|_||_i|_dD]"}t||s>q.t|||j|<q.t j | dt j d|_ dS)Nr)seekablereadabler)access) r-rrgetincrementaldecoderdecoderr attributesrgetattrrfilenoZ ACCESS_READ)rJrr-rr attributer7r7r8rs z_MMapWrapper.__init__rcs$jkrfddStjS)Ncs jSrK)rr7rrJr7r8z*_MMapWrapper.__getattr__..)rrrrJrr7rr8 __getattr__s z_MMapWrapper.__getattr__r@cCs|SrKr7rLr7r7r8__iter__sz_MMapWrapper.__iter__intz str | bytes)sizerAcCsD|j|}|jr@|jdkr@|dkp.t||k}|jj||dS|S)Nrqr)final)rrrr-rr)rJrcontentrr7r7r8rs  z_MMapWrapper.readcCs,|j}|j|}|dkr"t|dS)Nr& )rreadlinerr StopIterationlstrip)rJZnewbytesrr7r7r8__next__s   z_MMapWrapper.__next__N)rqrT)r) r2r3r4r5rrrrrr7r7r7r8rs rc@sReZdZddddZddddZd d d d Zd d d dZd d ddZdS)rrbuffercCs ||_dSrKr)rJrr7r7r8rsz_IOWrapper.__init__r,rcCs t|j|SrKrrrr7r7r8rsz_IOWrapper.__getattr__r0r@cCst|jdr|jSdS)NrT)rrrrLr7r7r8rs  z_IOWrapper.readablecCst|jdr|jSdS)NrT)rrrrLr7r7r8rs  z_IOWrapper.seekablecCst|jdr|jSdS)NwritableT)rrrrLr7r7r8rs  z_IOWrapper.writableN)r2r3r4rrrrrr7r7r7r8rs rc@s>eZdZddddddZdddd Zdd d d ddZdS)rrqzStringIO | TextIOBaser,)rr-cCs||_||_d|_dS)Nr)rr-overflow)rJrr-r7r7r8rsz_BytesIOWrapper.__init__)attrcCs t|j|SrKr)rJrr7r7r8rsz_BytesIOWrapper.__getattr__rz int | Noner)nrAcCst|jdk st|j||j}|j|}|dksH|dksH|t|krRd|_|S|d|}||d|_|SdS)Nrr)rrCrencoder-rr)rJrZ bytestringZcombined_bytestringZ to_returnr7r7r8rs  z_BytesIOWrapper.readN)rq)r)r2r3r4rrrr7r7r7r8rs rr*z/tuple[str | BaseBuffer, bool, list[BaseBuffer]])r:rr-r.rrrAc Csg}|t|dpt|tM}|s*|||fSt|trh|rTd|krTt||||dd}n t||}||zttt ||||}W5t|D] }|qX|||||fS)zTry to memory map file/buffer.rrur&r) rrBr,rrreversedrGrrr)r:rr-r.rrrwrappedr7r7r8r"s$          rc CsHd}t|}t|ts|Sztj|}Wnttfk rBYnX|S)zTest whether file exists.F)rdrBr,r\r]existsr`rI)r+r r7r7r8 file_existsJs r )r:r.rAcCsVd|ksd|krd|kStjtjtjf}tt||r:dSt|tpTdt|d|kS)z+Whether the handle is opened in binary moder{ruFr.) r StreamWriter StreamReaderStreamReaderWriter issubclassrrB_get_binary_io_classesr)r:r.Z text_classesr7r7r8rXsrztuple[type, ...]r@c CsHttf}tddd}|dk rD|d}|t|f7}W5QRX|S)z!IO classes that that expect bytesrignorerNr)rrr!rZ stream_readerr)Zbinary_classesrrr7r7r8rms  r).).)F)rqNrrN)^r5 __future__rryr collectionsrrP functoolsr}iorrrrrr rr\pathlibr retypingr r r rrrrrr urllib.parserrrUrrrrrZpandas._typingrrrrrrrZ pandas.compatr Zpandas.compat._optionalr!Zpandas.util._decoratorsr"Zpandas.util._exceptionsr#Zpandas.core.dtypes.commonr$Zpandas.core.shared_docsr%setrWdiscardcompilerlr'Z dataclassr)r9rYr[rardrirorrrrrrrZipFilerIteratorrrrrr r lru_cacherr7r7r7r8s    , $          0 "  8 #:$$$EC(