U
    +ifI                     @   s  d Z ddlZddlZddlZddlmZ ddlZddlZddlZddlZddl	m
Z
 ddlZddlZddlmZmZmZ ddlZddlZddlZddlmZmZ ddlmZ ddlm  mZ ddlZddlm Z m!Z!m"Z" ddl#m$Z% ddl&m  m'Z( dd	l)m*Z*m+Z+ ej,d
Z-ej.dddd Z/dzddZ0dd Z1dd Z2dd Z3dd Z4dd Z5dd Z6dd Z7dd  Z8ej9:e;Z<ej9=e<d!d"Z>eej9=e>d#d$Z?ej.e?d%d&d" Z@d'd( ZAd)d* ZBd+d, ZCej,Dd-ejEeBd.d/ejEejFd0d/ejEejejFejGd1d2d/ejEejejFd3d1d4d/ejEejejFd5d1d6d/gej,d7d8d9 ZHd:d; ZId<d= ZJd>d? ZKej.d@dA ZLG dBdC dCZMG dDdE dEZNej,DdFdGgdHe dIdJdKgfdLejOPdIdJdKgdMdNdOggfgdPdQ ZQdRdS ZRej,DdTdUdVdWgdXdY ZSeTdZd[d\ ZUG d]d^ d^ejVZWd_d` ZXdadb ZYdcdd ZZdedf Z[ej,Ddgdhdidgedjdkdkdlgdmdn Z\dodp Z]dqdr Z^ej_dsdt Z`ej,DduejaejGgdvdw Zbdxdy ZcdS ){a<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    N)partial)Path)catch_warningsfilterwarningssimplefilter)get_lzma_fileis_platform_little_endian)import_optional_dependency)IndexSeriesperiod_range)DayMonthEndz1ignore:Timestamp.freq is deprecated:FutureWarningmodule)scopec               
   C   s>   ddl m}  t " tddtd |  W  5 Q R  S Q R X d S )Nr   create_pickle_dataignorez The 'freq' argument in Timestamp)category)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r   FutureWarningr    r   N/home/mars/bis/venv/lib/python3.8/site-packages/pandas/tests/io/test_pickle.pycurrent_pickle_data<   s      r   c                 C   s   t |trt||  d S |dr6tj}|| | n`|dkrv|tjkrX| tjksttq| |ksdt| j	|j	kstn t
td| dtj}|| | d S )NZsp_	timestampZassert_Z_equal)
isinstancer
   tmassert_index_equal
startswithZassert_equalpdZNaTAssertionErrorfreqgetattrZassert_almost_equal)resultexpectedtypversion
comparatorr   r   r   compare_elementL   s    


r'   c           
      C   sr   t |} t }|  D ]T\}}| D ]B\}}| | | }d| d| }	||	|d }	|	|||| q(q| S )NZcompare__r'   )r   read_pickleglobalsitemsget)
dataZvfr%   mr$   dvdtr"   r#   r&   r   r   r   compare_   s    
r1   c                 C   s   t | | | jj|jjks t| jjjr.tt | dk|dk | jj}|td tdksbt|tjdd }t	|tjst|tjdddkst|tjdd }t	|tjst|tjdddkstd S )Nr         )hours)daysr4   )nanoseconds)r5   r6   )
r   assert_series_equalindexr    r   	normalizer   r   Z	Timedeltar   )r"   r#   r$   r%   r    resr   r   r   compare_series_tsq   s    r;   c                 C   s   t | | d S Nr   r7   r"   r#   r$   r%   r   r   r   compare_series_dt_tz   s    r?   c                 C   s   t | | d S r<   r=   r>   r   r   r   compare_series_cat   s    r@   c                 C   s   t | | d S r<   r   assert_frame_equalr>   r   r   r   compare_frame_dt_mixed_tzs   s    rC   c                 C   s   t | | d S r<   rA   r>   r   r   r   compare_frame_cat_onecol   s    rD   c                 C   s   t | ||| d S r<   )rD   r>   r   r   r   compare_frame_cat_and_float   s    rE   c                 C   sV   t | | t| jtst| jt ks,t| jdks:tt | d|d d S )NMr3   )r   r   r   r    r   r   Zfreqstrshiftr>   r   r   r   compare_index_period   s
    rH   r-   legacy_pickle*z*.pickle)paramsc                 C   s
   || j S r<   )param)requestdatapathr   r   r   rI      s    c              	   C   sR   t  std tjtj|}tdd td t	| || W 5 Q R X d S )Nz"known failure on non-little endianT)recordr   )
r   pytestskipospathbasenamedirnamer   r   r1   )r   rI   r%   r   r   r   test_pickles   s    
rV   c              	   C   s*   t |d}tj| |dd W 5 Q R X d S NwbprotocolopenpickledumpobjrS   fhr   r   r   python_pickler   s    rc   c              
   C   s6   t | d"}|d t|W  5 Q R  S Q R X d S )Nrbr   )r]   seekr^   load)rS   rb   r   r   r   python_unpickler   s    
rg   pickle_writerpython)idZpandas_proto_defaultrZ   Zpandas_proto_highest   Zpandas_proto_4   Zpandas_proto_5z5ignore:The 'freq' argument in Timestamp:FutureWarningc                 C   s   | }|  D ]\}}|  D ]\}}tjtfD ]}t }||| t|}	t|	|| t|}	t|	|| t	|dd}
||| |

d W 5 Q R X t	|dd}
t|
}	|

d W 5 Q R X t|	|| W 5 Q R X q.qqd S )NrX   )moder   rd   )r+   r   	to_picklerc   r   ensure_cleanr)   r'   rg   r]   re   )r   rh   r-   r$   r/   r0   r#   writerrS   r"   handler   r   r   test_round_trip_current   s"    




rr   c                  C   s(   t  } t | jtj}t | | d S r<   )r   makeDataFrameround_trip_pathlibrn   r   r)   rB   dfr"   r   r   r   test_pickle_path_pathlib   s    rw   c                  C   s(   t  } t | jtj}t | | d S r<   )r   rs   Zround_trip_localpathrn   r   r)   rB   ru   r   r   r   test_pickle_path_localpath   s    rx   c              	   C   st   t t( tdt tj| dddddd W 5 Q R X t t( tdt tj| dddddd W 5 Q R X d	S )
ap  

    Generated with

    >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [0, 0, 1, 1]}).to_sparse()
    >>> df.to_pickle("pandas/tests/io/data/pickle/sparseframe-0.20.3.pickle.gz",
    ...              compression="gzip")

    >>> s = df['B']
    >>> s.to_pickle("pandas/tests/io/data/pickle/sparseseries-0.20.3.pickle.gz",
    ...             compression="gzip")
    r   ior-   r^   zsparseseries-0.20.3.pickle.gzgzipcompressionzsparseframe-0.20.3.pickle.gzN)r   Zassert_produces_warningr   r   DeprecationWarningr   r)   )rN   r   r   r   test_legacy_sparse_warning   s    

r~   c                   C   s   dt d dS )N__
   z	__.pickle)r   Zrandsr   r   r   r   get_random_path  s    r   c                   @   sf   e Zd Zdd ej D Zdd Zdd Ze	j
ddd	d
dgdd Zdd Zdd Zdd ZdS )TestCompressionc                 C   s   i | ]\}}||qS r   r   ).0r|   extr   r   r   
<dictcomp>#  s     zTestCompression.<dictcomp>c              
   C   s  |d krt || d S |dkr.t|d}n|dkrDt|d}n~|dkrtj|dtjd}|	|t
j| W 5 Q R X nB|dkrt |d}n,|dkrtd|d	}nd
| }t||dkrt|d$}| |	|  W 5 Q R X W 5 Q R X d S )Nrz   wbz2zipr{   xzZzstdZ	zstandardrX   zUnrecognized compression type: rd   )shutilcopyfilerz   r]   r   BZ2FilezipfileZipFileZIP_DEFLATEDwriterR   rS   rT   r   r	   
ValueErrorread)selfZsrc_path	dest_pathr|   fmsgrb   r   r   r   compress_file'  s&     
zTestCompression.compress_filec                 C   s   |}|d }|d }t |}t |t}t  }|j||d t j||d*}	t|d}
|
|	  W 5 Q R X W 5 Q R X tj	|d d}t 
|| W 5 Q R X W 5 Q R X d S )N.compressed.rawr{   rX   )r   ro   rs   rn   decompress_filer]   r   r   r   r)   rB   )r   r|   r   basepath1path2p1p2rv   r   rb   df2r   r   r   test_write_explicit?  s    "z#TestCompression.test_write_explicitr|    NonebadZ7zc              
   C   sJ   t jtdd2 t|}t }|j||d W 5 Q R X W 5 Q R X d S )NzUnrecognized compression type)matchr{   )rP   Zraisesr   r   ro   rs   rn   )r   r|   r   rS   rv   r   r   r   test_write_explicit_badT  s    z'TestCompression.test_write_explicit_badc                 C   s   |}|| }|d }| j | }t|}t|p}t }	|	| tj||d*}
t|d}|	|

  W 5 Q R X W 5 Q R X tj|d d}t|	| W 5 Q R X W 5 Q R X d S )Nr   r{   rX   )_extension_to_compressionr,   lowerr   ro   rs   rn   r   r]   r   r   r   r)   rB   )r   compression_extr   r   r   r   r|   r   r   rv   r   rb   r   r   r   r   test_write_infer[  s    
"z TestCompression.test_write_inferc           
   
   C   s   |}|d }|d }t |\}t |F}t  }|j|d d | j|||d tj||d}	t ||	 W 5 Q R X W 5 Q R X d S )Nr   r   r{   )r   ro   rs   rn   r   r   r)   rB   )
r   r|   r   r   r   r   r   r   rv   r   r   r   r   test_read_explicitq  s    z"TestCompression.test_read_explicitc              
   C   s   |}|d }|| }| j | }t|X}t|B}t }	|	j|d d | j|||d t	|}
t
|	|
 W 5 Q R X W 5 Q R X d S )Nr   r{   )r   r,   r   r   ro   rs   rn   r   r   r)   rB   )r   r   r   r   r   r   r|   r   r   rv   r   r   r   r   test_read_infer  s    
zTestCompression.test_read_inferN)__name__
__module____qualname__icomZ_compression_to_extensionr+   r   r   r   rP   markparametrizer   r   r   r   r   r   r   r   r   !  s   
r   c                   @   s*   e Zd Zejdddddgdd ZdS )	TestProtocolr[   rY   r   r2   r3   c              	   C   sF   t |2}t  }|j||d t|}t || W 5 Q R X d S )NrZ   r   ro   rs   rn   r   r)   rB   )r   r[   r   rS   rv   r   r   r   r   	test_read  s
    
zTestProtocol.test_readN)r   r   r   rP   r   r   r   r   r   r   r   r     s   r   pickle_fileexcolsztest_py27.pklabcztest_mi_py27.pklABCc                 C   s*   | ddd|}t |}t|j| d S )Nry   r-   r^   )r   r)   r   r   columns)rN   r   r   rS   rv   r   r   r   test_unicode_decode_error  s    
r   c               
   C   sl   t  Z} t  }t| d}|| W 5 Q R X t| d}t|}W 5 Q R X t || W 5 Q R X d S )NrX   rd   )r   ro   rs   r]   rn   r   r)   rB   )rS   rv   rb   r"   r   r   r   test_pickle_buffer_roundtrip  s    
r   mockurlzhttp://url.comzftp://test.comzhttp://gzip.comc              	      sp   dd }G dd d t  H fdd}t  }|| | d| t|}t || W 5 Q R X d S )Nc              	   S   s*   t |d}tj| |dd W 5 Q R X d S rW   r\   r`   r   r   r   rc     s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsec                 S   s.   t |d| _d|kr ddi| _n
dd i| _d S )Nrd   rz   zContent-Encoding)r]   fileheaders)r   rS   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r<   r   r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |    d S r<   )close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  S r<   )r   r   r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  S r<   )r   r   r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeN)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                     s    S r<   r   )r   kwargsr   rS   r   r   mock_urlopen_read  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   ro   rs   setattrr   r)   rB   )Zmonkeypatchr   rc   r   rv   r"   r   r   r   test_pickle_generalurl_read  s    


r   Zfsspecc               	   C   sD   t  2 d} t  }||  t| }t || W 5 Q R X d S )Nzmemory://afiler   )r   rv   r"   r   r   r   test_pickle_fsspec_roundtrip  s    


r   c                   @   s   e Zd Zdd ZdS )MyTzc                 C   s   d S r<   r   r   r   r   r   r     s    zMyTz.__init__N)r   r   r   r   r   r   r   r   r     s   r   c                  C   sD   t tdt f} t| }t|d | d  t|d ts@td S )N)Zdtyper   r2   )r   objectr   r   round_trip_pickler7   r   r   )r#   r"   r   r   r   test_read_pickle_with_subclass  s    
r   c              	   C   s   t  }t   }|j|| d t| }W 5 Q R X t }|j|| d |d |	 |ksn| dksnt
tj|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r{   r   )rz   r   N)r   rs   ro   rn   r   
read_bytesry   BytesIOre   getvaluer   r   r)   rB   )r|   rv   rS   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    


r   c                 C   s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S   s   t | }t | | d S r<   )r   r   rB   )frame	unpickledr   r   r   _test_roundtrip2  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdr   r   r   r   r   +test_pickle_dataframe_with_multilevel_index+  s    
r   c                  C   sB   t dddd} ttjt| | }t|}|jj	dks>t
d S )Nz1/1/2011z1/1/2012rF   )r    )r   r   nprandomZrandnlenr   r   r8   r    r   )ZprngtsZnew_tsr   r   r   "test_pickle_timeseries_periodindex<  s    
r   namei	  g     H@i     )r2   r3   c                 C   s$   t t j| d}|j| ks td S )N)r   )r   r   ZmakeTimeSeriesr   r   )r   r   r   r   r   test_pickle_preserve_nameD  s    r   c                 C   s   t | }t ||  d S r<   r   r   r7   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimesM  s    
r   c                 C   s   t | }t ||  d S r<   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_stringsR  s    
r   c                  C   sf   t tddjdg } t| }|jjd jdks:t	|jjd j
dksPt	t|dg |  d S )Nabcr   r   r2   )r2   T)r   listZastypeZilocr   r   Z_mgrblocksndimr   shaper7   )Zserr:   r   r   r    test_pickle_preserves_block_ndimW  s
    
r   r[   c                 C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )r[   r|   r{   )	r   	DataFrameranger   rt   r   rn   r)   rB   )r[   r|   rv   r"   r   r   r   %test_pickle_big_dataframe_compressiond  s    r   c               	   C   sH   t jtdd} t| d}t|}W 5 Q R X t }t	
|| d S )Nz1.2.4zempty_frame_v1_2_4-GH#42345.pklrd   )rR   rS   joinlegacy_dirnamer]   r^   rf   r   r   r   rB   )rS   fdrv   r#   r   r   r   #test_pickle_frame_v124_unpickle_130o  s
    r   )N)d__doc__r   datetime	functoolsr   globrz   ry   rR   pathlibr   r^   r   warningsr   r   r   r   numpyr   rP   Zpandas.compatr   r   Zpandas.compat._optionalr	   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r
   r   r   Zpandas._testingZ_testingr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   r   Z
pytestmarkZfixturer   r'   r1   r;   r?   r@   rC   rD   rE   rH   rS   rU   __file__herer   r   filesrI   rV   rc   rg   r   rL   rn   HIGHEST_PROTOCOLrr   rw   rx   r~   r   r   r   Z
MultiIndexZfrom_arraysr   r   r   Z
skip_if_nor   tzinfor   r   r   r   r   r   r   r   Zskip_array_manager_invalid_testr   DEFAULT_PROTOCOLr   r   r   r   r   r   <module>   s   






!
}


 
%
		 



