U
    MfQ                     @  s  d dl mZ d dlZd dlmZmZmZ d dlZd dl	m
Z
mZ d dlmZmZmZmZ d dlmZ d dlmZmZ d dlmZmZmZmZ d d	lmZmZ d d
lm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z'm(Z( d dl)m*Z*m+Z+ d dl,m-Z- erd dl.m/Z/ d dl)m0Z0 dddddddZ1dddddZ2dddddddZ3ddd d!d"d#Z4dd$d$d%d&d'Z5d d(d)d*Z6G d+d, d,Z7d-d.dd/d0d1Z8d2ddd3d4d5Z9d-d6d7d8Z:d9d-d:d;d<Z;d2dd:d=d>Z<dd?d@dAZ=d,dd,dBdCdDZ>dEdF Z?dS )G    )annotationsN)TYPE_CHECKINGSequencecast)NaT	internals)	ArrayLikeDtypeObjManagerShape)cache_readonly)ensure_dtype_can_hold_nafind_common_type)is_1d_only_ea_dtypeis_1d_only_ea_objis_datetime64tz_dtypeis_dtype_equal)cast_to_common_typeconcat_compat)ExtensionDtype)DatetimeArrayExtensionArray)ensure_wrapped_if_datetimelike)ArrayManagerNullArrayProxy)ensure_block_shapenew_block_2d)BlockManager)Index)Blockzlist[Index]intboolr
   )axesconcat_axiscopyreturnc              	     s   g  | D ]l\}}d}|  D ]4\}}|j|| ||ddd}|dkr|dk	rd}q|rj|dkrj|sj| } | q|dkr fddtt d jD }	n&|dksttt	j
d	d  D }	t|	|d |d gdd
}
|
S )z
    Concatenate array managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (ArrayManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    ArrayManager
    FT)axis
allow_dupsuse_na_proxy   Nr   c                   s,   g | ]$ t  fd dttD qS )c                   s   g | ]}| j   qS  arrays.0i)jmgrsr*   @/tmp/pip-unpacked-wheel-eb6vo0j3/pandas/core/internals/concat.py
<listcomp>`   s     z:_concatenate_array_managers.<locals>.<listcomp>.<listcomp>)concat_arraysrangelen)r.   r1   )r0   r2   r3   _   s   z/_concatenate_array_managers.<locals>.<listcomp>c                 S  s   g | ]
}|j qS r*   r+   )r.   mgrr*   r*   r2   r3   f   s     )Zverify_integrity)itemsreindex_indexerr$   appendr5   r6   r,   AssertionErrorlist	itertoolschainfrom_iterabler   )mgrs_indexersr"   r#   r$   r8   indexersZaxis1_made_copyZaxindexerr,   Znew_mgrr*   r7   r2   _concatenate_array_managers<   s0        
rD   r=   r   )	to_concatr%   c                   s"  dd | D }dd |D }t |dk}|r8|d j n6tdd |D r\tt|g  ntd	d |D   jd
kr fdd| D } t|d j| ddS  fdd| D } t	| d t
rt| d }|| S t| }t |dkrdd |D }t |dkrd|kr|t}|S )a  
    Alternative for concat_compat but specialized for use in the ArrayManager.

    Differences: only deals with 1D arrays (no axis keyword), assumes
    ensure_wrapped_if_datetimelike and does not skip empty arrays to determine
    the dtype.
    In addition ensures that all NullArrayProxies get replaced with actual
    arrays.

    Parameters
    ----------
    to_concat : list of arrays

    Returns
    -------
    np.ndarray or ExtensionArray
    c                 S  s   g | ]}t |ts|qS r*   )
isinstancer   r.   xr*   r*   r2   r3      s     
 z!concat_arrays.<locals>.<listcomp>c                 S  s   h | ]
}|j qS r*   dtyperG   r*   r*   r2   	<setcomp>   s     z concat_arrays.<locals>.<setcomp>r)   r   c                 s  s$   | ]}|j d kot|tjV  qdS ))r/   ubN)kindrF   nprJ   rG   r*   r*   r2   	<genexpr>   s     z concat_arrays.<locals>.<genexpr>c                 S  s   g | ]
}|j qS r*   rI   r.   Zarrr*   r*   r2   r3      s     mMc                   s$   g | ]}t |tr| n|qS r*   )rF   r   to_arrayrQ   Ztarget_dtyper*   r2   r3      s   r&   c                   s*   g | ]"}t |tr| nt| qS r*   )rF   r   rU   r   rQ   rV   r*   r2   r3      s   c                 S  s   h | ]}|j jqS r*   )rJ   rN   )r.   objr*   r*   r2   rK      s     rM   )r6   rJ   allrO   r   r=   rN   typeZ_concat_same_typerF   r   concatenateZastypeobject)rE   Zto_concat_no_proxydtypesZsingle_dtypeclsresultkindsr*   rV   r2   r4   l   s4    






r4   c                 C  sH  t | d d tr t| |||S |dkr4t| ||S t|| } dd | D }t|}g }|D ]\}}|d }	|	j}
t|dkr|
j}|r|	 }n|
 }d}nlt|rdd |D }|
jstj|dd}nt|dd}t|dd	}t|}|
jj|jk}nt||d
}d}|r"|
j||d}nt||d}|| q\tt||S )z
    Concatenate block managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    BlockManager
    r   c                 S  s   g | ]\}}t |qS r*   )_get_mgr_concatenation_plan)r.   r8   _r*   r*   r2   r3      s     z(concatenate_managers.<locals>.<listcomp>r)   Tc                 S  s   g | ]}|j jqS r*   )blockvaluesr.   Zjur*   r*   r2   r3      s     rW      )ndim)r$   F)	placement)rF   r   rD   _concat_managers_axis0_maybe_reindex_columns_na_proxy_combine_concat_plansrc   r6   rd   r$   view_is_uniform_join_unitsis_extensionrO   r[   r   r   r   rJ   _concatenate_join_unitsZmake_block_same_classr   r;   r   tuple)rA   r"   r#   r$   Zconcat_plansZconcat_planblocksrh   
join_unitsunitblkrd   ZfastpathvalsrM   r*   r*   r2   concatenate_managers   s@    

rv   r   )r"   r$   r%   c                   s    fddt t D }t|  dd  D }d}g }t|D ]p\}}|| }	|jD ]J}
|	rn|
jdd}n|r||
 }n|
td}|j	||_|
| qX|t|j7 }qBtt||S )	z
    concat_managers specialized to concat_axis=0, with reindexing already
    having been done in _maybe_reindex_columns_na_proxy.
    c                   s"   i | ]}|t  | d  dkqS )r)   r   )r6   r-   rA   r*   r2   
<dictcomp>  s     z*_concat_managers_axis0.<locals>.<dictcomp>c                 S  s   g | ]}|d  qS )r   r*   rG   r*   r*   r2   r3   
  s     z*_concat_managers_axis0.<locals>.<listcomp>r   F)deepN)r5   r6   rj   	enumeraterq   r$   getitem_blocksliceZ	_mgr_locsaddr;   r9   r   rp   )rA   r"   r$   Zhad_reindexersr1   offsetrq   r/   r8   Z	made_copyrt   nbr*   rw   r2   ri      s&    




ri   z0list[tuple[BlockManager, dict[int, np.ndarray]]])r"   rA   r%   c                 C  sV   g }|D ]H\}}|  D ](\}}|j| | || |ddddd}q||i f q|S )z
    Reindex along columns so that all of the BlockManagers being concatenated
    have matching columns.

    Columns added in this reindexing have dtype=np.void, indicating they
    should be ignored when choosing a column's final dtype.
    FT)r&   r$   Z
only_slicer'   r(   )r9   r:   r;   )r"   rA   Znew_mgrs_indexersr8   rB   r/   rC   r*   r*   r2   rj   "  s    
	rj   )r8   c                 C  s  t | j}t|}| jr4| jd }|jt||fgS | j}| j}g }t	j
|ddD ]\}}|jsdt|dksptt |}	t||	d< t|	}
| j| }||j }t|t|jko|jjr|jjjdkpt|dk }|s||}t||
}|||f qR|S )z
    Construct concatenation plan for given block manager.

    Parameters
    ----------
    mgr : BlockManager

    Returns
    -------
    plan : list of (BlockPlacement, JoinUnit) tuples

    r   F)groupr)   )r=   shaperp   Zis_single_blockrq   Zmgr_locsJoinUnitblknosblklocslibinternalsZget_blkno_placementsZis_slice_liker<   r6   rC   Zas_slicesteprO   ZdiffrY   r{   r;   )r8   Zmgr_shape_listZ	mgr_shapert   r   r   ZplanZblkno
placementsZ
shape_listr   Zax0_blk_indexerZunit_no_ax0_reindexingrs   r*   r*   r2   ra   @  s2    






ra   c                   @  sL   e Zd ZdddddZdddd	Zed
dddZdddddZdS )r   r   r   rc   r   c                 C  s   || _ || _d S Nr   )selfrc   r   r*   r*   r2   __init__  s    zJoinUnit.__init__strr%   c                 C  s   t | j dt| j dS )N())rZ   __name__reprrc   )r   r*   r*   r2   __repr__  s    zJoinUnit.__repr__r!   c                 C  s   | j }|jjdkrdS dS )NVTF)rc   rJ   rN   )r   rt   r*   r*   r2   is_na  s    zJoinUnit.is_nar	   r   )empty_dtyper%   c                 C  s.   | j rt|| jS | jjs"| jjS | jj}|S r   )r   make_na_arrayr   rc   Z_can_consolidaterd   )r   r   rd   r*   r*   r2   get_reindexed_values  s    zJoinUnit.get_reindexed_valuesN)r   
__module____qualname__r   r   r   r   r   r*   r*   r*   r2   r     s
   r   r	   r   )rJ   r   r%   c                 C  s   t | r"t|tj}t|| dS t| rztt| } | 	 }|j
g | d}|d }dtj|ftjd }|j|d| jdS t| tr| 	 }|j|| d}| j|dd< |S tj|| d}t| }|| |S dS )zk
    Construct an np.ndarray or ExtensionArray of the given dtype and shape
    holding all-NA values.
    rI   r   T)Z
allow_fill
fill_value)r   rJ   N)r   rO   fullr   valuer   r   r   r   Zconstruct_array_typeZ_from_sequenceZonesZintpZtakena_valuerF   _emptyempty_dtype_to_na_valuefill)rJ   r   Zi8valuesr^   Zmissing_arrZnrowsZtakerr   r*   r*   r2   r     s&    


r   zlist[JoinUnit])rr   r$   r%   c                   s   t |   fdd| D }t|dkr\|d }|rt|tjrR|jdk	rZ| }q| }nFtdd |D rdd |D }t|dd	d
}t	|d}nt|dd}|S )zB
    Concatenate values from several join units along axis=1.
    c                   s   g | ]}|j  d qS )r   )r   re   r   r*   r2   r3     s     z+_concatenate_join_units.<locals>.<listcomp>r)   r   Nc                 s  s   | ]}t |V  qd S r   r   r.   tr*   r*   r2   rP     s     z*_concatenate_join_units.<locals>.<genexpr>c                 S  s(   g | ] }t |r|n|d ddf qS )r   Nr   r   r*   r*   r2   r3     s   T)r&   Zea_compat_axisrf   rW   )
_get_empty_dtyper6   rF   rO   Zndarraybaser$   anyr   r   )rr   r$   rE   Zconcat_valuesr*   r   r2   ro     s"    


ro   rI   c                 C  sn   t | tr| jS | jdkr$| dS | jdkr8| dS | jdkrFdS | jdkrVtjS | jdkrftjS tdS )	z2
    Find the NA value to go with this dtype.
    rR   r   )fcNaNrM   N)r/   rL   O)rF   r   r   rN   rZ   rO   nanNotImplementedErrorrI   r*   r*   r2   r     s    







r   zSequence[JoinUnit])rr   r%   c                 C  sl   t | dkr| d j}|jS t| r4| d jj}|S tdd | D }dd | D }t|}|rht|}|S )z
    Return dtype and N/A values to use when concatenating specified units.

    Returned N/A value may be None which means there was no casting involved.

    Returns
    -------
    dtype
    r)   r   c                 s  s   | ]}|j V  qd S r   )r   r.   rs   r*   r*   r2   rP     s     z#_get_empty_dtype.<locals>.<genexpr>c                 S  s   g | ]}|j s|jjqS r*   )r   rc   rJ   r   r*   r*   r2   r3     s      z$_get_empty_dtype.<locals>.<listcomp>)r6   rc   rJ   _is_uniform_reindexr   r   r   )rr   rt   r   Zneeds_can_hold_nar]   rJ   r*   r*   r2   r     s    

r   c                   sd   | d j   jjdkrdS t fdd| D obt fdd| D obtdd | D obt| dkS )	z
    Check if the join units consist of blocks of uniform type that can
    be concatenated using Block.concat_same_type instead of the generic
    _concatenate_join_units (which uses `concat_compat`).

    r   r   Fc                 3  s    | ]}t |jt  kV  qd S r   )rZ   rc   re   firstr*   r2   rP   0  s     z)_is_uniform_join_units.<locals>.<genexpr>c                 3  s,   | ]$}t |jj jp"|jjjd kV  qdS ))rM   r/   rL   N)r   rc   rJ   rN   re   r   r*   r2   rP   3  s   c                 s  s   | ]}|j  p|jjV  qd S r   )r   rc   rn   re   r*   r*   r2   rP   =  s     r)   )rc   rJ   rN   rY   r6   rr   r*   r   r2   rm   $  s    

rm   r   c                 C  s(   t dd | D o&tdd | D dkS )Nc                 s  s   | ]}|j jV  qd S r   )rc   rn   re   r*   r*   r2   rP   G  s     z&_is_uniform_reindex.<locals>.<genexpr>c                 S  s   h | ]}|j jjqS r*   )rc   rJ   namere   r*   r*   r2   rK   H  s     z&_is_uniform_reindex.<locals>.<setcomp>r)   )rY   r6   r   r*   r*   r2   r   D  s    r   )	join_unitlengthr%   c                 C  sd   | j t|d}| j t|| _ | jd | f| jdd  }|f| jdd  | _t||dS )z
    Reduce join_unit's shape along item axis to length.

    Extra items that didn't fit are returned as a separate block.
    Nr   r)   r   )rc   r{   r|   r   r   )r   r   Zextra_blockZextra_shaper*   r*   r2   _trim_join_unitL  s
    r   c                 #  sZ  t | dkr2| d D ]}|d |d gfV  qn$dg  fdd}ttt| } tt|| } d t |krV d dkrtdt| \}}ttt |}t|t| }}||kr|d |fV  t|| |dd< q`d}	dgt | }
t|D ]R\}\}}||
|< t ||kr4||d t	||f||< q|}	|| | ||< q|	|
fV  q`dS )z`
    Combine multiple concatenation plans into one.

    existing_plan is updated in-place.
    r)   r   c                   s&   t | d }|d kr" d  d7  < |S )Nr   r)   )next)seqretvalZ	num_endedr*   r2   _next_or_nonej  s    
z,_combine_concat_plans.<locals>._next_or_nonezPlan shapes are not alignedN)
r6   r=   mapiter
ValueErrorzipminmaxrz   r   )Zplanspr   Z
next_itemsr   ZunitslengthsZmin_lenmax_lenZyielded_placementZyielded_unitsr/   Zplcrs   r*   r   r2   rk   \  s2    rk   )@
__future__r   r>   typingr   r   r   ZnumpyrO   Zpandas._libsr   r   r   Zpandas._typingr   r	   r
   r   Zpandas.util._decoratorsr   Zpandas.core.dtypes.castr   r   Zpandas.core.dtypes.commonr   r   r   r   Zpandas.core.dtypes.concatr   r   Zpandas.core.dtypes.dtypesr   Zpandas.core.arraysr   r   Zpandas.core.constructionr   Z#pandas.core.internals.array_managerr   r   Zpandas.core.internals.blocksr   r   Zpandas.core.internals.managersr   Zpandasr   r   rD   r4   rv   ri   rj   ra   r   r   ro   r   r   rm   r   r   rk   r*   r*   r*   r2   <module>   sB   0BP$E#"* 