U
     fh=                     @   s   d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZmZ ddlmZ ddlmZ zddlZW n ek
r   dZY nX d	ZG d
d deZdS )a	  
Sync Media to S3
================

Django command that scans all files in your settings.MEDIA_ROOT and
settings.STATIC_ROOT folders and uploads them to S3 with the same directory
structure.

This command can optionally do the following but it is off by default:
* gzip compress any CSS and Javascript files it finds and adds the appropriate
  'Content-Encoding' header.
* set a far future 'Expires' header for optimal caching.
* upload only media or static files.
* use any other provider compatible with Amazon S3.
* set other than 'public-read' ACL.

Note: This script requires the Python boto library and valid Amazon Web
Services API keys.

Required settings.py variables:
AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''
AWS_BUCKET_NAME = ''

When you call this command with the `--renamegzip` param, it will add
the '.gz' extension to the file name. But Safari just doesn't recognize
'.gz' files and your site won't work on it! To fix this problem, you can
set any other extension (like .jgz) in the `SYNC_S3_RENAME_GZIP_EXT`
variable.

Command options are:
  -p PREFIX, --prefix=PREFIX
                        The prefix to prepend to the path on S3.
  --gzip                Enables gzipping CSS and Javascript files.
  --expires             Enables setting a far future expires header.
  --force               Skip the file mtime check to force upload of all
                        files.
  --filter-list         Override default directory and file exclusion
                        filters. (enter as comma separated line)
  --renamegzip          Enables renaming of gzipped files by appending '.gz'.
                        to the original file name. This way your original
                        assets will not be replaced by the gzipped ones.
                        You can change the extension setting the
                        `SYNC_S3_RENAME_GZIP_EXT` var in your settings.py
                        file.
  --invalidate          Invalidates the objects in CloudFront after uploading
                        stuff to s3.
  --media-only          Only MEDIA_ROOT files will be uploaded to S3.
  --static-only         Only STATIC_ROOT files will be uploaded to S3.
  --s3host              Override default s3 host.
  --acl                 Override default ACL settings ('public-read' if
                        settings.AWS_DEFAULT_ACL is not defined).

TODO:
 * Use fnmatch (or regex) to allow more complex FILTER_LIST rules.

    N)List)settings)BaseCommandCommandError)StringIO)signalcommandFTc                       s   e Zd ZdZdZdZdZdZdZdddddgZ	dZ
g ZdZdZd	Zd
ZdZ fddZedd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Z  ZS )Command z	.DS_Storez.svnz.hgz.gitz	Thumbs.db)ztext/csszapplication/javascriptzapplication/x-javascriptztext/javascriptr   zSSyncs the complete MEDIA_ROOT structure and files to S3 into the given bucket name.bucket_nameTc                    s  t  | |jdddttdddd |jdd	d
dd |jddttdddd |jddttdddd |jdddddd |jdddddd |jddd dd!d |jd"dd#dd$d |jd%d&d'dd(d) |jd*d+ddd,d- |jd.d/ddd0d- |jd1d2ddd3d- d S )4Nz-pz--prefixprefixZSYNC_S3_PREFIXr	   z(The prefix to prepend to the path on S3.)destdefaulthelpz-dz--dirdirz#Custom static root directory to use)r   r   z--s3hosts3hostZAWS_S3_HOSTz;The s3 host (enables connecting to other providers/regions)z--aclaclZAWS_DEFAULT_ACLzpublic-readz.Enables to override default acl (public-read).z--gzip
store_truegzipFz*Enables gzipping CSS and Javascript files.)actionr   r   r   z--renamegzip
renamegzipzJEnables renaming of gzipped assets to have '.gz' appended to the filename.z	--expiresexpiresz,Enables setting a far future expires header.z--forceforcez7Skip the file mtime check to force upload of all files.z--filter-listfilter_liststorezVOverride default directory and file exclusion filters. (enter as comma seperated line))r   r   r   r   z--invalidate
invalidatez0Invalidates the associated objects in CloudFront)r   r   r   r   z--media-only
media_onlyz,Only MEDIA_ROOT files will be uploaded to S3z--static-onlystatic_onlyz-Only STATIC_ROOT files will be uploaded to S3)superadd_argumentsadd_argumentgetattrr   )selfparser	__class__ Q/tmp/pip-unpacked-wheel-1isl55vw/django_extensions/management/commands/sync_s3.pyr   l   s     
 

                zCommand.add_argumentsc                 O   s  t stdttdr ttds*tdntj| _tj| _ttdsNtdntjs\tdtj| _ttdsxtd	ntjstd	ttd
d| _	ttdd| _
|d | _|d | _|d | _|d | _|d | _|d | _|d | _|d | _|d | _|d | _ttd| j| _|d }|r0|d| _|d | _|d | _| jr^| jr^tdnJ| jrrtjg| _n6| jrtjg| _n"| jr| jg| _ntjtjg| _|   | jr|   td td| j  td| j  d S ) Nz>Please install the 'boto' Python library. ($ pip install boto)AWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYzeMissing AWS keys from settings file.  Please supply both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.AWS_BUCKET_NAMEz]Missing bucket name from settings file. Please add the AWS_BUCKET_NAME to your settings file.z AWS_BUCKET_NAME cannot be empty.
MEDIA_ROOTz(MEDIA_ROOT must be set in your settings.AWS_CLOUDFRONT_DISTRIBUTIONr	   SYNC_S3_RENAME_GZIP_EXTz.gz	verbosityr   r   r   r   r   r   r   r   r   FILTER_LISTr   ,r   r   zMCan't use --media-only and --static-only together. Better not use anything...z%d files uploaded.z%d files skipped.)HAS_BOTOr   hasattrr   r'   r(   r)   r*   r    r+   r,   r-   r   do_gziprename_gzip
do_expiresdo_forcer   DIRECTORIESr   default_aclr.   splitr   r   ZSTATIC_ROOTsync_s3invalidate_objects_cfprintupload_count
skip_count)r!   argsoptionsr   r%   r%   r&   handle   sb    


















zCommand.handlec                 C   s   t | j| jS )z'Return an open connection to CloudFront)botoZconnect_cloudfrontr'   r(   )r!   r%   r%   r&   open_cf   s     zCommand.open_cfc                    s\   | j stdd |  }| j fddtdt D }|D ]}|| j | qDdS )z8Split the invalidation request in groups of 1000 objectszrAn object invalidation was requested but the variable AWS_CLOUDFRONT_DISTRIBUTION is not present in your settings.i  c                    s   g | ]}||   qS r%   r%   ).0ichunkobjsr%   r&   
<listcomp>  s     z1Command.invalidate_objects_cf.<locals>.<listcomp>r   N)r+   r   rB   uploaded_filesrangelenZcreate_invalidation_request)r!   connchunkspathsr%   rE   r&   r:      s      zCommand.invalidate_objects_cfc                 C   sL   |   \}}| jD ]4}t|D ]$\}}}| ||| j|f||| q qdS )z7Walk the media/static directories and syncs files to S3N)open_s3r6   oswalk	upload_s3r)   )r!   bucketkey	directoryrootdirsfilesr%   r%   r&   r9     s    
zCommand.sync_s3c                 C   s0   t  }tjdd|d}|| |  | S )zGzip a given string.wb   )modecompresslevelfileobj)r   r   GzipFilewriteclosegetvalue)r!   sZzbufzfiler%   r%   r&   compress_string  s
    
zCommand.compress_stringc                 C   s   i }| j r| j |d< |S )z"Return connection kwargs as a dicthost)r   )r!   kwargsr%   r%   r&   get_s3connection_kwargs#  s    
zCommand.get_s3connection_kwargsc                 C   s`   t j| j| jf|  }z|| j}W n$ t jjk
rL   |	| j}Y nX |t j
j|fS )z.Open connection to S3 returning bucket and key)rA   Z
connect_s3r'   r(   rg   Z
get_bucketr)   	exceptionZS3ResponseErrorZcreate_bucketZs3rT   ZKey)r!   rL   rS   r%   r%   r&   rO   *  s    zCommand.open_s3c                 C   s  |\}}}}t j|| jkr>t j|| jkr>|d d = d S |t jjsX|t jj }|D ]}	i }
|	| jkrrq\t j||	}t j	|rq\|t
|d  }| jrd| j|f }| js*||}|r*tjt|jddd  }tjt |j}||k r*|  jd7  _| jdkr\td|  q\| jdkrBtd|  t|d }|r`||
d< nd	|
d< t|d
}t | j}| }| jr|dkr|| j kr| !|}| j"rd|| j#f }d|
d< | jdkrtd|d t
|d f  | j$rhdt%j&'t(tj) tj*dd +  |
d< dd |
d< | jdkrhtd|
d   td|
d   z||_,|j-||
d| j.d W n` t/j0j1k
r } ztd|  W 5 d }~X Y nH t2k
r } zt|  W 5 d }~X Y nX |  j3d7  _3| j45| |6  q\d S )Nz%s/%sz%a, %d %b %Y %H:%M:%S %Zr   rZ      z6File %s hasn't been modified since last being uploadedzUploading %s...zContent-Typezapplication/octet-streamrbi   z%s.%sr   zContent-Encodingz	gzipped: %dk to %dkz%s GMTi  )daysZExpiresz
max-age %di gzCache-Controlz	expires: %sz	cache-control: %sT)replacepolicyz
Failed: %s)7rP   pathbasenamer.   dirnamer6   endswithsepjoinisdirrK   r   r5   get_keydatetimetimestrptimeZlast_modifiedutcfromtimestampstatst_mtimer=   r-   r;   	mimetypes
guess_typeopenfstatfilenost_sizereadr2   GZIP_CONTENT_TYPESrd   r3   r,   r4   emailZUtils
formatdatemktimenow	timedelta	timetuplenameZset_contents_from_stringr7   rA   rh   ZS3CreateError	Exceptionr<   rI   appendr`   )r!   argrp   namesrW   rS   rT   r
   root_dirfileheadersfilenameZfile_keyZs3_keyZs3_datetimeZlocal_datetimecontent_typeZfile_obj	file_sizeZfiledataer%   r%   r&   rR   6  s    $



  





 0

zCommand.upload_s3)__name__
__module____qualname__r'   r(   r)   r+   r,   r6   r.   r   rI   r<   r=   r   r>   Zcan_import_settingsr   r   r@   rB   r:   r9   rd   rg   rO   rR   __classcell__r%   r%   r#   r&   r   R   s0   B
Ir   )__doc__rv   r   r   r|   rP   rw   typingr   Zdjango.confr   Zdjango.core.management.baser   r   ior   Z"django_extensions.management.utilsr   rA   ImportErrorr0   r   r%   r%   r%   r&   <module>   s"   9
