# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # https://aws.amazon.com/apache2.0/ # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import copy as python_copy import logging from functools import partial from botocore.exceptions import ClientError from boto3 import utils from boto3.compat import is_append_mode from boto3.s3.transfer import ( ProgressCallbackInvoker, S3Transfer, TransferConfig, create_transfer_manager, ) try: from botocore.context import with_current_context except ImportError: from functools import wraps def with_current_context(hook=None): def decorator(func): @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) return wrapper return decorator try: from botocore.useragent import register_feature_id except ImportError: def register_feature_id(feature_id): pass logger = logging.getLogger(__name__) def inject_s3_transfer_methods(class_attributes, **kwargs): utils.inject_attribute(class_attributes, 'upload_file', upload_file) utils.inject_attribute(class_attributes, 'download_file', download_file) utils.inject_attribute(class_attributes, 'copy', copy) utils.inject_attribute(class_attributes, 'upload_fileobj', upload_fileobj) utils.inject_attribute( class_attributes, 'download_fileobj', download_fileobj ) def inject_bucket_methods(class_attributes, **kwargs): utils.inject_attribute(class_attributes, 'load', bucket_load) utils.inject_attribute(class_attributes, 'upload_file', bucket_upload_file) utils.inject_attribute( class_attributes, 'download_file', bucket_download_file ) utils.inject_attribute(class_attributes, 'copy', bucket_copy) utils.inject_attribute( class_attributes, 'upload_fileobj', bucket_upload_fileobj ) utils.inject_attribute( class_attributes, 'download_fileobj', bucket_download_fileobj ) def inject_object_methods(class_attributes, **kwargs): utils.inject_attribute(class_attributes, 'upload_file', object_upload_file) utils.inject_attribute( class_attributes, 'download_file', object_download_file ) utils.inject_attribute(class_attributes, 'copy', object_copy) utils.inject_attribute( class_attributes, 'upload_fileobj', object_upload_fileobj ) utils.inject_attribute( class_attributes, 'download_fileobj', object_download_fileobj ) def inject_object_summary_methods(class_attributes, **kwargs): utils.inject_attribute(class_attributes, 'load', object_summary_load) def bucket_load(self, *args, **kwargs): """ Calls s3.Client.list_buckets() to update the attributes of the Bucket resource. """ # The docstring above is phrased this way to match what the autogenerated # docs produce. # We can't actually get the bucket's attributes from a HeadBucket, # so we need to use a ListBuckets and search for our bucket. # However, we may fail if we lack permissions to ListBuckets # or the bucket is in another account. In which case, creation_date # will be None. self.meta.data = {} try: response = self.meta.client.list_buckets() for bucket_data in response['Buckets']: if bucket_data['Name'] == self.name: self.meta.data = bucket_data break except ClientError as e: if not e.response.get('Error', {}).get('Code') == 'AccessDenied': raise def object_summary_load(self, *args, **kwargs): """ Calls s3.Client.head_object to update the attributes of the ObjectSummary resource. """ response = self.meta.client.head_object( Bucket=self.bucket_name, Key=self.key ) if 'ContentLength' in response: response['Size'] = response.pop('ContentLength') self.meta.data = response @with_current_context(partial(register_feature_id, 'S3_TRANSFER')) def upload_file( self, Filename, Bucket, Key, ExtraArgs=None, Callback=None, Config=None ): """Upload a file to an S3 object. Usage:: import boto3 s3 = boto3.client('s3') s3.upload_file('/tmp/hello.txt', 'amzn-s3-demo-bucket', 'hello.txt') Similar behavior as S3Transfer's upload_file() method, except that argument names are capitalized. Detailed examples can be found at :ref:`S3Transfer's Usage `. :type Filename: str :param Filename: The path to the file to upload. :type Bucket: str :param Bucket: The name of the bucket to upload to. :type Key: str :param Key: The name of the key to upload to. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed upload arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the upload. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the transfer. """ with S3Transfer(self, Config) as transfer: return transfer.upload_file( filename=Filename, bucket=Bucket, key=Key, extra_args=ExtraArgs, callback=Callback, ) @with_current_context(partial(register_feature_id, 'S3_TRANSFER')) def download_file( self, Bucket, Key, Filename, ExtraArgs=None, Callback=None, Config=None ): """Download an S3 object to a file. Usage:: import boto3 s3 = boto3.client('s3') s3.download_file('amzn-s3-demo-bucket', 'hello.txt', '/tmp/hello.txt') Similar behavior as S3Transfer's download_file() method, except that parameters are capitalized. Detailed examples can be found at :ref:`S3Transfer's Usage `. :type Bucket: str :param Bucket: The name of the bucket to download from. :type Key: str :param Key: The name of the key to download from. :type Filename: str :param Filename: The path to the file to download to. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed download arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the download. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the transfer. """ with S3Transfer(self, Config) as transfer: return transfer.download_file( bucket=Bucket, key=Key, filename=Filename, extra_args=ExtraArgs, callback=Callback, ) def bucket_upload_file( self, Filename, Key, ExtraArgs=None, Callback=None, Config=None ): """Upload a file to an S3 object. Usage:: import boto3 s3 = boto3.resource('s3') s3.Bucket('amzn-s3-demo-bucket').upload_file('/tmp/hello.txt', 'hello.txt') Similar behavior as S3Transfer's upload_file() method, except that parameters are capitalized. Detailed examples can be found at :ref:`S3Transfer's Usage `. :type Filename: str :param Filename: The path to the file to upload. :type Key: str :param Key: The name of the key to upload to. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed upload arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the upload. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the transfer. """ return self.meta.client.upload_file( Filename=Filename, Bucket=self.name, Key=Key, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, ) def bucket_download_file( self, Key, Filename, ExtraArgs=None, Callback=None, Config=None ): """Download an S3 object to a file. Usage:: import boto3 s3 = boto3.resource('s3') s3.Bucket('amzn-s3-demo-bucket').download_file('hello.txt', '/tmp/hello.txt') Similar behavior as S3Transfer's download_file() method, except that parameters are capitalized. Detailed examples can be found at :ref:`S3Transfer's Usage `. :type Key: str :param Key: The name of the key to download from. :type Filename: str :param Filename: The path to the file to download to. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed download arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the download. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the transfer. """ return self.meta.client.download_file( Bucket=self.name, Key=Key, Filename=Filename, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, ) def object_upload_file( self, Filename, ExtraArgs=None, Callback=None, Config=None ): """Upload a file to an S3 object. Usage:: import boto3 s3 = boto3.resource('s3') s3.Object('amzn-s3-demo-bucket', 'hello.txt').upload_file('/tmp/hello.txt') Similar behavior as S3Transfer's upload_file() method, except that parameters are capitalized. Detailed examples can be found at :ref:`S3Transfer's Usage `. :type Filename: str :param Filename: The path to the file to upload. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed upload arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the upload. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the transfer. """ return self.meta.client.upload_file( Filename=Filename, Bucket=self.bucket_name, Key=self.key, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, ) def object_download_file( self, Filename, ExtraArgs=None, Callback=None, Config=None ): """Download an S3 object to a file. Usage:: import boto3 s3 = boto3.resource('s3') s3.Object('amzn-s3-demo-bucket', 'hello.txt').download_file('/tmp/hello.txt') Similar behavior as S3Transfer's download_file() method, except that parameters are capitalized. Detailed examples can be found at :ref:`S3Transfer's Usage `. :type Filename: str :param Filename: The path to the file to download to. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed download arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the download. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the transfer. """ return self.meta.client.download_file( Bucket=self.bucket_name, Key=self.key, Filename=Filename, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, ) @with_current_context(partial(register_feature_id, 'S3_TRANSFER')) def copy( self, CopySource, Bucket, Key, ExtraArgs=None, Callback=None, SourceClient=None, Config=None, ): """Copy an object from one S3 location to another. This is a managed transfer which will perform a multipart copy in multiple threads if necessary. Usage:: import boto3 s3 = boto3.resource('s3') copy_source = { 'Bucket': 'amzn-s3-demo-bucket1', 'Key': 'mykey' } s3.meta.client.copy(copy_source, 'amzn-s3-demo-bucket2', 'otherkey') :type CopySource: dict :param CopySource: The name of the source bucket, key name of the source object, and optional version ID of the source object. The dictionary format is: ``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note that the ``VersionId`` key is optional and may be omitted. :type Bucket: str :param Bucket: The name of the bucket to copy to :type Key: str :param Key: The name of the key to copy to :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed copy arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the copy. :type SourceClient: botocore or boto3 Client :param SourceClient: The client to be used for operations that may happen at the source object. For example, this client is used for the head_object that determines the size of the copy. If no client is provided, the current client is used as the client for the source object. The current client still requires IAM permissions to access both buckets. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the copy. """ subscribers = None if Callback is not None: subscribers = [ProgressCallbackInvoker(Callback)] config = Config if config is None: config = TransferConfig() # copy is not supported in the CRT new_config = python_copy.copy(config) new_config.preferred_transfer_client = "classic" with create_transfer_manager(self, new_config) as manager: future = manager.copy( copy_source=CopySource, bucket=Bucket, key=Key, extra_args=ExtraArgs, subscribers=subscribers, source_client=SourceClient, ) return future.result() def bucket_copy( self, CopySource, Key, ExtraArgs=None, Callback=None, SourceClient=None, Config=None, ): """Copy an object from one S3 location to an object in this bucket. This is a managed transfer which will perform a multipart copy in multiple threads if necessary. Usage:: import boto3 s3 = boto3.resource('s3') copy_source = { 'Bucket': 'amzn-s3-demo-bucket1', 'Key': 'mykey' } bucket = s3.Bucket('amzn-s3-demo-bucket2') bucket.copy(copy_source, 'otherkey') :type CopySource: dict :param CopySource: The name of the source bucket, key name of the source object, and optional version ID of the source object. The dictionary format is: ``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note that the ``VersionId`` key is optional and may be omitted. :type Key: str :param Key: The name of the key to copy to :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed copy arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the copy. :type SourceClient: botocore or boto3 Client :param SourceClient: The client to be used for operations that may happen at the source object. For example, this client is used for the head_object that determines the size of the copy. If no client is provided, the current client is used as the client for the source object. The current client still requires IAM permissions to access both buckets. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the copy. """ return self.meta.client.copy( CopySource=CopySource, Bucket=self.name, Key=Key, ExtraArgs=ExtraArgs, Callback=Callback, SourceClient=SourceClient, Config=Config, ) def object_copy( self, CopySource, ExtraArgs=None, Callback=None, SourceClient=None, Config=None, ): """Copy an object from one S3 location to this object. This is a managed transfer which will perform a multipart copy in multiple threads if necessary. Usage:: import boto3 s3 = boto3.resource('s3') copy_source = { 'Bucket': 'amzn-s3-demo-bucket1', 'Key': 'mykey' } bucket = s3.Bucket('amzn-s3-demo-bucket2') obj = bucket.Object('otherkey') obj.copy(copy_source) :type CopySource: dict :param CopySource: The name of the source bucket, key name of the source object, and optional version ID of the source object. The dictionary format is: ``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note that the ``VersionId`` key is optional and may be omitted. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed copy arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the copy. :type SourceClient: botocore or boto3 Client :param SourceClient: The client to be used for operations that may happen at the source object. For example, this client is used for the head_object that determines the size of the copy. If no client is provided, the current client is used as the client for the source object. The current client still requires IAM permissions to access both buckets. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the copy. """ return self.meta.client.copy( CopySource=CopySource, Bucket=self.bucket_name, Key=self.key, ExtraArgs=ExtraArgs, Callback=Callback, SourceClient=SourceClient, Config=Config, ) @with_current_context(partial(register_feature_id, 'S3_TRANSFER')) def upload_fileobj( self, Fileobj, Bucket, Key, ExtraArgs=None, Callback=None, Config=None ): """Upload a file-like object to S3. The file-like object must be in binary mode. This is a managed transfer which will perform a multipart upload in multiple threads if necessary. Usage:: import boto3 s3 = boto3.client('s3') with open('filename', 'rb') as data: s3.upload_fileobj(data, 'amzn-s3-demo-bucket', 'mykey') :type Fileobj: a file-like object :param Fileobj: A file-like object to upload. At a minimum, it must implement the `read` method, and must return bytes. :type Bucket: str :param Bucket: The name of the bucket to upload to. :type Key: str :param Key: The name of the key to upload to. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed upload arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the upload. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the upload. """ if not hasattr(Fileobj, 'read'): raise ValueError('Fileobj must implement read') subscribers = None if Callback is not None: subscribers = [ProgressCallbackInvoker(Callback)] config = Config if config is None: config = TransferConfig() with create_transfer_manager(self, config) as manager: future = manager.upload( fileobj=Fileobj, bucket=Bucket, key=Key, extra_args=ExtraArgs, subscribers=subscribers, ) return future.result() def bucket_upload_fileobj( self, Fileobj, Key, ExtraArgs=None, Callback=None, Config=None ): """Upload a file-like object to this bucket. The file-like object must be in binary mode. This is a managed transfer which will perform a multipart upload in multiple threads if necessary. Usage:: import boto3 s3 = boto3.resource('s3') bucket = s3.Bucket('amzn-s3-demo-bucket') with open('filename', 'rb') as data: bucket.upload_fileobj(data, 'mykey') :type Fileobj: a file-like object :param Fileobj: A file-like object to upload. At a minimum, it must implement the `read` method, and must return bytes. :type Key: str :param Key: The name of the key to upload to. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed upload arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the upload. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the upload. """ return self.meta.client.upload_fileobj( Fileobj=Fileobj, Bucket=self.name, Key=Key, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, ) def object_upload_fileobj( self, Fileobj, ExtraArgs=None, Callback=None, Config=None ): """Upload a file-like object to this object. The file-like object must be in binary mode. This is a managed transfer which will perform a multipart upload in multiple threads if necessary. Usage:: import boto3 s3 = boto3.resource('s3') bucket = s3.Bucket('amzn-s3-demo-bucket') obj = bucket.Object('mykey') with open('filename', 'rb') as data: obj.upload_fileobj(data) :type Fileobj: a file-like object :param Fileobj: A file-like object to upload. At a minimum, it must implement the `read` method, and must return bytes. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed upload arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the upload. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the upload. """ return self.meta.client.upload_fileobj( Fileobj=Fileobj, Bucket=self.bucket_name, Key=self.key, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, ) def disable_threading_if_append_mode(config, fileobj): """Set `TransferConfig.use_threads` to `False` if file-like object is in append mode. :type config: boto3.s3.transfer.TransferConfig :param config: The transfer configuration to be used when performing the download. :type fileobj: A file-like object :param fileobj: A file-like object to inspect for append mode. """ if is_append_mode(fileobj): config.use_threads = False logger.warning( 'A single thread will be used because the provided file object ' 'is in append mode. Writes may always be appended to the end of ' 'the file regardless of seek position, so a single thread must be ' 'used to ensure sequential writes.' ) @with_current_context(partial(register_feature_id, 'S3_TRANSFER')) def download_fileobj( self, Bucket, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None ): """Download an object from S3 to a file-like object. The file-like object must be in binary mode. This is a managed transfer which will perform a multipart download in multiple threads if necessary. Usage:: import boto3 s3 = boto3.client('s3') with open('filename', 'wb') as data: s3.download_fileobj('amzn-s3-demo-bucket', 'mykey', data) :type Bucket: str :param Bucket: The name of the bucket to download from. :type Key: str :param Key: The name of the key to download from. :type Fileobj: a file-like object :param Fileobj: A file-like object to download into. At a minimum, it must implement the `write` method and must accept bytes. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed download arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the download. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the download. """ if not hasattr(Fileobj, 'write'): raise ValueError('Fileobj must implement write') subscribers = None if Callback is not None: subscribers = [ProgressCallbackInvoker(Callback)] config = Config if config is None: config = TransferConfig() new_config = python_copy.copy(config) disable_threading_if_append_mode(new_config, Fileobj) with create_transfer_manager(self, new_config) as manager: future = manager.download( bucket=Bucket, key=Key, fileobj=Fileobj, extra_args=ExtraArgs, subscribers=subscribers, ) return future.result() def bucket_download_fileobj( self, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None ): """Download an object from this bucket to a file-like-object. The file-like object must be in binary mode. This is a managed transfer which will perform a multipart download in multiple threads if necessary. Usage:: import boto3 s3 = boto3.resource('s3') bucket = s3.Bucket('amzn-s3-demo-bucket') with open('filename', 'wb') as data: bucket.download_fileobj('mykey', data) :type Fileobj: a file-like object :param Fileobj: A file-like object to download into. At a minimum, it must implement the `write` method and must accept bytes. :type Key: str :param Key: The name of the key to download from. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed download arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the download. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the download. """ return self.meta.client.download_fileobj( Bucket=self.name, Key=Key, Fileobj=Fileobj, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, ) def object_download_fileobj( self, Fileobj, ExtraArgs=None, Callback=None, Config=None ): """Download this object from S3 to a file-like object. The file-like object must be in binary mode. This is a managed transfer which will perform a multipart download in multiple threads if necessary. Usage:: import boto3 s3 = boto3.resource('s3') bucket = s3.Bucket('amzn-s3-demo-bucket') obj = bucket.Object('mykey') with open('filename', 'wb') as data: obj.download_fileobj(data) :type Fileobj: a file-like object :param Fileobj: A file-like object to download into. At a minimum, it must implement the `write` method and must accept bytes. :type ExtraArgs: dict :param ExtraArgs: Extra arguments that may be passed to the client operation. For allowed download arguments see :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`. :type Callback: function :param Callback: A method which takes a number of bytes transferred to be periodically called during the download. :type Config: boto3.s3.transfer.TransferConfig :param Config: The transfer configuration to be used when performing the download. """ return self.meta.client.download_fileobj( Bucket=self.bucket_name, Key=self.key, Fileobj=Fileobj, ExtraArgs=ExtraArgs, Callback=Callback, Config=Config, )