diff --git a/awscli/customizations/s3/factory.py b/awscli/customizations/s3/factory.py index 50ba57294400..fcafd645cd38 100644 --- a/awscli/customizations/s3/factory.py +++ b/awscli/customizations/s3/factory.py @@ -13,6 +13,7 @@ import logging import awscrt.s3 +from botocore import UNSIGNED from botocore.client import Config from botocore.httpsession import DEFAULT_CA_BUNDLE from s3transfer.crt import ( @@ -114,9 +115,11 @@ def _acquire_crt_s3_process_lock(self): def _create_crt_transfer_manager(self, params, runtime_config): self._acquire_crt_s3_process_lock() + serializer_client = self._create_serializer_client(params) return CRTTransferManager( self._create_crt_client(params, runtime_config), - self._create_crt_request_serializer(params), + self._create_crt_request_serializer(params, serializer_client), + client=serializer_client, ) def _create_crt_client(self, params, runtime_config): @@ -154,13 +157,26 @@ def _create_crt_client(self, params, runtime_config): return create_s3_crt_client(**create_crt_client_kwargs) - def _create_crt_request_serializer(self, params): + def _create_serializer_client(self, params): + client_kwargs = { + 'region_name': self._resolve_region(params), + 'endpoint_url': params.get('endpoint_url'), + } + user_provided_config = self._session.get_default_client_config() + if 'config' in client_kwargs: + user_provided_config = client_kwargs['config'] + client_config = Config(signature_version=UNSIGNED) + if user_provided_config: + client_config = user_provided_config.merge(client_config) + client_kwargs['config'] = client_config + return self._session.create_client( + service_name='s3', **client_kwargs + ) + + def _create_crt_request_serializer(self, params, client): return BotocoreCRTRequestSerializer( self._session, - { - 'region_name': self._resolve_region(params), - 'endpoint_url': params.get('endpoint_url'), - }, + client=client, ) def _create_classic_transfer_manager( diff --git a/awscli/s3transfer/crt.py b/awscli/s3transfer/crt.py index ae4e5195a4b2..af8e46486ccf 100644 --- a/awscli/s3transfer/crt.py +++ b/awscli/s3transfer/crt.py @@ -202,7 +202,9 @@ def _get_crt_throughput_target_gbps(provided_throughput_target_bytes=None): class CRTTransferManager: - def __init__(self, crt_s3_client, crt_request_serializer, osutil=None): + def __init__( + self, crt_s3_client, crt_request_serializer, osutil=None, client=None + ): """A transfer manager interface for Amazon S3 on CRT s3 client. :type crt_s3_client: awscrt.s3.S3Client @@ -216,12 +218,16 @@ def __init__(self, crt_s3_client, crt_request_serializer, osutil=None): :type osutil: s3transfer.utils.OSUtils :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. + + :type client: botocore.client.BaseClient + :param client: The botocore client used for + serializing requests and resolving client configuration. """ if osutil is None: self._osutil = OSUtils() self._crt_s3_client = crt_s3_client self._s3_args_creator = S3ClientArgsCreator( - crt_request_serializer, self._osutil + crt_request_serializer, self._osutil, client ) self._crt_exception_translator = ( crt_request_serializer.translate_crt_exception @@ -449,7 +455,7 @@ def translate_crt_exception(self, exception): class BotocoreCRTRequestSerializer(BaseCRTRequestSerializer): - def __init__(self, session, client_kwargs=None): + def __init__(self, session, client_kwargs=None, client=None): """Serialize CRT HTTP request using botocore logic It also takes into account configuration from both the session and any keyword arguments that could be passed to @@ -460,12 +466,19 @@ def __init__(self, session, client_kwargs=None): :type client_kwargs: Optional[Dict[str, str]]) :param client_kwargs: The kwargs for the botocore s3 client initialization. + + :type client: Optional[botocore.client.BaseClient] + :param client: A pre-configured botocore S3 client. If provided, + session and client_kwargs are ignored for client creation. """ self._session = session - if client_kwargs is None: - client_kwargs = {} - self._resolve_client_config(session, client_kwargs) - self._client = session.create_client(**client_kwargs) + if client is not None: + self._client = client + else: + if client_kwargs is None: + client_kwargs = {} + self._resolve_client_config(session, client_kwargs) + self._client = session.create_client(**client_kwargs) self._client.meta.events.register( 'request-created.s3.*', self._capture_http_request ) @@ -715,9 +728,10 @@ def set_s3_request(self, s3_request): class S3ClientArgsCreator: - def __init__(self, crt_request_serializer, os_utils): + def __init__(self, crt_request_serializer, os_utils, client=None): self._request_serializer = crt_request_serializer self._os_utils = os_utils + self._client = client def get_make_request_args( self, request_type, call_args, coordinator, future, on_done_after_calls @@ -819,7 +833,14 @@ def _get_make_request_args_get_object( ): recv_filepath = None on_body = None - checksum_config = awscrt.s3.S3ChecksumConfig(validate_response=True) + validate = not ( + self._client is not None + and self._client.meta.config.response_checksum_validation + == 'when_required' + ) + checksum_config = awscrt.s3.S3ChecksumConfig( + validate_response=validate + ) if isinstance(call_args.fileobj, str): final_filepath = call_args.fileobj recv_filepath = self._os_utils.get_temp_filename(final_filepath) diff --git a/tests/functional/s3transfer/test_crt.py b/tests/functional/s3transfer/test_crt.py index d092982f082c..1de864c452dd 100644 --- a/tests/functional/s3transfer/test_crt.py +++ b/tests/functional/s3transfer/test_crt.py @@ -16,6 +16,8 @@ import time from concurrent.futures import Future +from botocore import UNSIGNED +from botocore.config import Config from botocore.session import Session from s3transfer.subscribers import BaseSubscriber @@ -720,3 +722,34 @@ def test_crt_s3_client_error_handling(self): ) with self.assertRaises(awscrt.exceptions.AwsCrtError): future.result() + + def test_download_checksum_validation_disabled_when_required(self): + session = Session() + session.set_config_variable('region', self.region) + client = session.create_client( + 's3', + config=Config( + signature_version=UNSIGNED, + response_checksum_validation='when_required', + ), + ) + request_serializer = s3transfer.crt.BotocoreCRTRequestSerializer( + session, client=client + ) + transfer_manager = s3transfer.crt.CRTTransferManager( + crt_s3_client=self.s3_crt_client, + crt_request_serializer=request_serializer, + client=client, + ) + future = transfer_manager.download( + self.bucket, self.key, self.filename, {}, [self.record_subscriber] + ) + future.result() + + callargs_kwargs = self.s3_crt_client.make_request.call_args[1] + self.assertEqual( + callargs_kwargs['checksum_config'], + self._get_expected_download_checksum_config( + validate_response=False + ), + )