Here are examples of accessing S3 with Boto 3.
Upload a File
import boto3
from botocore.exceptions import ClientError
with open("hello.txt", "w") as f:
f.write('Hello World!')
s3Client = boto3.client('s3')
# upload a file
try:
response = s3Client.upload_file(
Filename = 'hello.txt',
Bucket = 'my-test-bucket',
Key = 'hello.txt',
ExtraArgs={'ACL': 'public-read'}
)
print(response)
except ClientError as e:
print(e)
Copy the File
import boto3
from botocore.exceptions import ClientError
s3Client = boto3.client('s3')
# copy the file
try:
response = s3Client.copy(
CopySource = '/my-test-bucket/hello.txt',
Bucket = 'my-test-bucket',
Key = 'hello-copy.txt',
)
print(response)
except ClientError as e:
print(e)
Multipart Transfer
- Upload a big file with automatic multipart transfer.
- Multipart transfers occur when the file size exceeds the value of the multipart_threshold attribute.
import boto3
from botocore.exceptions import ClientError
from boto3.s3.transfer import TransferConfig
s3Client = boto3.client('s3')
try:
# Set the desired multipart threshold value (5GB)
GB = 1024 ** 3
config = TransferConfig(multipart_threshold=5*GB)
response = s3Client.upload_file(
Filename = 'hello.txt',
Bucket = 'my-test-bucket',
Key = 'hello.txt',
Config = config,
ExtraArgs={'ACL': 'public-read'}
)
print(response)
except ClientError as e:
print(e)
Getting the list of Objects in S3
import boto3
from botocore.exceptions import ClientError
s3Resource = boto3.resource('s3')
try:
bucket = s3Resource.Bucket('my-test-bucket')
print(bucket.objects)
for obj in bucket.objects.all():
print(obj)
except ClientError as e:
print(e)
Multipart Upload
Multipart upload initiation
- Send a multipart upload initiation request and receive a response with a UploadId.
- You need to use the UploadId with any request, such as uploading parts, complete an upload, or stop an upload.
Parts upload
- You need a uploadId and the part number (1 ~ 10,000).
- The part number does not need to be in a consecutive sequence.
- S3 returns an ETag header in its response.
Multipart upload completion
- S3 creates an object by concatenating all parts in ascending order (part number).
Multipart Upload API Calls
- CreateMultipartUpload ()
- returns Bucket, Key, and UploadID.
- UnloadPart (Bucket, Key, Part number, UploadID)
- returns ETag.
- CompleteMultipartUpload (Bucket, Key, UploadID, all Part numbers, all ETags)
- returns Bucket, ETag, and Key.
import boto3
import json
from botocore.exceptions import ClientError
key = 'hello.txt'
bucket = 'my-test-bucket'
s3_client = boto3.client('s3')
# Initiation - Multipart Upload
response = s3_client.create_multipart_upload(
Bucket = bucket,
Key = key
)
upload_id = response['UploadId']
print(f'Starting upload: {upload_id}')
# read a file
file = 'hello.txt'
file_upload = open(file, 'rb')
chunk_size = (16 * 1024) * 1024 # 16MB
part_number = 1
chunk = file_upload.read(chunk_size)
# Upload each part
parts = []
while len(chunk) > 0:
response = s3_client.upload_part(
Body = chunk,
Bucket = bucket,
Key = key,
PartNumber = part_number,
UploadId = upload_id
)
print(f"Part Uploaded: {part_number}, ETag: {response['ETag']}")
parts.put({'PartNumber': part_number, 'ETag': response['ETag']})
part_number += 1
chunk = file_upload.read(chunk_size)
# End - Multipart Upload
response = s3_client.complete_multipart_upload(
Bucket = bucket,
Key = key,
MultipartUpload={
'Parts': parts
},
UploadId = upload_id
)
print(json.dumps(response, sort_keys=True, indent=4))
Scriptorium 