ContinuePresignedUpload

continue_presigned_upload(file_path, is_multipart_upload, parts)
Expand source code
continue_presigned_upload(file_path, is_multipart_upload, parts):
"""
Description
-----------
High level wrapper function to upload artifact.
It checks to upload file in chunks or not based on the size.
Args
----
file_path: Local path of the artifact to be uploaded.
is_multipart_upload: flag returned by post_artifacts_presigned_startupload.
parts: presigned urls returned by post_artifacts_presigned_startupload.
Returns
-------
parts: parts with ETag updated for multipart upload.
Usages
------
parts = continue_presigned_upload(file_path, start_upload_response.parsed.is_multipart_upload, start_upload_response.parsed.parts)
"""
try:
with open(file_path, 'rb') as f:
for part_index, part in enumerate(parts):
chunk = f.read(CHUNK_SIZE)
if not chunk:
break
print(f"uploading file: {file_path}, part {part_index + 1}/{len(parts)}")
response = requests.put(part.url, data=chunk)
if response.status_code != 200:
raise Exception(f"failed to upload file: {file_path}, part {part_index + 1}: {response.status_code}, {response.content.decode('utf-8')}")
if is_multipart_upload:
etag = response.headers.get('ETag')
if not etag:
raise Exception(f"missing ETag for file: {file_path}, part {part_index + 1}")
part.e_tag = etag
print(f"successfully uploaded file: {file_path}, part {part_index + 1}, etag: {etag if is_multipart_upload else 'N/A'}")
return parts
except Exception as e:
raise Exception(f"error during chunk upload file: {file_path}: {e}")

Description

High level wrapper function to upload artifact. It checks to upload file in chunks or not based on the size.

Arguments

  • file_path: Local path of the artifact to be uploaded.
  • is_multipart_upload: flag returned by post_artifacts_presigned_startupload.
  • parts: presigned urls returned by post_artifacts_presigned_startupload.

Returns

  • parts: parts with ETag updated for multipart upload.

Usages

parts = continue_presigned_upload(file_path, start_upload_response.parsed.is_multipart_upload, start_upload_response.parsed.parts)