diff --git a/.codegen.json b/.codegen.json index 0c86ca5b..bf9506f7 100644 --- a/.codegen.json +++ b/.codegen.json @@ -1 +1 @@ -{ "engineHash": "3d7ee46", "specHash": "88cd5aa", "version": "4.15.0" } +{ "engineHash": "0072e8b", "specHash": "88cd5aa", "version": "4.15.0" } diff --git a/box_sdk_gen/managers/chunked_uploads.py b/box_sdk_gen/managers/chunked_uploads.py index 3c6a5291..3b790a17 100644 --- a/box_sdk_gen/managers/chunked_uploads.py +++ b/box_sdk_gen/managers/chunked_uploads.py @@ -20,6 +20,8 @@ from box_sdk_gen.internal.utils import Iterator +from box_sdk_gen.schemas.upload_part_plan_hit import UploadPartPlanHit + from box_sdk_gen.schemas.upload_session import UploadSession from box_sdk_gen.schemas.client_error import ClientError @@ -81,12 +83,15 @@ def __init__( file_size: int, upload_part_url: str, file_hash: Hash, + *, + plan_url: str = '' ): self.last_index = last_index self.parts = parts self.file_size = file_size self.upload_part_url = upload_part_url self.file_hash = file_hash + self.plan_url = plan_url class ChunkedUploadsManager: @@ -932,6 +937,7 @@ def _reducer(self, acc: _PartAccumulator, chunk: ByteStream) -> _PartAccumulator file_size=acc.file_size, upload_part_url=acc.upload_part_url, file_hash=acc.file_hash, + plan_url=acc.plan_url, ) def upload_big_file( @@ -982,3 +988,129 @@ def upload_big_file( self.create_file_upload_session_commit_by_url(commit_url, parts, digest) ) return committed_session.entries[0] + + def _get_cached_upload_part( + self, plan_url: str, offset: int, size: int, sha_512: str + ) -> Optional[UploadPart]: + plan: UploadSessionPlanResponse = self.create_file_upload_session_plan_by_url( + plan_url, [UploadPartPlan(offset=offset, size=size, sha_512=sha_512)] + ) + if len(plan.hits) > 0: + hit: UploadPartPlanHit = plan.hits[0] + return UploadPart(part_id=hit.part_id, offset=hit.offset, size=hit.size) + return None + + def _reducer_for_file_version( + self, acc: _PartAccumulator, chunk: ByteStream + ) -> _PartAccumulator: + last_index: int = acc.last_index + parts: List[UploadPart] = acc.parts + chunk_buffer: Buffer = read_byte_stream(chunk) + hash: Hash = Hash(algorithm=HashName.SHA1) + hash.update_hash(chunk_buffer) + sha_1: str = hash.digest_hash('base64') + digest: str = ''.join(['sha=', sha_1]) + chunk_size: int = buffer_length(chunk_buffer) + bytes_start: int = last_index + 1 + bytes_end: int = last_index + chunk_size + content_range: str = ''.join( + [ + 'bytes ', + to_string(bytes_start), + '-', + to_string(bytes_end), + '/', + to_string(acc.file_size), + ] + ) + sha_512_hash: Hash = Hash(algorithm=HashName.SHA512) + sha_512_hash.update_hash(chunk_buffer) + sha_512: str = sha_512_hash.digest_hash('hex') + cached_part: Optional[UploadPart] = self._get_cached_upload_part( + acc.plan_url, bytes_start, chunk_size, sha_512 + ) + if not cached_part == None: + acc.file_hash.update_hash(chunk_buffer) + return _PartAccumulator( + last_index=bytes_end, + parts=parts + [cached_part], + file_size=acc.file_size, + upload_part_url=acc.upload_part_url, + file_hash=acc.file_hash, + plan_url=acc.plan_url, + ) + uploaded_part: UploadedPart = self.upload_file_part_by_url( + acc.upload_part_url, + generate_byte_stream_from_buffer(chunk_buffer), + digest, + content_range, + ) + part: UploadPart = uploaded_part.part + part_sha_1: str = hex_to_base_64(part.sha_1) + assert part_sha_1 == sha_1 + assert part.size == chunk_size + assert part.offset == bytes_start + acc.file_hash.update_hash(chunk_buffer) + return _PartAccumulator( + last_index=bytes_end, + parts=parts + [part], + file_size=acc.file_size, + upload_part_url=acc.upload_part_url, + file_hash=acc.file_hash, + plan_url=acc.plan_url, + ) + + def upload_big_file_version( + self, + file_id: str, + file: ByteStream, + file_size: int, + *, + file_name: Optional[str] = None + ) -> Optional[FileFull]: + """ + Starts the process of chunk uploading a new version of a big file. Should return a File object representing the uploaded file version. Returns nothing when commit responds with 202 because the file did not change. + :param file_id: The ID of the file to upload a new version of. + :type file_id: str + :param file: The stream of the file to upload. + :type file: ByteStream + :param file_size: The total size of the file for the chunked upload in bytes. + :type file_size: int + :param file_name: The optional new name of the file., defaults to None + :type file_name: Optional[str], optional + """ + upload_session: UploadSession = ( + self.create_file_upload_session_for_existing_file( + file_id, file_size, file_name=file_name + ) + ) + upload_part_url: str = upload_session.session_endpoints.upload_part + commit_url: str = upload_session.session_endpoints.commit + plan_url: str = upload_session.session_endpoints.plan + part_size: int = upload_session.part_size + total_parts: int = upload_session.total_parts + assert part_size * total_parts >= file_size + assert upload_session.num_parts_processed == 0 + file_hash: Hash = Hash(algorithm=HashName.SHA1) + chunks_iterator: Iterator = iterate_chunks(file, part_size, file_size) + results: _PartAccumulator = reduce_iterator( + chunks_iterator, + self._reducer_for_file_version, + _PartAccumulator( + last_index=-1, + parts=[], + file_size=file_size, + upload_part_url=upload_part_url, + file_hash=file_hash, + plan_url=plan_url, + ), + ) + parts: List[UploadPart] = results.parts + sha_1: str = file_hash.digest_hash('base64') + digest: str = ''.join(['sha=', sha_1]) + committed_session: Optional[Files] = ( + self.create_file_upload_session_commit_by_url(commit_url, parts, digest) + ) + if committed_session == None: + return None + return committed_session.entries[0] diff --git a/docs/box_sdk_gen/chunked_uploads.md b/docs/box_sdk_gen/chunked_uploads.md index 9f4e26ed..87fae8b0 100644 --- a/docs/box_sdk_gen/chunked_uploads.md +++ b/docs/box_sdk_gen/chunked_uploads.md @@ -17,6 +17,7 @@ This is a manager for chunked uploads (allowed for files at least 20MB). - [Commit upload session by URL](#commit-upload-session-by-url) - [Commit upload session](#commit-upload-session) - [Upload big file](#upload-big-file) +- [Upload big file version](#upload-big-file-version) ## Create upload session @@ -532,3 +533,28 @@ client.chunked_uploads.upload_big_file(generate_byte_stream_from_buffer(file_buf ### Returns This function returns a value of type `FileFull`. + +## Upload big file version + +Starts the process of chunk uploading a new version of a big file. Should return a File object representing the uploaded file version. Returns nothing when commit responds with 202 because the file did not change. + +This operation is performed by calling function `upload_big_file_version`. + +```python +client.chunked_uploads.upload_big_file_version(uploaded_file.id, generate_byte_stream(version_file_size), version_file_size, file_name=version_name) +``` + +### Arguments + +- file_id `str` + - The ID of the file to upload a new version of. +- file `ByteStream` + - The stream of the file to upload. +- file_size `int` + - The total size of the file for the chunked upload in bytes. +- file_name `Optional[str]` + - The optional new name of the file. + +### Returns + +This function returns a value of type `Optional[FileFull]`. diff --git a/test/box_sdk_gen/test/chunked_uploads.py b/test/box_sdk_gen/test/chunked_uploads.py index 8f7bde27..73e77f34 100644 --- a/test/box_sdk_gen/test/chunked_uploads.py +++ b/test/box_sdk_gen/test/chunked_uploads.py @@ -12,6 +12,8 @@ from box_sdk_gen.schemas.upload_part_plan_hit import UploadPartPlanHit +from box_sdk_gen.schemas.file_full import FileFull + from box_sdk_gen.internal.utils import generate_byte_stream_from_buffer from box_sdk_gen.internal.utils import hex_to_base_64 @@ -347,3 +349,31 @@ def testChunkedUploadConvenienceMethod(): assert uploaded_file.size == file_size assert uploaded_file.parent.id == parent_folder_id client.files.delete_file_by_id(uploaded_file.id) + + +def testChunkedUploadFileVersionConvenienceMethod(): + file_name: str = get_uuid() + file_size: int = (20 * 1024) * 1024 + parent_folder_id: str = '0' + uploaded_file: File = client.chunked_uploads.upload_big_file( + generate_byte_stream(file_size), file_name, file_size, parent_folder_id + ) + assert uploaded_file.name == file_name + assert uploaded_file.size == file_size + version_file_size: int = (21 * 1024) * 1024 + version_name: str = get_uuid() + uploaded_file_version: Optional[FileFull] = ( + client.chunked_uploads.upload_big_file_version( + uploaded_file.id, + generate_byte_stream(version_file_size), + version_file_size, + file_name=version_name, + ) + ) + assert not uploaded_file_version == None + assert uploaded_file_version.id == uploaded_file.id + assert uploaded_file_version.name == version_name + assert uploaded_file_version.size == version_file_size + assert not uploaded_file_version.name == file_name + assert not uploaded_file_version.size == uploaded_file.size + client.files.delete_file_by_id(uploaded_file.id)