From 73201a326f8c6514669b66a61f40dcfe87e0c1c1 Mon Sep 17 00:00:00 2001 From: Michael Chen Date: Wed, 10 Apr 2024 18:00:24 -0400 Subject: [PATCH 1/4] add support for resumable uploads --- google/generativeai/client.py | 11 ++++++++++- google/generativeai/files.py | 24 +++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/google/generativeai/client.py b/google/generativeai/client.py index 80e3c3b76..64ff6334d 100644 --- a/google/generativeai/client.py +++ b/google/generativeai/client.py @@ -59,6 +59,8 @@ def create_file( mime_type: str | None = None, name: str | None = None, display_name: str | None = None, + resumable: bool | None = None, + chunksize: float | None = None, ) -> glm.File: if self._discovery_api is None: self._setup_discovery_api() @@ -69,7 +71,14 @@ def create_file( if display_name is not None: file["displayName"] = display_name - media = googleapiclient.http.MediaFileUpload(filename=path, mimetype=mime_type) + # Rely on MediaFileUpload for resumable uploads. + if resumable is not None: + if chunksize is None: + chunksize = googleapiclient.http.DEFAULT_CHUNK_SIZE + media = googleapiclient.http.MediaFileUpload( + filename=path, mimetype=mime_type, resumable=resumable, chunksize=chunksize) + else: + media = googleapiclient.http.MediaFileUpload(filename=path, mimetype=mime_type) request = self._discovery_api.media().upload(body={"file": file}, media_body=media) result = request.execute() diff --git a/google/generativeai/files.py b/google/generativeai/files.py index ffd9e6ed8..4ad654387 100644 --- a/google/generativeai/files.py +++ b/google/generativeai/files.py @@ -35,7 +35,28 @@ def upload_file( mime_type: str | None = None, name: str | None = None, display_name: str | None = None, + resumable: bool | None = None, + chunksize: float | None = None, ) -> file_types.File: + """Uploads a file using a supported file service. + + Args: + path: The path to the file to be uploaded. + mime_type: The MIME type of the file. If not provided, it will be + inferred from the file extension. + name: The name of the file in the destination (e.g., 'files/sample-image'). + If not provided, a system generated ID will be created. + display_name: Optional display name of the file. + resumable: Whether to use the resumable upload protocol. By default, this is disabled. + See details at + https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaFileUpload-class.html#resumable + chunksize: The size of file chunks for resumable uploads, in bytes. + If not provided, a default value is used. For more information, + see https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaFileUpload-class.html#chunksize + + Returns: + file_types.File: The response of the uploaded file. + """ client = get_default_file_client() path = pathlib.Path(os.fspath(path)) @@ -50,7 +71,8 @@ def upload_file( display_name = path.name response = client.create_file( - path=path, mime_type=mime_type, name=name, display_name=display_name + path=path, mime_type=mime_type, name=name, display_name=display_name, + resumable=resumable, chunksize=chunksize ) return file_types.File(response) From a2b91d11ff2cbe07a8633bf3badf5131348a0b58 Mon Sep 17 00:00:00 2001 From: Michael Chen Date: Wed, 10 Apr 2024 20:46:17 -0400 Subject: [PATCH 2/4] run formatter --- google/generativeai/client.py | 3 ++- google/generativeai/files.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/google/generativeai/client.py b/google/generativeai/client.py index 64ff6334d..455b8cc3f 100644 --- a/google/generativeai/client.py +++ b/google/generativeai/client.py @@ -76,7 +76,8 @@ def create_file( if chunksize is None: chunksize = googleapiclient.http.DEFAULT_CHUNK_SIZE media = googleapiclient.http.MediaFileUpload( - filename=path, mimetype=mime_type, resumable=resumable, chunksize=chunksize) + filename=path, mimetype=mime_type, resumable=resumable, chunksize=chunksize + ) else: media = googleapiclient.http.MediaFileUpload(filename=path, mimetype=mime_type) request = self._discovery_api.media().upload(body={"file": file}, media_body=media) diff --git a/google/generativeai/files.py b/google/generativeai/files.py index 4ad654387..9cb969005 100644 --- a/google/generativeai/files.py +++ b/google/generativeai/files.py @@ -56,7 +56,7 @@ def upload_file( Returns: file_types.File: The response of the uploaded file. - """ + """ client = get_default_file_client() path = pathlib.Path(os.fspath(path)) @@ -71,8 +71,12 @@ def upload_file( display_name = path.name response = client.create_file( - path=path, mime_type=mime_type, name=name, display_name=display_name, - resumable=resumable, chunksize=chunksize + path=path, + mime_type=mime_type, + name=name, + display_name=display_name, + resumable=resumable, + chunksize=chunksize, ) return file_types.File(response) From cd7671c7e66c7743f939e8fce6fb54d07fc655d5 Mon Sep 17 00:00:00 2001 From: Michael Chen Date: Wed, 10 Apr 2024 20:59:47 -0400 Subject: [PATCH 3/4] update docs --- google/generativeai/client.py | 2 +- google/generativeai/files.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google/generativeai/client.py b/google/generativeai/client.py index 455b8cc3f..124500530 100644 --- a/google/generativeai/client.py +++ b/google/generativeai/client.py @@ -60,7 +60,7 @@ def create_file( name: str | None = None, display_name: str | None = None, resumable: bool | None = None, - chunksize: float | None = None, + chunksize: int | None = None, ) -> glm.File: if self._discovery_api is None: self._setup_discovery_api() diff --git a/google/generativeai/files.py b/google/generativeai/files.py index 9cb969005..875823a3c 100644 --- a/google/generativeai/files.py +++ b/google/generativeai/files.py @@ -36,7 +36,7 @@ def upload_file( name: str | None = None, display_name: str | None = None, resumable: bool | None = None, - chunksize: float | None = None, + chunksize: int | None = None, ) -> file_types.File: """Uploads a file using a supported file service. @@ -50,7 +50,7 @@ def upload_file( resumable: Whether to use the resumable upload protocol. By default, this is disabled. See details at https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaFileUpload-class.html#resumable - chunksize: The size of file chunks for resumable uploads, in bytes. + chunksize: The size of file chunk bytes for resumable uploads, in int. If not provided, a default value is used. For more information, see https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaFileUpload-class.html#chunksize From c356dee86639ba803b81f13db95c1972a2565217 Mon Sep 17 00:00:00 2001 From: Michael Chen Date: Tue, 7 May 2024 10:22:12 -0400 Subject: [PATCH 4/4] simplify resumable flow and set as default --- google/generativeai/client.py | 15 ++++----------- google/generativeai/files.py | 15 +++------------ 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/google/generativeai/client.py b/google/generativeai/client.py index 124500530..d9935e1ec 100644 --- a/google/generativeai/client.py +++ b/google/generativeai/client.py @@ -59,8 +59,7 @@ def create_file( mime_type: str | None = None, name: str | None = None, display_name: str | None = None, - resumable: bool | None = None, - chunksize: int | None = None, + resumable: bool = True, ) -> glm.File: if self._discovery_api is None: self._setup_discovery_api() @@ -71,15 +70,9 @@ def create_file( if display_name is not None: file["displayName"] = display_name - # Rely on MediaFileUpload for resumable uploads. - if resumable is not None: - if chunksize is None: - chunksize = googleapiclient.http.DEFAULT_CHUNK_SIZE - media = googleapiclient.http.MediaFileUpload( - filename=path, mimetype=mime_type, resumable=resumable, chunksize=chunksize - ) - else: - media = googleapiclient.http.MediaFileUpload(filename=path, mimetype=mime_type) + media = googleapiclient.http.MediaFileUpload( + filename=path, mimetype=mime_type, resumable=resumable + ) request = self._discovery_api.media().upload(body={"file": file}, media_body=media) result = request.execute() diff --git a/google/generativeai/files.py b/google/generativeai/files.py index 875823a3c..13535c47f 100644 --- a/google/generativeai/files.py +++ b/google/generativeai/files.py @@ -35,8 +35,7 @@ def upload_file( mime_type: str | None = None, name: str | None = None, display_name: str | None = None, - resumable: bool | None = None, - chunksize: int | None = None, + resumable: bool = True, ) -> file_types.File: """Uploads a file using a supported file service. @@ -47,12 +46,9 @@ def upload_file( name: The name of the file in the destination (e.g., 'files/sample-image'). If not provided, a system generated ID will be created. display_name: Optional display name of the file. - resumable: Whether to use the resumable upload protocol. By default, this is disabled. + resumable: Whether to use the resumable upload protocol. By default, this is enabled. See details at https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaFileUpload-class.html#resumable - chunksize: The size of file chunk bytes for resumable uploads, in int. - If not provided, a default value is used. For more information, - see https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaFileUpload-class.html#chunksize Returns: file_types.File: The response of the uploaded file. @@ -71,12 +67,7 @@ def upload_file( display_name = path.name response = client.create_file( - path=path, - mime_type=mime_type, - name=name, - display_name=display_name, - resumable=resumable, - chunksize=chunksize, + path=path, mime_type=mime_type, name=name, display_name=display_name, resumable=resumable ) return file_types.File(response)