From 0f177fd8e68f857dfdcb50aeec00ace443ef341c Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Fri, 1 Aug 2025 17:13:05 +0800 Subject: [PATCH 01/17] fix --- modelscope/hub/api.py | 65 +++++++++-- modelscope/hub/utils/aigc.py | 212 +++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 8 deletions(-) create mode 100644 modelscope/hub/utils/aigc.py diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index da97e2d63..fc2e72ff9 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -4,6 +4,7 @@ import datetime import fnmatch import functools +import glob import io import os import pickle @@ -61,6 +62,7 @@ raise_for_http_status, raise_on_error) from modelscope.hub.git import GitCommandWrapper from modelscope.hub.repository import Repository +from modelscope.hub.utils.aigc import DEFAULT_AIGC_COVER_IMAGE, AigcModel from modelscope.hub.utils.utils import (add_content_to_file, get_domain, get_endpoint, get_readable_folder_size, get_release_datetime, is_env_true, @@ -188,22 +190,27 @@ def create_model(self, chinese_name: Optional[str] = None, original_model_id: Optional[str] = '', endpoint: Optional[str] = None, - token: Optional[str] = None) -> str: + token: Optional[str] = None, + aigc_config: Optional['AigcModel'] = None, + **kwargs) -> str: """Create model repo at ModelScope Hub. Args: - model_id (str): The model id + model_id (str): The model id in format {owner}/{name} visibility (int, optional): visibility of the model(1-private, 5-public), default 5. - license (str, optional): license of the model, default none. + license (str, optional): license of the model, default apache-2.0. chinese_name (str, optional): chinese name of the model. original_model_id (str, optional): the base model id which this model is trained from endpoint: the endpoint to use, default to None to use endpoint specified in the class + token (str, optional): access token for authentication + aigc_config (AigcModel, optional): AigcModel instance for AIGC model creation. + If provided, will create an AIGC model with automatic file upload. Returns: - Name of the model created + str: URL of the created model repository Raises: - InvalidParameter: If model_id is invalid. + InvalidParameter: If model_id is invalid or required AIGC parameters are missing. ValueError: If not login. Note: @@ -219,17 +226,54 @@ def create_model(self, cookies = self.get_cookies(token) if not endpoint: endpoint = self.endpoint - path = f'{endpoint}/api/v1/models' + owner_or_group, name = model_id_to_group_owner_name(model_id) + + # Base body configuration body = { 'Path': owner_or_group, 'Name': name, 'ChineseName': chinese_name, - 'Visibility': visibility, # server check + 'Visibility': visibility, 'License': license, 'OriginalModelId': original_model_id, - 'TrainId': os.environ.get('MODELSCOPE_TRAIN_ID', ''), + 'TrainId': os.environ.get('MODELSCOPE_TRAIN_ID', '') } + + # Set path based on model type + if aigc_config is not None: + # Use AIGC model endpoint + path = f'{endpoint}/api/v1/models/aigc' + + # Validate AIGC parameters + valid_aigc_types = [v.value for v in AigcModel.AigcType] + valid_vision_foundations = [v.value for v in AigcModel.BaseModelType] + + assert aigc_config.aigc_type.value in valid_aigc_types, ( + f'aigc_type must be one of {valid_aigc_types}, got: {aigc_config.aigc_type.value}' + ) + assert aigc_config.base_model_type.value in valid_vision_foundations, ( + f'vision_foundation must be one of {valid_vision_foundations}, got: {aigc_config.base_model_type.value}' + ) + + # Add AIGC-specific fields to body + body.update({ + 'TagShowName': aigc_config.tag, + 'CoverImages': aigc_config.cover_images, + 'AigcType': aigc_config.aigc_type.value, + 'TagDescription': aigc_config.tag_description, + 'VisionFoundation': aigc_config.base_model_type.value, + 'BaseModel': aigc_config.base_model_id, + 'WeightsName': aigc_config.weights_filename, + 'WeightsSha256': aigc_config.weights_sha256, + 'WeightsSize': aigc_config.weights_size, + 'ModelPath': aigc_config.model_path + }) + + else: + # Use regular model endpoint + path = f'{endpoint}/api/v1/models' + r = self.session.post( path, json=body, @@ -238,6 +282,11 @@ def create_model(self, handle_http_post_error(r, path, body) raise_on_error(r.json()) model_repo_url = f'{endpoint}/models/{model_id}' + + # Upload model files for AIGC models + if aigc_config is not None: + aigc_config.upload_to_repo(self, model_id, token) + return model_repo_url def delete_model(self, model_id: str, endpoint: Optional[str] = None): diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py new file mode 100644 index 000000000..7c0e1fc7e --- /dev/null +++ b/modelscope/hub/utils/aigc.py @@ -0,0 +1,212 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import glob +import os +from enum import Enum +from typing import List, Optional + +# Default AIGC model cover image +DEFAULT_AIGC_COVER_IMAGE = ( + 'https://modelscope.cn/models/modelscope/modelscope_aigc_default_logo/resolve/master/' + 'aigc_default_logo.png') + + +class AigcModel: + """ + Helper class to encapsulate AIGC-specific model creation parameters. + """ + + class AigcType(str, Enum): + CHECKPOINT = 'Checkpoint' + LORA = 'LoRA' + VAE = 'VAE' + + class BaseModelType(str, Enum): + SD_1_5 = 'SD_1_5' + SD_XL = 'SD_XL' + SD_3 = 'SD_3' + FLUX_1 = 'FLUX_1' + WAN_VIDEO_2_1_T2V_1_3_B = 'WAN_VIDEO_2_1_T2V_1_3_B' + WAN_VIDEO_2_1_T2V_14_B = 'WAN_VIDEO_2_1_T2V_14_B' + WAN_VIDEO_2_1_I2V_14_B = 'WAN_VIDEO_2_1_I2V_14_B' + WAN_VIDEO_2_1_FLF2V_14_B = 'WAN_VIDEO_2_1_FLF2V_14_B' + WAN_VIDEO_2_2_T2V_5_B = 'WAN_VIDEO_2_2_T2V_5_B' + WAN_VIDEO_2_2_T2V_14_B = 'WAN_VIDEO_2_2_T2V_14_B' + WAN_VIDEO_2_2_I2V_14_B = 'WAN_VIDEO_2_2_I2V_14_B' + + def __init__( + self, + aigc_type: AigcType, + base_model_type: BaseModelType, + model_path: str, + tag: Optional[str] = 'v1.0', + tag_description: Optional[str] = 'this is an aigc model', + cover_images: Optional[List[str]] = None, + base_model_id: str = '' + ): + """ + Initializes the AigcModel helper. + + Args: + aigc_type (AigcType): AIGC model type. + Valid values: Checkpoint, LoRA, VAE + base_model_type (BaseModelType): Vision foundation model. + Valid values: SD_1_5, SD_XL, SD_3, FLUX_1, WAN_VIDEO_2_1_T2V_1_3_B... + model_path (str, required): The path of checkpoint/LoRA weights file (.safetensors) or folder + tag (str, optional): Tag name for AIGC model, default 'v1.0' + tag_description (str, optional): Tag description, + default: 'this is a aigc model' + cover_images (List[str], optional): List of cover image URLs, + default: DEFAULT_AIGC_COVER_IMAGE + base_model_id (str, optional): Base model name, + default: '', e.g.'AI-ModelScope/FLUX.1-dev' + """ + self.aigc_type = aigc_type + self.base_model_type = base_model_type + self.model_path = model_path + self.tag = tag + self.tag_description = tag_description + self.cover_images = cover_images if cover_images is not None else [ + DEFAULT_AIGC_COVER_IMAGE + ] + self.base_model_id = base_model_id + + # Process model path and calculate weights information + self._process_model_path() + + def _process_model_path(self): + """Process model_path to extract weights information""" + from modelscope.utils.file_utils import get_file_hash + + # Expand user path + self.model_path = os.path.expanduser(self.model_path) + + if not os.path.exists(self.model_path): + raise ValueError(f'Model path does not exist: {self.model_path}') + + target_file = None + + if os.path.isfile(self.model_path): + target_file = self.model_path + print(f'Using file: {os.path.basename(target_file)}') + elif os.path.isdir(self.model_path): + safetensors_files = glob.glob( + os.path.join(self.model_path, '*.safetensors')) + + if safetensors_files: + target_file = safetensors_files[0] + print( + f'✅ Found safetensors file: {os.path.basename(target_file)}' + ) + if len(safetensors_files) > 1: + print( + f'Multiple safetensors files found, using: {os.path.basename(target_file)}' + ) + print( + f'Other safetensors: {[os.path.basename(f) for f in safetensors_files[1:]]}' + ) + else: + # No .safetensors files, try to find any other file + all_files = [ + f for f in os.listdir(self.model_path) + if os.path.isfile(os.path.join(self.model_path, f)) + ] + + if all_files: + # Use the first available file + target_file = os.path.join(self.model_path, all_files[0]) + print( + f'using: {os.path.basename(target_file)}' + ) + print(f'Available files: {all_files}') + else: + raise ValueError( + f'No files found in directory: {self.model_path}. ' + f'AIGC models require at least one model file (.safetensors recommended).' + ) + + else: + raise ValueError( + f'Model path must be a file or directory: {self.model_path}') + + if target_file: + # Calculate file hash and size for the target file + print(f'Computing hash and size for {target_file}...') + hash_info = get_file_hash(target_file) + + # Store weights information + self.weights_filename = os.path.basename(target_file) + self.weights_sha256 = hash_info['file_hash'] + self.weights_size = hash_info['file_size'] + self.target_file = target_file + + def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): + """Upload model files to repository""" + + if not hasattr(self, 'target_file') or self.target_file is None: + print(f'⚠️ Warning: No specific file to upload for {model_id}') + + if os.path.isdir(self.model_path): + files_in_dir = os.listdir(self.model_path) + if files_in_dir: + print( + f'📁 Uploading directory contents ({len(files_in_dir)} items)...' + ) + try: + api.upload_folder( + repo_id=model_id, + folder_path=self.model_path, + token=token, + commit_message='Upload model folder for AIGC model' + ) + print(f'✅ Successfully uploaded folder to {model_id}') + return True + except Exception as e: + print(f'❌ Failed to upload folder: {e}') + return False + else: + print('❌ Directory is empty, nothing to upload') + return False + else: + print('❌ No files to upload') + return False + + print(f'Uploading model to {model_id}...') + try: + if os.path.isfile(self.model_path): + # Upload single file + api.upload_file( + path_or_fileobj=self.target_file, + path_in_repo=self.weights_filename, + repo_id=model_id, + token=token, + commit_message= + f'Upload {self.weights_filename} for AIGC model') + else: + # Upload entire folder + api.upload_folder( + repo_id=model_id, + folder_path=self.model_path, + token=token, + commit_message='Upload model folder for AIGC model') + print(f'✅ Successfully uploaded model to {model_id}') + return True + except Exception as e: + print(f'⚠️ Warning: Failed to upload model: {e}') + print('You may need to upload the model manually after creation.') + return False + + def to_dict(self) -> dict: + """Converts the AIGC parameters to a dictionary suitable for API calls.""" + return { + 'aigc_type': self.aigc_type.value, + 'base_model_type': self.base_model_type.value, + 'tag': self.tag, + 'tag_description': self.tag_description, + 'cover_images': self.cover_images, + 'base_model_id': self.base_model_id, + 'model_path': self.model_path, + 'weights_filename': self.weights_filename, + 'weights_sha256': self.weights_sha256, + 'weights_size': self.weights_size + } From 16fa0fc8e9dbebf85c2d8390d60ee6da318425b4 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Fri, 1 Aug 2025 17:48:40 +0800 Subject: [PATCH 02/17] fix --- modelscope/hub/api.py | 50 +++++++++++++++-------------- modelscope/hub/utils/aigc.py | 62 ++++++++++++++++++++---------------- 2 files changed, 60 insertions(+), 52 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index fc2e72ff9..7c7aec0ab 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -191,8 +191,7 @@ def create_model(self, original_model_id: Optional[str] = '', endpoint: Optional[str] = None, token: Optional[str] = None, - aigc_config: Optional['AigcModel'] = None, - **kwargs) -> str: + aigc_model: Optional['AigcModel'] = None) -> str: """Create model repo at ModelScope Hub. Args: @@ -216,8 +215,6 @@ def create_model(self, Note: model_id = {owner}/{name} """ - if model_id is None: - raise InvalidParameter('model_id is required!') cookies = ModelScopeConfig.get_cookies() if cookies is None: if token is None: @@ -241,33 +238,38 @@ def create_model(self, } # Set path based on model type - if aigc_config is not None: + if aigc_model is not None: # Use AIGC model endpoint path = f'{endpoint}/api/v1/models/aigc' # Validate AIGC parameters valid_aigc_types = [v.value for v in AigcModel.AigcType] - valid_vision_foundations = [v.value for v in AigcModel.BaseModelType] + valid_vision_foundations = [ + v.value for v in AigcModel.BaseModelType + ] - assert aigc_config.aigc_type.value in valid_aigc_types, ( - f'aigc_type must be one of {valid_aigc_types}, got: {aigc_config.aigc_type.value}' - ) - assert aigc_config.base_model_type.value in valid_vision_foundations, ( - f'vision_foundation must be one of {valid_vision_foundations}, got: {aigc_config.base_model_type.value}' - ) + if aigc_model.aigc_type.value not in valid_aigc_types: + raise InvalidParameter( + f'aigc_type must be one of {valid_aigc_types}, got: {aigc_model.aigc_type.value}' + ) + if aigc_model.base_model_type.value not in valid_vision_foundations: + raise InvalidParameter( + f'vision_foundation must be one of {valid_vision_foundations}, ' + f'got: {aigc_model.base_model_type.value}' + ) # Add AIGC-specific fields to body body.update({ - 'TagShowName': aigc_config.tag, - 'CoverImages': aigc_config.cover_images, - 'AigcType': aigc_config.aigc_type.value, - 'TagDescription': aigc_config.tag_description, - 'VisionFoundation': aigc_config.base_model_type.value, - 'BaseModel': aigc_config.base_model_id, - 'WeightsName': aigc_config.weights_filename, - 'WeightsSha256': aigc_config.weights_sha256, - 'WeightsSize': aigc_config.weights_size, - 'ModelPath': aigc_config.model_path + 'TagShowName': aigc_model.tag, + 'CoverImages': aigc_model.cover_images, + 'AigcType': aigc_model.aigc_type.value, + 'TagDescription': aigc_model.tag_description, + 'VisionFoundation': aigc_model.base_model_type.value, + 'BaseModel': aigc_model.base_model_id, + 'WeightsName': aigc_model.weights_filename, + 'WeightsSha256': aigc_model.weights_sha256, + 'WeightsSize': aigc_model.weights_size, + 'ModelPath': aigc_model.model_path }) else: @@ -284,8 +286,8 @@ def create_model(self, model_repo_url = f'{endpoint}/models/{model_id}' # Upload model files for AIGC models - if aigc_config is not None: - aigc_config.upload_to_repo(self, model_id, token) + if aigc_model is not None: + aigc_model.upload_to_repo(self, model_id, token) return model_repo_url diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index 7c0e1fc7e..ff865d4df 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -5,6 +5,11 @@ from enum import Enum from typing import List, Optional +from modelscope.utils.logger import get_logger + +# Get a logger instance from modelscope +logger = get_logger() + # Default AIGC model cover image DEFAULT_AIGC_COVER_IMAGE = ( 'https://modelscope.cn/models/modelscope/modelscope_aigc_default_logo/resolve/master/' @@ -34,16 +39,14 @@ class BaseModelType(str, Enum): WAN_VIDEO_2_2_T2V_14_B = 'WAN_VIDEO_2_2_T2V_14_B' WAN_VIDEO_2_2_I2V_14_B = 'WAN_VIDEO_2_2_I2V_14_B' - def __init__( - self, - aigc_type: AigcType, - base_model_type: BaseModelType, - model_path: str, - tag: Optional[str] = 'v1.0', - tag_description: Optional[str] = 'this is an aigc model', - cover_images: Optional[List[str]] = None, - base_model_id: str = '' - ): + def __init__(self, + aigc_type: AigcType, + base_model_type: BaseModelType, + model_path: str, + tag: Optional[str] = 'v1.0', + tag_description: Optional[str] = 'this is an aigc model', + cover_images: Optional[List[str]] = None, + base_model_id: str = ''): """ Initializes the AigcModel helper. @@ -88,21 +91,21 @@ def _process_model_path(self): if os.path.isfile(self.model_path): target_file = self.model_path - print(f'Using file: {os.path.basename(target_file)}') + logger.info(f'Using file: {os.path.basename(target_file)}') elif os.path.isdir(self.model_path): safetensors_files = glob.glob( os.path.join(self.model_path, '*.safetensors')) if safetensors_files: target_file = safetensors_files[0] - print( + logger.info( f'✅ Found safetensors file: {os.path.basename(target_file)}' ) if len(safetensors_files) > 1: - print( + logger.warning( f'Multiple safetensors files found, using: {os.path.basename(target_file)}' ) - print( + logger.info( f'Other safetensors: {[os.path.basename(f) for f in safetensors_files[1:]]}' ) else: @@ -115,10 +118,10 @@ def _process_model_path(self): if all_files: # Use the first available file target_file = os.path.join(self.model_path, all_files[0]) - print( - f'using: {os.path.basename(target_file)}' + logger.warning( + f'No safetensors file found, using: {os.path.basename(target_file)}' ) - print(f'Available files: {all_files}') + logger.info(f'Available files: {all_files}') else: raise ValueError( f'No files found in directory: {self.model_path}. ' @@ -131,7 +134,7 @@ def _process_model_path(self): if target_file: # Calculate file hash and size for the target file - print(f'Computing hash and size for {target_file}...') + logger.info(f'Computing hash and size for {target_file}...') hash_info = get_file_hash(target_file) # Store weights information @@ -144,12 +147,13 @@ def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): """Upload model files to repository""" if not hasattr(self, 'target_file') or self.target_file is None: - print(f'⚠️ Warning: No specific file to upload for {model_id}') + logger.warning( + f'⚠️ Warning: No specific file to upload for {model_id}') if os.path.isdir(self.model_path): files_in_dir = os.listdir(self.model_path) if files_in_dir: - print( + logger.info( f'📁 Uploading directory contents ({len(files_in_dir)} items)...' ) try: @@ -159,19 +163,20 @@ def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): token=token, commit_message='Upload model folder for AIGC model' ) - print(f'✅ Successfully uploaded folder to {model_id}') + logger.info( + f'✅ Successfully uploaded folder to {model_id}') return True except Exception as e: - print(f'❌ Failed to upload folder: {e}') + logger.error(f'❌ Failed to upload folder: {e}') return False else: - print('❌ Directory is empty, nothing to upload') + logger.warning('❌ Directory is empty, nothing to upload') return False else: - print('❌ No files to upload') + logger.warning('❌ No files to upload') return False - print(f'Uploading model to {model_id}...') + logger.info(f'Uploading model to {model_id}...') try: if os.path.isfile(self.model_path): # Upload single file @@ -189,11 +194,12 @@ def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): folder_path=self.model_path, token=token, commit_message='Upload model folder for AIGC model') - print(f'✅ Successfully uploaded model to {model_id}') + logger.info(f'✅ Successfully uploaded model to {model_id}') return True except Exception as e: - print(f'⚠️ Warning: Failed to upload model: {e}') - print('You may need to upload the model manually after creation.') + logger.warning(f'⚠️ Warning: Failed to upload model: {e}') + logger.warning( + 'You may need to upload the model manually after creation.') return False def to_dict(self) -> dict: From f7d7850bfe17e618e63c6a02bd51c33d1dfb69e4 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Mon, 4 Aug 2025 15:49:03 +0800 Subject: [PATCH 03/17] restruct aigc utils --- modelscope/hub/api.py | 9 ++-- modelscope/hub/utils/aigc.py | 86 ++++++++++++------------------------ 2 files changed, 32 insertions(+), 63 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 7c7aec0ab..32c7e6f8a 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -4,7 +4,6 @@ import datetime import fnmatch import functools -import glob import io import os import pickle @@ -62,7 +61,7 @@ raise_for_http_status, raise_on_error) from modelscope.hub.git import GitCommandWrapper from modelscope.hub.repository import Repository -from modelscope.hub.utils.aigc import DEFAULT_AIGC_COVER_IMAGE, AigcModel +from modelscope.hub.utils.aigc import AigcModel from modelscope.hub.utils.utils import (add_content_to_file, get_domain, get_endpoint, get_readable_folder_size, get_release_datetime, is_env_true, @@ -202,7 +201,7 @@ def create_model(self, original_model_id (str, optional): the base model id which this model is trained from endpoint: the endpoint to use, default to None to use endpoint specified in the class token (str, optional): access token for authentication - aigc_config (AigcModel, optional): AigcModel instance for AIGC model creation. + aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation. If provided, will create an AIGC model with automatic file upload. Returns: @@ -243,9 +242,9 @@ def create_model(self, path = f'{endpoint}/api/v1/models/aigc' # Validate AIGC parameters - valid_aigc_types = [v.value for v in AigcModel.AigcType] + valid_aigc_types = [item.value for item in AigcModel.AigcType] valid_vision_foundations = [ - v.value for v in AigcModel.BaseModelType + item.value for item in AigcModel.BaseModelType ] if aigc_model.aigc_type.value not in valid_aigc_types: diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index ff865d4df..7086f665b 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -91,23 +91,22 @@ def _process_model_path(self): if os.path.isfile(self.model_path): target_file = self.model_path - logger.info(f'Using file: {os.path.basename(target_file)}') + logger.info('Using file: %s', os.path.basename(target_file)) elif os.path.isdir(self.model_path): safetensors_files = glob.glob( os.path.join(self.model_path, '*.safetensors')) if safetensors_files: target_file = safetensors_files[0] - logger.info( - f'✅ Found safetensors file: {os.path.basename(target_file)}' - ) + logger.info('✅ Found safetensors file: %s', + os.path.basename(target_file)) if len(safetensors_files) > 1: logger.warning( - f'Multiple safetensors files found, using: {os.path.basename(target_file)}' - ) + 'Multiple safetensors files found, using: %s', + os.path.basename(target_file)) logger.info( - f'Other safetensors: {[os.path.basename(f) for f in safetensors_files[1:]]}' - ) + 'Other safetensors: %s', + [os.path.basename(f) for f in safetensors_files[1:]]) else: # No .safetensors files, try to find any other file all_files = [ @@ -118,10 +117,9 @@ def _process_model_path(self): if all_files: # Use the first available file target_file = os.path.join(self.model_path, all_files[0]) - logger.warning( - f'No safetensors file found, using: {os.path.basename(target_file)}' - ) - logger.info(f'Available files: {all_files}') + logger.warning('No safetensors file found, using: %s', + os.path.basename(target_file)) + logger.info('Available files: %s', all_files) else: raise ValueError( f'No files found in directory: {self.model_path}. ' @@ -134,7 +132,7 @@ def _process_model_path(self): if target_file: # Calculate file hash and size for the target file - logger.info(f'Computing hash and size for {target_file}...') + logger.info('Computing hash and size for %s...', target_file) hash_info = get_file_hash(target_file) # Store weights information @@ -144,60 +142,32 @@ def _process_model_path(self): self.target_file = target_file def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): - """Upload model files to repository""" - - if not hasattr(self, 'target_file') or self.target_file is None: - logger.warning( - f'⚠️ Warning: No specific file to upload for {model_id}') - - if os.path.isdir(self.model_path): - files_in_dir = os.listdir(self.model_path) - if files_in_dir: - logger.info( - f'📁 Uploading directory contents ({len(files_in_dir)} items)...' - ) - try: - api.upload_folder( - repo_id=model_id, - folder_path=self.model_path, - token=token, - commit_message='Upload model folder for AIGC model' - ) - logger.info( - f'✅ Successfully uploaded folder to {model_id}') - return True - except Exception as e: - logger.error(f'❌ Failed to upload folder: {e}') - return False - else: - logger.warning('❌ Directory is empty, nothing to upload') - return False - else: - logger.warning('❌ No files to upload') - return False - - logger.info(f'Uploading model to {model_id}...') + """Upload model files to repository.""" + logger.info('Uploading model to %s...', model_id) try: - if os.path.isfile(self.model_path): - # Upload single file - api.upload_file( - path_or_fileobj=self.target_file, - path_in_repo=self.weights_filename, - repo_id=model_id, - token=token, - commit_message= - f'Upload {self.weights_filename} for AIGC model') - else: + if os.path.isdir(self.model_path): # Upload entire folder + logger.info('Uploading directory: %s', self.model_path) api.upload_folder( repo_id=model_id, folder_path=self.model_path, token=token, commit_message='Upload model folder for AIGC model') - logger.info(f'✅ Successfully uploaded model to {model_id}') + elif os.path.isfile(self.model_path): + # Upload single file, target_file is guaranteed to be set by _process_model_path + logger.info('Uploading file: %s', self.target_file) + api.upload_file( + path_or_fileobj=self.target_file, + path_in_repo=self.weights_filename, + repo_id=model_id, + token=token, + commit_message=f'Upload {self.weights_filename} ' + 'for AIGC model') + + logger.info('✅ Successfully uploaded model to %s', model_id) return True except Exception as e: - logger.warning(f'⚠️ Warning: Failed to upload model: {e}') + logger.warning('⚠️ Warning: Failed to upload model: %s', e) logger.warning( 'You may need to upload the model manually after creation.') return False From 3dce6d5329e91c22a200b8d74026c4b25f1ea919 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 10:28:58 +0800 Subject: [PATCH 04/17] fix upload --- modelscope/hub/api.py | 9 ++--- modelscope/hub/utils/aigc.py | 70 +++++++++++++++++++++--------------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 32c7e6f8a..633ab252b 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -259,10 +259,10 @@ def create_model(self, # Add AIGC-specific fields to body body.update({ - 'TagShowName': aigc_model.tag, + 'TagShowName': aigc_model.revision, 'CoverImages': aigc_model.cover_images, 'AigcType': aigc_model.aigc_type.value, - 'TagDescription': aigc_model.tag_description, + 'TagDescription': aigc_model.description, 'VisionFoundation': aigc_model.base_model_type.value, 'BaseModel': aigc_model.base_model_id, 'WeightsName': aigc_model.weights_filename, @@ -284,9 +284,10 @@ def create_model(self, raise_on_error(r.json()) model_repo_url = f'{endpoint}/models/{model_id}' + # TODO: due to server error, the upload function is not working # Upload model files for AIGC models - if aigc_model is not None: - aigc_model.upload_to_repo(self, model_id, token) + # if aigc_model is not None: + # aigc_model.upload_to_repo(self, model_id, token) return model_repo_url diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index 7086f665b..cf9174ca1 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -43,10 +43,11 @@ def __init__(self, aigc_type: AigcType, base_model_type: BaseModelType, model_path: str, - tag: Optional[str] = 'v1.0', - tag_description: Optional[str] = 'this is an aigc model', + revision: Optional[str] = 'v1.0', + description: Optional[str] = 'this is an aigc model', cover_images: Optional[List[str]] = None, - base_model_id: str = ''): + base_model_id: str = '', + path_in_repo: Optional[str] = ''): """ Initializes the AigcModel helper. @@ -56,23 +57,25 @@ def __init__(self, base_model_type (BaseModelType): Vision foundation model. Valid values: SD_1_5, SD_XL, SD_3, FLUX_1, WAN_VIDEO_2_1_T2V_1_3_B... model_path (str, required): The path of checkpoint/LoRA weights file (.safetensors) or folder - tag (str, optional): Tag name for AIGC model, default 'v1.0' - tag_description (str, optional): Tag description, + revision (str, optional): revision for AIGC model, default 'master' + description (str, optional): Model description, default: 'this is a aigc model' cover_images (List[str], optional): List of cover image URLs, default: DEFAULT_AIGC_COVER_IMAGE base_model_id (str, optional): Base model name, default: '', e.g.'AI-ModelScope/FLUX.1-dev' + path_in_repo (str, optional): Path in repository """ self.aigc_type = aigc_type self.base_model_type = base_model_type self.model_path = model_path - self.tag = tag - self.tag_description = tag_description + self.revision = revision + self.description = description self.cover_images = cover_images if cover_images is not None else [ DEFAULT_AIGC_COVER_IMAGE ] self.base_model_id = base_model_id + self.path_in_repo = path_in_repo # Process model path and calculate weights information self._process_model_path() @@ -93,32 +96,36 @@ def _process_model_path(self): target_file = self.model_path logger.info('Using file: %s', os.path.basename(target_file)) elif os.path.isdir(self.model_path): - safetensors_files = glob.glob( - os.path.join(self.model_path, '*.safetensors')) - - if safetensors_files: - target_file = safetensors_files[0] - logger.info('✅ Found safetensors file: %s', + # Priority order for metadata file: safetensors -> pth -> bin -> first file + file_extensions = ['.safetensors', '.pth', '.bin'] + target_file = None + + for ext in file_extensions: + files = glob.glob(os.path.join(self.model_path, f'*{ext}')) + if files: + target_file = files[0] + logger.info(f'✅ Found {ext} file: %s', + os.path.basename(target_file)) + if len(files) > 1: + logger.warning( + f'Multiple {ext} files found, using: %s for metadata', os.path.basename(target_file)) - if len(safetensors_files) > 1: - logger.warning( - 'Multiple safetensors files found, using: %s', - os.path.basename(target_file)) - logger.info( - 'Other safetensors: %s', - [os.path.basename(f) for f in safetensors_files[1:]]) - else: - # No .safetensors files, try to find any other file + logger.info(f'Other {ext} files: %s', + [os.path.basename(f) for f in files[1:]]) + break + + # If no preferred files found, use the first available file + if not target_file: all_files = [ f for f in os.listdir(self.model_path) if os.path.isfile(os.path.join(self.model_path, f)) ] if all_files: - # Use the first available file target_file = os.path.join(self.model_path, all_files[0]) - logger.warning('No safetensors file found, using: %s', - os.path.basename(target_file)) + logger.warning( + 'No safetensors/pth/bin files found, using: %s for metadata', + os.path.basename(target_file)) logger.info('Available files: %s', all_files) else: raise ValueError( @@ -146,19 +153,24 @@ def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): logger.info('Uploading model to %s...', model_id) try: if os.path.isdir(self.model_path): - # Upload entire folder + # Upload entire folder with path_in_repo support logger.info('Uploading directory: %s', self.model_path) api.upload_folder( + revision=self.revision, repo_id=model_id, folder_path=self.model_path, + path_in_repo=self.path_in_repo, token=token, commit_message='Upload model folder for AIGC model') elif os.path.isfile(self.model_path): # Upload single file, target_file is guaranteed to be set by _process_model_path logger.info('Uploading file: %s', self.target_file) api.upload_file( + revision=self.revision, path_or_fileobj=self.target_file, - path_in_repo=self.weights_filename, + path_in_repo=self.path_in_repo + '/' + + self.weights_filename + if self.path_in_repo else self.weights_filename, repo_id=model_id, token=token, commit_message=f'Upload {self.weights_filename} ' @@ -177,8 +189,8 @@ def to_dict(self) -> dict: return { 'aigc_type': self.aigc_type.value, 'base_model_type': self.base_model_type.value, - 'tag': self.tag, - 'tag_description': self.tag_description, + 'revision': self.revision, + 'description': self.description, 'cover_images': self.cover_images, 'base_model_id': self.base_model_id, 'model_path': self.model_path, From c854486149548014fb381c190b9b85b7576b3f05 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 11:23:04 +0800 Subject: [PATCH 05/17] fix cokkies --- modelscope/hub/api.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 633ab252b..77bf9d3bd 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -214,12 +214,13 @@ def create_model(self, Note: model_id = {owner}/{name} """ - cookies = ModelScopeConfig.get_cookies() - if cookies is None: - if token is None: + # Get cookies for authentication. + if token: + cookies = self.get_cookies(access_token=token) + else: + cookies = ModelScopeConfig.get_cookies() + if cookies is None: raise ValueError('Token does not exist, please login first.') - else: - cookies = self.get_cookies(token) if not endpoint: endpoint = self.endpoint @@ -442,9 +443,11 @@ def repo_exists( if (repo_id is None) or repo_id.count('/') != 1: raise Exception('Invalid repo_id: %s, must be of format namespace/name' % repo_type) - cookies = ModelScopeConfig.get_cookies() - if cookies is None and token is not None: - cookies = self.get_cookies(token) + # Get cookies for authentication, following upload.py pattern + if token: + cookies = self.get_cookies(access_token=token) + else: + cookies = ModelScopeConfig.get_cookies() owner_or_group, name = model_id_to_group_owner_name(repo_id) if (repo_type is not None) and repo_type.lower() == REPO_TYPE_DATASET: path = f'{endpoint}/api/v1/datasets/{owner_or_group}/{name}' From 4b89ec48053f70007dae398f3c1e4c92b7520e6e Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 14:36:35 +0800 Subject: [PATCH 06/17] fix --- modelscope/hub/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 77bf9d3bd..ae168910a 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -203,6 +203,7 @@ def create_model(self, token (str, optional): access token for authentication aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation. If provided, will create an AIGC model with automatic file upload. + Refer to `modelscope.hub.utils.aigc.AigcModel` for details. Returns: str: URL of the created model repository @@ -285,7 +286,7 @@ def create_model(self, raise_on_error(r.json()) model_repo_url = f'{endpoint}/models/{model_id}' - # TODO: due to server error, the upload function is not working + # TODO: To be updated after server side is adapted # Upload model files for AIGC models # if aigc_model is not None: # aigc_model.upload_to_repo(self, model_id, token) From 918f6a92f122d17922194eaefb4aa7e45207270b Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 18:36:11 +0800 Subject: [PATCH 07/17] update --- modelscope/cli/aigc.py | 121 +++++++++++++++++++++ modelscope/cli/cli.py | 2 + modelscope/hub/api.py | 30 ++--- modelscope/hub/utils/aigc.py | 163 +++++++++++++++++++--------- tests/hub/test_create_aigc_model.py | 78 +++++++++++++ tests/hub/test_create_repo.py | 58 ---------- 6 files changed, 319 insertions(+), 133 deletions(-) create mode 100644 modelscope/cli/aigc.py create mode 100644 tests/hub/test_create_aigc_model.py diff --git a/modelscope/cli/aigc.py b/modelscope/cli/aigc.py new file mode 100644 index 000000000..93cf304fa --- /dev/null +++ b/modelscope/cli/aigc.py @@ -0,0 +1,121 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import argparse + +from modelscope.cli.base import CLICommand +from modelscope.hub.api import HubApi +from modelscope.hub.utils.aigc import AigcModel + + +class CreateAigcModelCMD(CLICommand): + name = 'create-aigc-model' + + def __init__(self, args): + self.args = args + + @staticmethod + def define_args(parsers: argparse._SubParsersAction): + """Define the arguments for the command.""" + parser = parsers.add_parser( + CreateAigcModelCMD.name, + description= + 'Create an AIGC model on ModelScope Hub from parameters or a JSON file.' + ) + + # Group for loading from JSON + json_group = parser.add_argument_group( + 'From JSON', 'Create model from a JSON config file.') + json_group.add_argument( + '--from-json', + type=str, + help='Path to a JSON file containing AIGC model configuration. ' + 'If used, all other parameters except --model-id are ignored.') + + # Group for creating from direct parameters + param_group = parser.add_argument_group( + 'From Parameters', 'Create model by providing direct parameters.') + param_group.add_argument( + '--model-id', + type=str, + help='The model ID, e.g., your-namespace/your-model-name.') + param_group.add_argument( + '--model-path', type=str, help='Path to the model file or folder.') + param_group.add_argument( + '--aigc-type', + type=str, + help="AIGC type. Recommended: 'Checkpoint', 'LoRA', 'VAE'.") + param_group.add_argument( + '--base-model-type', + type=str, + help='Base model type, e.g., SD_XL.') + param_group.add_argument( + '--revision', + type=str, + default='v1.0', + help="Model revision. Defaults to 'v1.0'.") + param_group.add_argument( + '--description', + type=str, + default='This is an AIGC model.', + help='Model description.') + param_group.add_argument( + '--base-model-id', + type=str, + default='', + help='Base model ID from ModelScope.') + param_group.add_argument( + '--path-in-repo', + type=str, + default='', + help='Path in the repository to upload to.') + parser.set_defaults(func=CreateAigcModelCMD) + + def execute(self): + """Execute the command.""" + # Basic validation + if not self.args.from_json and not self.args.model_id: + print('Error: Either --from-json or --model-id must be provided.') + return + + api = HubApi() + api.login(self.args.token) + + if self.args.from_json: + # Create from JSON file + print('Creating AIGC model from JSON file: ' + f'{self.args.from_json}') + aigc_model = AigcModel.from_json_file(self.args.from_json) + # model_id must still be provided if not in json, or for override + model_id = self.args.model_id or aigc_model.model_id + if not model_id: + print("Error: --model-id is required when it's not present " + 'in the JSON file.') + return + else: + # Create from command line arguments + print('Creating AIGC model from command line arguments...') + model_id = self.args.model_id + if not all([ + self.args.model_path, self.args.aigc_type, + self.args.base_model_type + ]): + print('Error: --model-path, --aigc-type, and ' + '--base-model-type are required when not using ' + '--from-json.') + return + + aigc_model = AigcModel( + model_path=self.args.model_path, + aigc_type=self.args.aigc_type, + base_model_type=self.args.base_model_type, + revision=self.args.revision, + description=self.args.description, + base_model_id=self.args.base_model_id, + path_in_repo=self.args.path_in_repo, + ) + + try: + model_url = api.create_model( + model_id=model_id, aigc_model=aigc_model) + print(f'Successfully created AIGC model: {model_url}') + except Exception as e: + print(f'Error creating AIGC model: {e}') diff --git a/modelscope/cli/cli.py b/modelscope/cli/cli.py index e6fc474c3..4283ce4de 100644 --- a/modelscope/cli/cli.py +++ b/modelscope/cli/cli.py @@ -3,6 +3,7 @@ import argparse import logging +from modelscope.cli.aigc import CreateAigcModelCMD from modelscope.cli.clearcache import ClearCacheCMD from modelscope.cli.download import DownloadCMD from modelscope.cli.llamafile import LlamafileCMD @@ -37,6 +38,7 @@ def run_cmd(): LoginCMD.define_args(subparsers) LlamafileCMD.define_args(subparsers) ScanCacheCMD.define_args(subparsers) + CreateAigcModelCMD.define_args(subparsers) args = parser.parse_args() diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index ae168910a..fbca68830 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -203,7 +203,7 @@ def create_model(self, token (str, optional): access token for authentication aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation. If provided, will create an AIGC model with automatic file upload. - Refer to `modelscope.hub.utils.aigc.AigcModel` for details. + Refer to modelscope.hub.utils.aigc.AigcModel for details. Returns: str: URL of the created model repository @@ -243,33 +243,17 @@ def create_model(self, # Use AIGC model endpoint path = f'{endpoint}/api/v1/models/aigc' - # Validate AIGC parameters - valid_aigc_types = [item.value for item in AigcModel.AigcType] - valid_vision_foundations = [ - item.value for item in AigcModel.BaseModelType - ] - - if aigc_model.aigc_type.value not in valid_aigc_types: - raise InvalidParameter( - f'aigc_type must be one of {valid_aigc_types}, got: {aigc_model.aigc_type.value}' - ) - if aigc_model.base_model_type.value not in valid_vision_foundations: - raise InvalidParameter( - f'vision_foundation must be one of {valid_vision_foundations}, ' - f'got: {aigc_model.base_model_type.value}' - ) - # Add AIGC-specific fields to body body.update({ 'TagShowName': aigc_model.revision, 'CoverImages': aigc_model.cover_images, - 'AigcType': aigc_model.aigc_type.value, + 'AigcType': aigc_model.aigc_type, 'TagDescription': aigc_model.description, - 'VisionFoundation': aigc_model.base_model_type.value, + 'VisionFoundation': aigc_model.base_model_type, 'BaseModel': aigc_model.base_model_id, - 'WeightsName': aigc_model.weights_filename, - 'WeightsSha256': aigc_model.weights_sha256, - 'WeightsSize': aigc_model.weights_size, + 'WeightsName': aigc_model.weight_filename, + 'WeightsSha256': aigc_model.weight_sha256, + 'WeightsSize': aigc_model.weight_size, 'ModelPath': aigc_model.model_path }) @@ -286,7 +270,7 @@ def create_model(self, raise_on_error(r.json()) model_repo_url = f'{endpoint}/models/{model_id}' - # TODO: To be updated after server side is adapted + # TODO: due to server error, the upload function is not working # Upload model files for AIGC models # if aigc_model is not None: # aigc_model.upload_to_repo(self, model_id, token) diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index cf9174ca1..3857828a9 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -7,7 +7,6 @@ from modelscope.utils.logger import get_logger -# Get a logger instance from modelscope logger = get_logger() # Default AIGC model cover image @@ -19,56 +18,65 @@ class AigcModel: """ Helper class to encapsulate AIGC-specific model creation parameters. + + This class can be initialized directly with parameters, or loaded from a + JSON configuration file using the `from_json_file` classmethod. + + Example of direct initialization: + >>> aigc_model = AigcModel( + ... aigc_type='Checkpoint', + ... base_model_type='SD_XL', + ... model_path='/path/to/your/model.safetensors' + ... base_model_id='AI-ModelScope/FLUX.1-dev' + ... ) + + Example of loading from a JSON file: + `config.json`: + { + "model_path": "/path/to/your/model.safetensors", + "aigc_type": "Checkpoint", + "base_model_type": "SD_XL", + "base_model_id": "AI-ModelScope/FLUX.1-dev" + } + + >>> aigc_model = AigcModel.from_json_file('config.json') """ - class AigcType(str, Enum): - CHECKPOINT = 'Checkpoint' - LORA = 'LoRA' - VAE = 'VAE' - - class BaseModelType(str, Enum): - SD_1_5 = 'SD_1_5' - SD_XL = 'SD_XL' - SD_3 = 'SD_3' - FLUX_1 = 'FLUX_1' - WAN_VIDEO_2_1_T2V_1_3_B = 'WAN_VIDEO_2_1_T2V_1_3_B' - WAN_VIDEO_2_1_T2V_14_B = 'WAN_VIDEO_2_1_T2V_14_B' - WAN_VIDEO_2_1_I2V_14_B = 'WAN_VIDEO_2_1_I2V_14_B' - WAN_VIDEO_2_1_FLF2V_14_B = 'WAN_VIDEO_2_1_FLF2V_14_B' - WAN_VIDEO_2_2_T2V_5_B = 'WAN_VIDEO_2_2_T2V_5_B' - WAN_VIDEO_2_2_T2V_14_B = 'WAN_VIDEO_2_2_T2V_14_B' - WAN_VIDEO_2_2_I2V_14_B = 'WAN_VIDEO_2_2_I2V_14_B' + AIGC_TYPES = {'Checkpoint', 'LoRA', 'VAE'} + + # Supported base model types for reference + BASE_MODEL_TYPES = { + 'SD_1_5', 'SD_XL', 'SD_3', 'FLUX_1', 'WAN_VIDEO_2_1_T2V_1_3_B', + 'WAN_VIDEO_2_1_T2V_14_B', 'WAN_VIDEO_2_1_I2V_14_B', + 'WAN_VIDEO_2_1_FLF2V_14_B', 'WAN_VIDEO_2_2_T2V_5_B', + 'WAN_VIDEO_2_2_T2V_14_B', 'WAN_VIDEO_2_2_I2V_14_B', 'QWEN_IMAGE_20B' + } def __init__(self, - aigc_type: AigcType, - base_model_type: BaseModelType, + aigc_type: str, + base_model_type: str, model_path: str, + base_model_id: str = '', revision: Optional[str] = 'v1.0', description: Optional[str] = 'this is an aigc model', cover_images: Optional[List[str]] = None, - base_model_id: str = '', path_in_repo: Optional[str] = ''): """ Initializes the AigcModel helper. Args: - aigc_type (AigcType): AIGC model type. - Valid values: Checkpoint, LoRA, VAE - base_model_type (BaseModelType): Vision foundation model. - Valid values: SD_1_5, SD_XL, SD_3, FLUX_1, WAN_VIDEO_2_1_T2V_1_3_B... - model_path (str, required): The path of checkpoint/LoRA weights file (.safetensors) or folder - revision (str, optional): revision for AIGC model, default 'master' - description (str, optional): Model description, - default: 'this is a aigc model' - cover_images (List[str], optional): List of cover image URLs, - default: DEFAULT_AIGC_COVER_IMAGE - base_model_id (str, optional): Base model name, - default: '', e.g.'AI-ModelScope/FLUX.1-dev' - path_in_repo (str, optional): Path in repository + model_path (str): The path of checkpoint/LoRA weight file or folder. + aigc_type (str): AIGC model type. Recommended: 'Checkpoint', 'LoRA', 'VAE'. + base_model_type (str): Vision foundation model type. Recommended values are in BASE_MODEL_TYPES. + revision (str, optional): Revision for the AIGC model. Defaults to 'v1.0'. + description (str, optional): Model description. Defaults to 'this is an aigc model'. + cover_images (List[str], optional): List of cover image URLs. + base_model_id (str, optional): Base model name. e.g., 'AI-ModelScope/FLUX.1-dev'. + path_in_repo (str, optional): Path in the repository. """ + self.model_path = model_path self.aigc_type = aigc_type self.base_model_type = base_model_type - self.model_path = model_path self.revision = revision self.description = description self.cover_images = cover_images if cover_images is not None else [ @@ -77,11 +85,32 @@ def __init__(self, self.base_model_id = base_model_id self.path_in_repo = path_in_repo + # Validate types and provide warnings + self._validate_aigc_type() + self._validate_base_model_type() + # Process model path and calculate weights information self._process_model_path() + def _validate_aigc_type(self): + """Validate aigc_type and provide a warning for unsupported types.""" + if self.aigc_type not in self.AIGC_TYPES: + supported_types = ', '.join(sorted(self.AIGC_TYPES)) + logger.warning(f'Unsupported aigc_type: "{self.aigc_type}". ' + f'Recommended values: {supported_types}. ' + 'Custom values are allowed but may cause issues.') + + def _validate_base_model_type(self): + """Validate base_model_type and provide warning for unsupported types.""" + if self.base_model_type not in self.BASE_MODEL_TYPES: + supported_types = ', '.join(sorted(self.BASE_MODEL_TYPES)) + logger.warning( + f'Your base_model_type: "{self.base_model_type}" may not be supported. ' + f'Recommended values: {supported_types}. ' + f'Custom values are allowed but may cause issues. ') + def _process_model_path(self): - """Process model_path to extract weights information""" + """Process model_path to extract weight information""" from modelscope.utils.file_utils import get_file_hash # Expand user path @@ -104,7 +133,7 @@ def _process_model_path(self): files = glob.glob(os.path.join(self.model_path, f'*{ext}')) if files: target_file = files[0] - logger.info(f'✅ Found {ext} file: %s', + logger.info(f'Found {ext} file: %s', os.path.basename(target_file)) if len(files) > 1: logger.warning( @@ -142,10 +171,10 @@ def _process_model_path(self): logger.info('Computing hash and size for %s...', target_file) hash_info = get_file_hash(target_file) - # Store weights information - self.weights_filename = os.path.basename(target_file) - self.weights_sha256 = hash_info['file_hash'] - self.weights_size = hash_info['file_size'] + # Store weight information + self.weight_filename = os.path.basename(target_file) + self.weight_sha256 = hash_info['file_hash'] + self.weight_size = hash_info['file_size'] self.target_file = target_file def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): @@ -168,18 +197,17 @@ def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): api.upload_file( revision=self.revision, path_or_fileobj=self.target_file, - path_in_repo=self.path_in_repo + '/' - + self.weights_filename - if self.path_in_repo else self.weights_filename, + path_in_repo=self.path_in_repo + '/' + self.weight_filename + if self.path_in_repo else self.weight_filename, repo_id=model_id, token=token, - commit_message=f'Upload {self.weights_filename} ' + commit_message=f'Upload {self.weight_filename} ' 'for AIGC model') - logger.info('✅ Successfully uploaded model to %s', model_id) + logger.info('Successfully uploaded model to %s', model_id) return True except Exception as e: - logger.warning('⚠️ Warning: Failed to upload model: %s', e) + logger.warning('Warning: Failed to upload model: %s', e) logger.warning( 'You may need to upload the model manually after creation.') return False @@ -187,14 +215,45 @@ def upload_to_repo(self, api, model_id: str, token: Optional[str] = None): def to_dict(self) -> dict: """Converts the AIGC parameters to a dictionary suitable for API calls.""" return { - 'aigc_type': self.aigc_type.value, - 'base_model_type': self.base_model_type.value, + 'aigc_type': self.aigc_type, + 'base_model_type': self.base_model_type, 'revision': self.revision, 'description': self.description, 'cover_images': self.cover_images, 'base_model_id': self.base_model_id, 'model_path': self.model_path, - 'weights_filename': self.weights_filename, - 'weights_sha256': self.weights_sha256, - 'weights_size': self.weights_size + 'weight_filename': self.weight_filename, + 'weight_sha256': self.weight_sha256, + 'weight_size': self.weight_size } + + @classmethod + def from_json_file(cls, json_path: str): + """ + Creates an AigcModel instance from a JSON configuration file. + + Args: + json_path (str): The path to the JSON configuration file. + + Returns: + AigcModel: An instance of the AigcModel. + """ + import json + json_path = os.path.expanduser(json_path) + if not os.path.exists(json_path): + raise FileNotFoundError( + f'JSON config file not found at: {json_path}') + + with open(json_path, 'r', encoding='utf-8') as f: + config = json.load(f) + + # Ensure required fields are present + required_fields = [ + 'model_path', 'aigc_type', 'base_model_type', 'base_model_id' + ] + for field in required_fields: + if field not in config: + raise ValueError( + f"Missing required field in JSON config: '{field}'") + + return cls(**config) diff --git a/tests/hub/test_create_aigc_model.py b/tests/hub/test_create_aigc_model.py new file mode 100644 index 000000000..43065952f --- /dev/null +++ b/tests/hub/test_create_aigc_model.py @@ -0,0 +1,78 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import tempfile +import unittest +import uuid + +from requests.exceptions import HTTPError + +from modelscope import HubApi +from modelscope.hub.utils.aigc import AigcModel +from modelscope.utils.logger import get_logger +from modelscope.utils.test_utils import (TEST_ACCESS_TOKEN1, TEST_MODEL_ORG, + delete_credential, test_level) + +logger = get_logger() + + +class TestCreateAigcModel(unittest.TestCase): + + def setUp(self): + self.api = HubApi() + self.repo_id: str = f'{TEST_MODEL_ORG}/test_create_aigc_model_{uuid.uuid4().hex[-6:]}' + + # Create a dummy file for AIGC model test + self.tmp_file = tempfile.NamedTemporaryFile( + suffix='.safetensors', delete=False) + self.tmp_file.write(b'This is a dummy weights file for testing.') + self.tmp_file.close() + self.tmp_file_path = self.tmp_file.name + + def tearDown(self): + # Login before cleaning up, ensuring token is valid for deletion. + try: + self.api.login(TEST_ACCESS_TOKEN1) + self.api.delete_model(model_id=self.repo_id) + except HTTPError: + pass # It's ok if the repo doesn't exist (e.g., creation failed) + os.remove(self.tmp_file_path) + delete_credential() + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_create_aigc_model_expects_sha256_error(self): + """Test creating an AIGC model repository. + + This test is expected to fail with a 'sha256 not exits' error from the server. + This is the correct behavior when the server does not know the file yet. + This test verifies that the SDK is correctly forming and sending the request. + """ + logger.info(f'TEST: Attempting to create AIGC repo {self.repo_id} ...') + + # Login just before making the authenticated call. + self.api.login(TEST_ACCESS_TOKEN1) + + # 1. Create AigcModel instance from a local file + aigc_model = AigcModel( + model_path=self.tmp_file_path, + aigc_type='Checkpoint', + base_model_type='SD_XL', + ) + + # 2. Attempt to create the model repository. + # We expect an HTTPError because the server API requires the file's sha256 + # to be known before creating the repo. + with self.assertRaises(HTTPError) as cm: + self.api.create_model( + model_id=self.repo_id, + aigc_model=aigc_model, + ) + + # Check if the error message is the one we expect. + # The actual error might be 'namespace is not valid' if run outside CI + # or 'sha256 not exits' if namespace is valid. Both are acceptable failures + # proving the SDK sent the request correctly. + error_str = str(cm.exception) + is_expected_error = 'sha256 not exits' in error_str or 'namespace' in error_str and 'is not valid' in error_str + self.assertTrue(is_expected_error, + f'Unexpected error message: {error_str}') + logger.info(f'TEST: Received expected error: {error_str}') diff --git a/tests/hub/test_create_repo.py b/tests/hub/test_create_repo.py index b56580750..e69de29bb 100644 --- a/tests/hub/test_create_repo.py +++ b/tests/hub/test_create_repo.py @@ -1,58 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import unittest -import uuid - -from modelscope import HubApi -from modelscope.utils.constant import REPO_TYPE_DATASET, REPO_TYPE_MODEL -from modelscope.utils.logger import get_logger -from modelscope.utils.test_utils import TEST_ACCESS_TOKEN1 -from modelscope.utils.test_utils import TEST_MODEL_ORG as TEST_ORG -from modelscope.utils.test_utils import delete_credential, test_level - -logger = get_logger() - - -class TestCreateRepo(unittest.TestCase): - - def setUp(self): - self.api = HubApi() - self.api.login(TEST_ACCESS_TOKEN1) - - self.repo_id_model: str = f'{TEST_ORG}/test_create_repo_model_{uuid.uuid4().hex[-6:]}' - self.repo_id_dataset: str = f'{TEST_ORG}/test_create_repo_dataset_{uuid.uuid4().hex[-6:]}' - - def tearDown(self): - self.api.delete_repo( - repo_id=self.repo_id_model, repo_type=REPO_TYPE_MODEL) - self.api.delete_repo( - repo_id=self.repo_id_dataset, repo_type=REPO_TYPE_DATASET) - delete_credential() - - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_create_repo(self): - - logger.info( - f'TEST: Creating repo {self.repo_id_model} and {self.repo_id_dataset} ...' - ) - - try: - self.api.create_repo( - repo_id=self.repo_id_model, - repo_type=REPO_TYPE_MODEL, - exist_ok=True) - except Exception as e: - logger.error(f'Failed to create repo {self.repo_id_model} !') - raise e - - try: - self.api.create_repo( - repo_id=self.repo_id_dataset, - repo_type=REPO_TYPE_DATASET, - exist_ok=True) - except Exception as e: - logger.error(f'Failed to create repo {self.repo_id_dataset} !') - raise e - - logger.info( - f'TEST: Created repo {self.repo_id_model} and {self.repo_id_dataset} successfully !' - ) From 51e5eb86ad0a986bf535939a9242773da2313cfa Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Tue, 5 Aug 2025 18:38:07 +0800 Subject: [PATCH 08/17] Add create repo cli --- modelscope/cli/cli.py | 2 + modelscope/cli/create.py | 100 ++++++++++++++++++++++++++++++++++++ modelscope/hub/constants.py | 7 +++ 3 files changed, 109 insertions(+) create mode 100644 modelscope/cli/create.py diff --git a/modelscope/cli/cli.py b/modelscope/cli/cli.py index e6fc474c3..139f14a56 100644 --- a/modelscope/cli/cli.py +++ b/modelscope/cli/cli.py @@ -4,6 +4,7 @@ import logging from modelscope.cli.clearcache import ClearCacheCMD +from modelscope.cli.create import CreateCMD from modelscope.cli.download import DownloadCMD from modelscope.cli.llamafile import LlamafileCMD from modelscope.cli.login import LoginCMD @@ -27,6 +28,7 @@ def run_cmd(): '--token', default=None, help='Specify ModelScope SDK token.') subparsers = parser.add_subparsers(help='modelscope commands helpers') + CreateCMD.define_args(subparsers) DownloadCMD.define_args(subparsers) UploadCMD.define_args(subparsers) ClearCacheCMD.define_args(subparsers) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py new file mode 100644 index 000000000..02ab1229e --- /dev/null +++ b/modelscope/cli/create.py @@ -0,0 +1,100 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from argparse import ArgumentParser, _SubParsersAction + +from modelscope.cli.base import CLICommand +from modelscope.hub.api import HubApi +from modelscope.hub.constants import Licenses, Visibility +from modelscope.utils.constant import REPO_TYPE_MODEL, REPO_TYPE_SUPPORT + + +def subparser_func(args): + """ Function which will be called for a specific sub parser. + """ + return CreateCMD(args) + + +class CreateCMD(CLICommand): + """ + Command for creating a new repository, supporting both model and dataset. + """ + + name = 'create' + + def __init__(self, args: _SubParsersAction): + self.args = args + + @staticmethod + def define_args(parsers: _SubParsersAction): + + parser: ArgumentParser = parsers.add_parser(CreateCMD.name) + + parser.add_argument( + 'repo_id', + type=str, + help='The ID of the repo to create (e.g. `username/repo-name`)') + parser.add_argument( + '--token', + type=str, + default=None, + help= + 'A User Access Token generated from https://modelscope.cn/my/myaccesstoken to authenticate the user. ' + 'If not provided, the CLI will use the local credentials if available.' + ) + parser.add_argument( + '--repo_type', + choices=REPO_TYPE_SUPPORT, + default=REPO_TYPE_MODEL, + help= + 'Type of the repo to create (e.g. `dataset`, `model`). Default to `model`.', + ) + parser.add_argument( + '--visibility', + choices=[ + Visibility.PUBLIC, Visibility.INTERNAL, Visibility.PRIVATE + ], + default=Visibility.PUBLIC, + help='Visibility of the repo to create. Default to `public`.', + ) + parser.add_argument( + '--chinese_name', + type=str, + default=None, + help='Optional, Chinese name of the repo. Default to `None`.', + ) + parser.add_argument( + '--license', + type=str, + choices=Licenses.to_list(), + default=Licenses.APACHE_V2, + help= + 'Optional, License of the repo. Default to `Apache License 2.0`.', + ) + parser.add_argument( + '--endpoint', + type=str, + default=None, + help='Optional, The modelscope server address. Default to None.', + ) + + parser.set_defaults(func=subparser_func) + + def execute(self): + + # Check token and login + # The cookies will be reused if the user has logged in before. + api = HubApi(endpoint=self.args.endpoint) + + # Create repo + api.create_repo( + repo_id=self.args.repo_id, + token=self.args.token, + visibility=self.args.visibility, + repo_type=self.args.repo_type, + chinese_name=self.args.chinese_name, + license=self.args.license, + exist_ok=True, + create_default_config=True, + endpoint=self.args.endpoint, + ) + + print(f'Successfully created the repo: {self.args.repo_id}.') diff --git a/modelscope/hub/constants.py b/modelscope/hub/constants.py index 5210b1ed7..bcc9afdf1 100644 --- a/modelscope/hub/constants.py +++ b/modelscope/hub/constants.py @@ -77,6 +77,13 @@ class Licenses(object): ECL_V2 = 'ECL-2.0' MIT = 'MIT' + @classmethod + def to_list(cls): + return [ + cls.__dict__[key] for key in cls.__dict__.keys() + if not key.startswith('_') and isinstance(cls.__dict__[key], str) + ] + class ModelVisibility(object): PRIVATE = 1 From e4b1d658aae5c94a823d520898b0b0dc8f6a3b19 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 18:43:02 +0800 Subject: [PATCH 09/17] update --- tests/hub/test_create_repo.py | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/hub/test_create_repo.py b/tests/hub/test_create_repo.py index e69de29bb..c66e94b71 100644 --- a/tests/hub/test_create_repo.py +++ b/tests/hub/test_create_repo.py @@ -0,0 +1,58 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest +import uuid + +from modelscope import HubApi +from modelscope.utils.constant import REPO_TYPE_DATASET, REPO_TYPE_MODEL +from modelscope.utils.logger import get_logger +from modelscope.utils.test_utils import TEST_ACCESS_TOKEN1 +from modelscope.utils.test_utils import TEST_MODEL_ORG as TEST_ORG +from modelscope.utils.test_utils import delete_credential, test_level + +logger = get_logger() + + +class TestCreateRepo(unittest.TestCase): + + def setUp(self): + self.api = HubApi() + self.api.login(TEST_ACCESS_TOKEN1) + + self.repo_id_model: str = f'{TEST_ORG}/test_create_repo_model_{uuid.uuid4().hex[-6:]}' + self.repo_id_dataset: str = f'{TEST_ORG}/test_create_repo_dataset_{uuid.uuid4().hex[-6:]}' + + def tearDown(self): + self.api.delete_repo( + repo_id=self.repo_id_model, repo_type=REPO_TYPE_MODEL) + self.api.delete_repo( + repo_id=self.repo_id_dataset, repo_type=REPO_TYPE_DATASET) + delete_credential() + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_create_repo(self): + + logger.info( + f'TEST: Creating repo {self.repo_id_model} and {self.repo_id_dataset} ...' + ) + + try: + self.api.create_repo( + repo_id=self.repo_id_model, + repo_type=REPO_TYPE_MODEL, + exist_ok=True) + except Exception as e: + logger.error(f'Failed to create repo {self.repo_id_model} !') + raise e + + try: + self.api.create_repo( + repo_id=self.repo_id_dataset, + repo_type=REPO_TYPE_DATASET, + exist_ok=True) + except Exception as e: + logger.error(f'Failed to create repo {self.repo_id_dataset} !') + raise e + + logger.info( + f'TEST: Created repo {self.repo_id_model} and {self.repo_id_dataset} successfully !' + ) \ No newline at end of file From ad022004b6c8785db59ed2ef7e164294fb0f44c5 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 18:49:01 +0800 Subject: [PATCH 10/17] update --- modelscope/hub/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index fbca68830..27e1fe90a 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -215,6 +215,8 @@ def create_model(self, Note: model_id = {owner}/{name} """ + if model_id is None: + raise InvalidParameter('model_id is required!') # Get cookies for authentication. if token: cookies = self.get_cookies(access_token=token) From ec8b27cb7195b649bf486bb86dc0d0c02130d981 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 19:41:31 +0800 Subject: [PATCH 11/17] update from json --- modelscope/cli/aigc.py | 121 ---------------------------------- modelscope/cli/cli.py | 2 - modelscope/cli/create.py | 105 +++++++++++++++++++++++++++++ modelscope/hub/api.py | 2 + tests/hub/test_create_repo.py | 2 +- 5 files changed, 108 insertions(+), 124 deletions(-) delete mode 100644 modelscope/cli/aigc.py diff --git a/modelscope/cli/aigc.py b/modelscope/cli/aigc.py deleted file mode 100644 index 93cf304fa..000000000 --- a/modelscope/cli/aigc.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -import argparse - -from modelscope.cli.base import CLICommand -from modelscope.hub.api import HubApi -from modelscope.hub.utils.aigc import AigcModel - - -class CreateAigcModelCMD(CLICommand): - name = 'create-aigc-model' - - def __init__(self, args): - self.args = args - - @staticmethod - def define_args(parsers: argparse._SubParsersAction): - """Define the arguments for the command.""" - parser = parsers.add_parser( - CreateAigcModelCMD.name, - description= - 'Create an AIGC model on ModelScope Hub from parameters or a JSON file.' - ) - - # Group for loading from JSON - json_group = parser.add_argument_group( - 'From JSON', 'Create model from a JSON config file.') - json_group.add_argument( - '--from-json', - type=str, - help='Path to a JSON file containing AIGC model configuration. ' - 'If used, all other parameters except --model-id are ignored.') - - # Group for creating from direct parameters - param_group = parser.add_argument_group( - 'From Parameters', 'Create model by providing direct parameters.') - param_group.add_argument( - '--model-id', - type=str, - help='The model ID, e.g., your-namespace/your-model-name.') - param_group.add_argument( - '--model-path', type=str, help='Path to the model file or folder.') - param_group.add_argument( - '--aigc-type', - type=str, - help="AIGC type. Recommended: 'Checkpoint', 'LoRA', 'VAE'.") - param_group.add_argument( - '--base-model-type', - type=str, - help='Base model type, e.g., SD_XL.') - param_group.add_argument( - '--revision', - type=str, - default='v1.0', - help="Model revision. Defaults to 'v1.0'.") - param_group.add_argument( - '--description', - type=str, - default='This is an AIGC model.', - help='Model description.') - param_group.add_argument( - '--base-model-id', - type=str, - default='', - help='Base model ID from ModelScope.') - param_group.add_argument( - '--path-in-repo', - type=str, - default='', - help='Path in the repository to upload to.') - parser.set_defaults(func=CreateAigcModelCMD) - - def execute(self): - """Execute the command.""" - # Basic validation - if not self.args.from_json and not self.args.model_id: - print('Error: Either --from-json or --model-id must be provided.') - return - - api = HubApi() - api.login(self.args.token) - - if self.args.from_json: - # Create from JSON file - print('Creating AIGC model from JSON file: ' - f'{self.args.from_json}') - aigc_model = AigcModel.from_json_file(self.args.from_json) - # model_id must still be provided if not in json, or for override - model_id = self.args.model_id or aigc_model.model_id - if not model_id: - print("Error: --model-id is required when it's not present " - 'in the JSON file.') - return - else: - # Create from command line arguments - print('Creating AIGC model from command line arguments...') - model_id = self.args.model_id - if not all([ - self.args.model_path, self.args.aigc_type, - self.args.base_model_type - ]): - print('Error: --model-path, --aigc-type, and ' - '--base-model-type are required when not using ' - '--from-json.') - return - - aigc_model = AigcModel( - model_path=self.args.model_path, - aigc_type=self.args.aigc_type, - base_model_type=self.args.base_model_type, - revision=self.args.revision, - description=self.args.description, - base_model_id=self.args.base_model_id, - path_in_repo=self.args.path_in_repo, - ) - - try: - model_url = api.create_model( - model_id=model_id, aigc_model=aigc_model) - print(f'Successfully created AIGC model: {model_url}') - except Exception as e: - print(f'Error creating AIGC model: {e}') diff --git a/modelscope/cli/cli.py b/modelscope/cli/cli.py index f4156458f..139f14a56 100644 --- a/modelscope/cli/cli.py +++ b/modelscope/cli/cli.py @@ -3,7 +3,6 @@ import argparse import logging -from modelscope.cli.aigc import CreateAigcModelCMD from modelscope.cli.clearcache import ClearCacheCMD from modelscope.cli.create import CreateCMD from modelscope.cli.download import DownloadCMD @@ -40,7 +39,6 @@ def run_cmd(): LoginCMD.define_args(subparsers) LlamafileCMD.define_args(subparsers) ScanCacheCMD.define_args(subparsers) - CreateAigcModelCMD.define_args(subparsers) args = parser.parse_args() diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index 02ab1229e..17bcf893b 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -4,6 +4,7 @@ from modelscope.cli.base import CLICommand from modelscope.hub.api import HubApi from modelscope.hub.constants import Licenses, Visibility +from modelscope.hub.utils.aigc import AigcModel from modelscope.utils.constant import REPO_TYPE_MODEL, REPO_TYPE_SUPPORT @@ -76,10 +77,63 @@ def define_args(parsers: _SubParsersAction): help='Optional, The modelscope server address. Default to None.', ) + # AIGC specific arguments + aigc_group = parser.add_argument_group( + 'AIGC Model Creation', + 'Arguments for creating an AIGC model. Use --is-aigc to enable.') + aigc_group.add_argument( + '--is-aigc', + action='store_true', + help='Flag to indicate that an AIGC model should be created.') + aigc_group.add_argument( + '--from-json', + type=str, + help='Path to a JSON file containing AIGC model configuration. ' + 'If used, all other parameters except --model-id are ignored.') + aigc_group.add_argument( + '--model-path', type=str, help='Path to the model file or folder.') + aigc_group.add_argument( + '--aigc-type', + type=str, + help="AIGC type. Recommended: 'Checkpoint', 'LoRA', 'VAE'.") + aigc_group.add_argument( + '--base-model-type', + type=str, + help='Base model type, e.g., SD_XL.') + aigc_group.add_argument( + '--revision', + type=str, + default='v1.0', + help="Model revision. Defaults to 'v1.0'.") + aigc_group.add_argument( + '--description', + type=str, + default='This is an AIGC model.', + help='Model description.') + aigc_group.add_argument( + '--base-model-id', + type=str, + default='', + help='Base model ID from ModelScope.') + aigc_group.add_argument( + '--path-in-repo', + type=str, + default='', + help='Path in the repository to upload to.') + parser.set_defaults(func=subparser_func) def execute(self): + if self.args.is_aigc: + if self.args.repo_type != REPO_TYPE_MODEL: + raise ValueError( + 'AIGC models can only be created when repo_type is "model".' + ) + self._create_aigc_model() + else: + self._create_regular_repo() + def _create_regular_repo(self): # Check token and login # The cookies will be reused if the user has logged in before. api = HubApi(endpoint=self.args.endpoint) @@ -98,3 +152,54 @@ def execute(self): ) print(f'Successfully created the repo: {self.args.repo_id}.') + + def _create_aigc_model(self): + """Execute the command.""" + # Basic validation + if not self.args.from_json and not self.args.repo_id: + print('Error: Either --from-json or --repo_id must be provided.') + return + + api = HubApi() + api.login(self.args.token) + + if self.args.from_json: + # Create from JSON file + print('Creating AIGC model from JSON file: ' + f'{self.args.from_json}') + aigc_model = AigcModel.from_json_file(self.args.from_json) + # model_id must still be provided if not in json, or for override + model_id = self.args.repo_id or aigc_model.model_id + if not model_id: + print("Error: --repo_id is required when it's not present " + 'in the JSON file.') + return + else: + # Create from command line arguments + print('Creating AIGC model from command line arguments...') + model_id = self.args.repo_id + if not all([ + self.args.model_path, self.args.aigc_type, + self.args.base_model_type + ]): + print('Error: --model-path, --aigc-type, and ' + '--base-model-type are required when not using ' + '--from-json.') + return + + aigc_model = AigcModel( + model_path=self.args.model_path, + aigc_type=self.args.aigc_type, + base_model_type=self.args.base_model_type, + revision=self.args.revision, + description=self.args.description, + base_model_id=self.args.base_model_id, + path_in_repo=self.args.path_in_repo, + ) + + try: + model_url = api.create_model( + model_id=model_id, aigc_model=aigc_model) + print(f'Successfully created AIGC model: {model_url}') + except Exception as e: + print(f'Error creating AIGC model: {e}') diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index c33ad306c..7c3199aea 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -1504,6 +1504,7 @@ def create_repo( endpoint: Optional[str] = None, exist_ok: Optional[bool] = False, create_default_config: Optional[bool] = True, + aigc_model: Optional[AigcModel] = None, **kwargs, ) -> str: """ @@ -1557,6 +1558,7 @@ def create_repo( visibility=visibility, license=license, chinese_name=chinese_name, + aigc_model=aigc_model ) if create_default_config: with tempfile.TemporaryDirectory() as temp_cache_dir: diff --git a/tests/hub/test_create_repo.py b/tests/hub/test_create_repo.py index c66e94b71..b56580750 100644 --- a/tests/hub/test_create_repo.py +++ b/tests/hub/test_create_repo.py @@ -55,4 +55,4 @@ def test_create_repo(self): logger.info( f'TEST: Created repo {self.repo_id_model} and {self.repo_id_dataset} successfully !' - ) \ No newline at end of file + ) From bae316c34c21bbe9bba9ae546d9fa677085c07f8 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 19:52:27 +0800 Subject: [PATCH 12/17] update from json --- modelscope/cli/create.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index 17bcf893b..574bbc9fd 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -82,22 +82,22 @@ def define_args(parsers: _SubParsersAction): 'AIGC Model Creation', 'Arguments for creating an AIGC model. Use --is-aigc to enable.') aigc_group.add_argument( - '--is-aigc', + '--is_aigc', action='store_true', help='Flag to indicate that an AIGC model should be created.') aigc_group.add_argument( - '--from-json', + '--from_json', type=str, help='Path to a JSON file containing AIGC model configuration. ' 'If used, all other parameters except --model-id are ignored.') aigc_group.add_argument( - '--model-path', type=str, help='Path to the model file or folder.') + '--model_path', type=str, help='Path to the model file or folder.') aigc_group.add_argument( - '--aigc-type', + '--aigc_type', type=str, help="AIGC type. Recommended: 'Checkpoint', 'LoRA', 'VAE'.") aigc_group.add_argument( - '--base-model-type', + '--base_model_type', type=str, help='Base model type, e.g., SD_XL.') aigc_group.add_argument( @@ -111,12 +111,12 @@ def define_args(parsers: _SubParsersAction): default='This is an AIGC model.', help='Model description.') aigc_group.add_argument( - '--base-model-id', + '--base_model_id', type=str, default='', help='Base model ID from ModelScope.') aigc_group.add_argument( - '--path-in-repo', + '--path_in_repo', type=str, default='', help='Path in the repository to upload to.') @@ -182,9 +182,9 @@ def _create_aigc_model(self): self.args.model_path, self.args.aigc_type, self.args.base_model_type ]): - print('Error: --model-path, --aigc-type, and ' - '--base-model-type are required when not using ' - '--from-json.') + print('Error: --model_path, --aigc_type, and ' + '--base_model_type are required when not using ' + '--from_json.') return aigc_model = AigcModel( From a67e8645831d23505a1628b19268a12cb38a562d Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 20:57:10 +0800 Subject: [PATCH 13/17] fix from json --- modelscope/cli/create.py | 54 ++++++++++++++++++++++------------------ modelscope/hub/api.py | 2 +- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index 574bbc9fd..5fd559e07 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -6,6 +6,9 @@ from modelscope.hub.constants import Licenses, Visibility from modelscope.hub.utils.aigc import AigcModel from modelscope.utils.constant import REPO_TYPE_MODEL, REPO_TYPE_SUPPORT +from modelscope.utils.logger import get_logger + +logger = get_logger() def subparser_func(args): @@ -80,7 +83,7 @@ def define_args(parsers: _SubParsersAction): # AIGC specific arguments aigc_group = parser.add_argument_group( 'AIGC Model Creation', - 'Arguments for creating an AIGC model. Use --is-aigc to enable.') + 'Arguments for creating an AIGC model. Use --is_aigc to enable.') aigc_group.add_argument( '--is_aigc', action='store_true', @@ -89,7 +92,7 @@ def define_args(parsers: _SubParsersAction): '--from_json', type=str, help='Path to a JSON file containing AIGC model configuration. ' - 'If used, all other parameters except --model-id are ignored.') + 'If used, all other parameters except --repo_id are ignored.') aigc_group.add_argument( '--model_path', type=str, help='Path to the model file or folder.') aigc_group.add_argument( @@ -155,37 +158,25 @@ def _create_regular_repo(self): def _create_aigc_model(self): """Execute the command.""" - # Basic validation - if not self.args.from_json and not self.args.repo_id: - print('Error: Either --from-json or --repo_id must be provided.') - return - - api = HubApi() - api.login(self.args.token) + api = HubApi(endpoint=self.args.endpoint) + model_id = self.args.repo_id if self.args.from_json: # Create from JSON file - print('Creating AIGC model from JSON file: ' - f'{self.args.from_json}') + logger.info('Creating AIGC model from JSON file: ' + f'{self.args.from_json}') aigc_model = AigcModel.from_json_file(self.args.from_json) - # model_id must still be provided if not in json, or for override - model_id = self.args.repo_id or aigc_model.model_id - if not model_id: - print("Error: --repo_id is required when it's not present " - 'in the JSON file.') - return else: # Create from command line arguments - print('Creating AIGC model from command line arguments...') - model_id = self.args.repo_id + logger.info('Creating AIGC model from command line arguments...') if not all([ self.args.model_path, self.args.aigc_type, self.args.base_model_type ]): - print('Error: --model_path, --aigc_type, and ' - '--base_model_type are required when not using ' - '--from_json.') - return + raise ValueError( + 'Error: --model_path, --aigc_type, and ' + '--base_model_type are required when not using ' + '--from_json.') aigc_model = AigcModel( model_path=self.args.model_path, @@ -197,9 +188,24 @@ def _create_aigc_model(self): path_in_repo=self.args.path_in_repo, ) + # Convert visibility string to int for the API call + visibility_int = self.args.visibility + if isinstance(visibility_int, str): + visibility_map = { + 'private': 1, + 'internal': 3, + 'public': 5, + } + visibility_int = visibility_map.get(visibility_int, 5) + try: model_url = api.create_model( - model_id=model_id, aigc_model=aigc_model) + model_id=model_id, + token=self.args.token, + visibility=visibility_int, + license=self.args.license, + chinese_name=self.args.chinese_name, + aigc_model=aigc_model) print(f'Successfully created AIGC model: {model_url}') except Exception as e: print(f'Error creating AIGC model: {e}') diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 7c3199aea..203fa8d13 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -252,7 +252,7 @@ def create_model(self, 'AigcType': aigc_model.aigc_type, 'TagDescription': aigc_model.description, 'VisionFoundation': aigc_model.base_model_type, - 'BaseModel': aigc_model.base_model_id, + 'BaseModel': aigc_model.base_model_id or original_model_id, 'WeightsName': aigc_model.weight_filename, 'WeightsSha256': aigc_model.weight_sha256, 'WeightsSize': aigc_model.weight_size, From dd28f47bfc2ca17158b02943fd0d396791ce2921 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Tue, 5 Aug 2025 21:21:59 +0800 Subject: [PATCH 14/17] fix from json --- modelscope/cli/create.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index 5fd559e07..43271c66c 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -3,7 +3,7 @@ from modelscope.cli.base import CLICommand from modelscope.hub.api import HubApi -from modelscope.hub.constants import Licenses, Visibility +from modelscope.hub.constants import Licenses, ModelVisibility, Visibility from modelscope.hub.utils.aigc import AigcModel from modelscope.utils.constant import REPO_TYPE_MODEL, REPO_TYPE_SUPPORT from modelscope.utils.logger import get_logger @@ -192,11 +192,12 @@ def _create_aigc_model(self): visibility_int = self.args.visibility if isinstance(visibility_int, str): visibility_map = { - 'private': 1, - 'internal': 3, - 'public': 5, + 'private': ModelVisibility.PRIVATE, + 'internal': ModelVisibility.INTERNAL, + 'public': ModelVisibility.PUBLIC, } - visibility_int = visibility_map.get(visibility_int, 5) + visibility_int = visibility_map.get(visibility_int, + ModelVisibility.PUBLIC) try: model_url = api.create_model( From 0f8ea70b1625dd4560a83ffab569445d51877734 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Wed, 6 Aug 2025 13:41:33 +0800 Subject: [PATCH 15/17] fix from json --- modelscope/cli/create.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index 43271c66c..461008b33 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -3,7 +3,7 @@ from modelscope.cli.base import CLICommand from modelscope.hub.api import HubApi -from modelscope.hub.constants import Licenses, ModelVisibility, Visibility +from modelscope.hub.constants import Licenses, Visibility, VisibilityMap from modelscope.hub.utils.aigc import AigcModel from modelscope.utils.constant import REPO_TYPE_MODEL, REPO_TYPE_SUPPORT from modelscope.utils.logger import get_logger @@ -191,13 +191,11 @@ def _create_aigc_model(self): # Convert visibility string to int for the API call visibility_int = self.args.visibility if isinstance(visibility_int, str): - visibility_map = { - 'private': ModelVisibility.PRIVATE, - 'internal': ModelVisibility.INTERNAL, - 'public': ModelVisibility.PUBLIC, - } - visibility_int = visibility_map.get(visibility_int, - ModelVisibility.PUBLIC) + # Create reverse mapping from existing VisibilityMap + reverse_visibility_map = {v: k for k, v in VisibilityMap.items()} + visibility_int = reverse_visibility_map.get( + visibility_int, + list(VisibilityMap.keys())[2]) # Default to PUBLIC try: model_url = api.create_model( From e7da9085f3436c64259ee314d20a99c12b8166cc Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Wed, 6 Aug 2025 16:50:01 +0800 Subject: [PATCH 16/17] update create.py for aigc model --- modelscope/cli/create.py | 31 ++++++++++++++----------------- modelscope/hub/utils/aigc.py | 3 --- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index 461008b33..42906af6f 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -3,7 +3,8 @@ from modelscope.cli.base import CLICommand from modelscope.hub.api import HubApi -from modelscope.hub.constants import Licenses, Visibility, VisibilityMap +from modelscope.hub.constants import (Licenses, ModelVisibility, Visibility, + VisibilityMap) from modelscope.hub.utils.aigc import AigcModel from modelscope.utils.constant import REPO_TYPE_MODEL, REPO_TYPE_SUPPORT from modelscope.utils.logger import get_logger @@ -83,9 +84,9 @@ def define_args(parsers: _SubParsersAction): # AIGC specific arguments aigc_group = parser.add_argument_group( 'AIGC Model Creation', - 'Arguments for creating an AIGC model. Use --is_aigc to enable.') + 'Arguments for creating an AIGC model. Use --aigc to enable.') aigc_group.add_argument( - '--is_aigc', + '--aigc', action='store_true', help='Flag to indicate that an AIGC model should be created.') aigc_group.add_argument( @@ -108,16 +109,16 @@ def define_args(parsers: _SubParsersAction): type=str, default='v1.0', help="Model revision. Defaults to 'v1.0'.") - aigc_group.add_argument( - '--description', - type=str, - default='This is an AIGC model.', - help='Model description.') aigc_group.add_argument( '--base_model_id', type=str, default='', help='Base model ID from ModelScope.') + aigc_group.add_argument( + '--description', + type=str, + default='This is an AIGC model.', + help='Model description.') aigc_group.add_argument( '--path_in_repo', type=str, @@ -127,7 +128,7 @@ def define_args(parsers: _SubParsersAction): parser.set_defaults(func=subparser_func) def execute(self): - if self.args.is_aigc: + if self.args.aigc: if self.args.repo_type != REPO_TYPE_MODEL: raise ValueError( 'AIGC models can only be created when repo_type is "model".' @@ -189,19 +190,15 @@ def _create_aigc_model(self): ) # Convert visibility string to int for the API call - visibility_int = self.args.visibility - if isinstance(visibility_int, str): - # Create reverse mapping from existing VisibilityMap - reverse_visibility_map = {v: k for k, v in VisibilityMap.items()} - visibility_int = reverse_visibility_map.get( - visibility_int, - list(VisibilityMap.keys())[2]) # Default to PUBLIC + reverse_visibility_map = {v: k for k, v in VisibilityMap.items()} + visibility_idx: int = reverse_visibility_map.get( + self.args.visibility, ModelVisibility.PUBLIC) try: model_url = api.create_model( model_id=model_id, token=self.args.token, - visibility=visibility_int, + visibility=visibility_idx, license=self.args.license, chinese_name=self.args.chinese_name, aigc_model=aigc_model) diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index 3857828a9..9f3dad07f 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -2,7 +2,6 @@ import glob import os -from enum import Enum from typing import List, Optional from modelscope.utils.logger import get_logger @@ -119,8 +118,6 @@ def _process_model_path(self): if not os.path.exists(self.model_path): raise ValueError(f'Model path does not exist: {self.model_path}') - target_file = None - if os.path.isfile(self.model_path): target_file = self.model_path logger.info('Using file: %s', os.path.basename(target_file)) From 72f47652452873c147282b904b852e8e768fbc66 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Wed, 6 Aug 2025 16:56:01 +0800 Subject: [PATCH 17/17] update --aigc for cli --- modelscope/cli/create.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index 42906af6f..c3344e80d 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -86,9 +86,7 @@ def define_args(parsers: _SubParsersAction): 'AIGC Model Creation', 'Arguments for creating an AIGC model. Use --aigc to enable.') aigc_group.add_argument( - '--aigc', - action='store_true', - help='Flag to indicate that an AIGC model should be created.') + '--aigc', action='store_true', help='Enable AIGC model creation.') aigc_group.add_argument( '--from_json', type=str,