Skip to content

Commit 67e97ab

Browse files
cookieyydscookieyyds
cookieyyds
and
cookieyyds
authored
FEAT: add download from openmind_hub (#2504)
Co-authored-by: cookieyyds <@[email protected]>
1 parent 07f9325 commit 67e97ab

File tree

15 files changed

+1527
-24
lines changed

15 files changed

+1527
-24
lines changed

‎doc/source/conf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@
121121
"type": "fontawesome",
122122
}])
123123
html_theme_options["external_links"] = [
124-
{"name": "产品官网", "url": "https://xorbits.cn/inference"},
124+
{"name": "产品官网", "url": "https://xorbits.cn"},
125125
]
126126

127127
html_favicon = "_static/favicon.svg"

‎xinference/constants.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ def get_xinference_home() -> str:
3939
# if user has already set `XINFERENCE_HOME` env, change huggingface and modelscope default download path
4040
os.environ["HUGGINGFACE_HUB_CACHE"] = os.path.join(home_path, "huggingface")
4141
os.environ["MODELSCOPE_CACHE"] = os.path.join(home_path, "modelscope")
42+
os.environ["XDG_CACHE_HOME"] = os.path.join(home_path, "openmind_hub")
4243
# In multi-tenant mode,
4344
# gradio's temporary files are stored in their respective home directories,
4445
# to prevent insufficient permissions

‎xinference/core/worker.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -785,7 +785,9 @@ async def launch_builtin_model(
785785
peft_model_config: Optional[PeftModelConfig] = None,
786786
request_limits: Optional[int] = None,
787787
gpu_idx: Optional[Union[int, List[int]]] = None,
788-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
788+
download_hub: Optional[
789+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
790+
] = None,
789791
model_path: Optional[str] = None,
790792
**kwargs,
791793
):

‎xinference/model/audio/core.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,9 @@ def generate_audio_description(
100100

101101
def match_audio(
102102
model_name: str,
103-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
103+
download_hub: Optional[
104+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
105+
] = None,
104106
) -> AudioModelFamilyV1:
105107
from ..utils import download_from_modelscope
106108
from . import BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS
@@ -152,7 +154,9 @@ def create_audio_model_instance(
152154
devices: List[str],
153155
model_uid: str,
154156
model_name: str,
155-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
157+
download_hub: Optional[
158+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
159+
] = None,
156160
model_path: Optional[str] = None,
157161
**kwargs,
158162
) -> Tuple[

‎xinference/model/core.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,9 @@ def create_model_instance(
5555
model_size_in_billions: Optional[Union[int, str]] = None,
5656
quantization: Optional[str] = None,
5757
peft_model_config: Optional[PeftModelConfig] = None,
58-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
58+
download_hub: Optional[
59+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
60+
] = None,
5961
model_path: Optional[str] = None,
6062
**kwargs,
6163
) -> Tuple[Any, ModelDescription]:

‎xinference/model/embedding/core.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,9 @@ def encode(
433433

434434
def match_embedding(
435435
model_name: str,
436-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
436+
download_hub: Optional[
437+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
438+
] = None,
437439
) -> EmbeddingModelSpec:
438440
from ..utils import download_from_modelscope
439441
from . import BUILTIN_EMBEDDING_MODELS, MODELSCOPE_EMBEDDING_MODELS
@@ -469,7 +471,9 @@ def create_embedding_model_instance(
469471
devices: List[str],
470472
model_uid: str,
471473
model_name: str,
472-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
474+
download_hub: Optional[
475+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
476+
] = None,
473477
model_path: Optional[str] = None,
474478
**kwargs,
475479
) -> Tuple[EmbeddingModel, EmbeddingModelDescription]:

‎xinference/model/image/core.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,9 @@ def generate_image_description(
125125

126126
def match_diffusion(
127127
model_name: str,
128-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
128+
download_hub: Optional[
129+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
130+
] = None,
129131
) -> ImageModelFamilyV1:
130132
from ..utils import download_from_modelscope
131133
from . import BUILTIN_IMAGE_MODELS, MODELSCOPE_IMAGE_MODELS
@@ -213,7 +215,9 @@ def create_image_model_instance(
213215
model_uid: str,
214216
model_name: str,
215217
peft_model_config: Optional[PeftModelConfig] = None,
216-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
218+
download_hub: Optional[
219+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
220+
] = None,
217221
model_path: Optional[str] = None,
218222
**kwargs,
219223
) -> Tuple[

‎xinference/model/llm/__init__.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES,
3333
BUILTIN_LLM_PROMPT_STYLE,
3434
BUILTIN_MODELSCOPE_LLM_FAMILIES,
35+
BUILTIN_OPENMIND_HUB_LLM_FAMILIES,
3536
LLAMA_CLASSES,
3637
LLM_ENGINES,
3738
LMDEPLOY_CLASSES,
@@ -258,6 +259,36 @@ def _install():
258259
if "tools" in model_spec.model_ability:
259260
BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES.add(model_spec.model_name)
260261

262+
openmind_hub_json_path = os.path.join(
263+
os.path.dirname(os.path.abspath(__file__)), "llm_family_openmind_hub.json"
264+
)
265+
for json_obj in json.load(
266+
codecs.open(openmind_hub_json_path, "r", encoding="utf-8")
267+
):
268+
model_spec = LLMFamilyV1.parse_obj(json_obj)
269+
BUILTIN_OPENMIND_HUB_LLM_FAMILIES.append(model_spec)
270+
271+
# register prompt style, in case that we have something missed
272+
# if duplicated with huggingface json, keep it as the huggingface style
273+
274+
if (
275+
"chat" in model_spec.model_ability
276+
and isinstance(model_spec.chat_template, str)
277+
and model_spec.model_name not in BUILTIN_LLM_PROMPT_STYLE
278+
):
279+
BUILTIN_LLM_PROMPT_STYLE[model_spec.model_name] = {
280+
"chat_template": model_spec.chat_template,
281+
"stop_token_ids": model_spec.stop_token_ids,
282+
"stop": model_spec.stop,
283+
}
284+
# register model family
285+
if "chat" in model_spec.model_ability:
286+
BUILTIN_LLM_MODEL_CHAT_FAMILIES.add(model_spec.model_name)
287+
else:
288+
BUILTIN_LLM_MODEL_GENERATE_FAMILIES.add(model_spec.model_name)
289+
if "tools" in model_spec.model_ability:
290+
BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES.add(model_spec.model_name)
291+
261292
csghub_json_path = os.path.join(
262293
os.path.dirname(os.path.abspath(__file__)), "llm_family_csghub.json"
263294
)
@@ -288,6 +319,7 @@ def _install():
288319
for llm_specs in [
289320
BUILTIN_LLM_FAMILIES,
290321
BUILTIN_MODELSCOPE_LLM_FAMILIES,
322+
BUILTIN_OPENMIND_HUB_LLM_FAMILIES,
291323
BUILTIN_CSGHUB_LLM_FAMILIES,
292324
]:
293325
for llm_spec in llm_specs:
@@ -298,6 +330,7 @@ def _install():
298330
for families in [
299331
BUILTIN_LLM_FAMILIES,
300332
BUILTIN_MODELSCOPE_LLM_FAMILIES,
333+
BUILTIN_OPENMIND_HUB_LLM_FAMILIES,
301334
BUILTIN_CSGHUB_LLM_FAMILIES,
302335
]:
303336
for family in families:

‎xinference/model/llm/core.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,9 @@ def create_llm_model_instance(
193193
model_size_in_billions: Optional[Union[int, str]] = None,
194194
quantization: Optional[str] = None,
195195
peft_model_config: Optional[PeftModelConfig] = None,
196-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
196+
download_hub: Optional[
197+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
198+
] = None,
197199
model_path: Optional[str] = None,
198200
**kwargs,
199201
) -> Tuple[LLM, LLMDescription]:

‎xinference/model/llm/llm_family.py

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
create_symlink,
4242
download_from_csghub,
4343
download_from_modelscope,
44+
download_from_openmind_hub,
4445
is_valid_model_uri,
4546
parse_uri,
4647
retry_download,
@@ -239,6 +240,7 @@ def parse_raw(
239240

240241
BUILTIN_LLM_FAMILIES: List["LLMFamilyV1"] = []
241242
BUILTIN_MODELSCOPE_LLM_FAMILIES: List["LLMFamilyV1"] = []
243+
BUILTIN_OPENMIND_HUB_LLM_FAMILIES: List["LLMFamilyV1"] = []
242244
BUILTIN_CSGHUB_LLM_FAMILIES: List["LLMFamilyV1"] = []
243245

244246
SGLANG_CLASSES: List[Type[LLM]] = []
@@ -301,6 +303,9 @@ def cache(
301303
elif llm_spec.model_hub == "modelscope":
302304
logger.info(f"Caching from Modelscope: {llm_spec.model_id}")
303305
return cache_from_modelscope(llm_family, llm_spec, quantization)
306+
elif llm_spec.model_hub == "openmind_hub":
307+
logger.info(f"Caching from openmind_hub: {llm_spec.model_id}")
308+
return cache_from_openmind_hub(llm_family, llm_spec, quantization)
304309
elif llm_spec.model_hub == "csghub":
305310
logger.info(f"Caching from CSGHub: {llm_spec.model_id}")
306311
return cache_from_csghub(llm_family, llm_spec, quantization)
@@ -474,14 +479,17 @@ def _skip_download(
474479
model_revision: Optional[str],
475480
quantization: Optional[str] = None,
476481
) -> bool:
477-
if model_format == "pytorch":
482+
if model_format in ["pytorch", "mindspore"]:
478483
model_hub_to_meta_path = {
479484
"huggingface": _get_meta_path(
480485
cache_dir, model_format, "huggingface", quantization
481486
),
482487
"modelscope": _get_meta_path(
483488
cache_dir, model_format, "modelscope", quantization
484489
),
490+
"openmind_hub": _get_meta_path(
491+
cache_dir, model_format, "openmind_hub", quantization
492+
),
485493
"csghub": _get_meta_path(cache_dir, model_format, "csghub", quantization),
486494
}
487495
if valid_model_revision(model_hub_to_meta_path[model_hub], model_revision):
@@ -702,6 +710,50 @@ def cache_from_modelscope(
702710
return cache_dir
703711

704712

713+
def cache_from_openmind_hub(
714+
llm_family: LLMFamilyV1,
715+
llm_spec: "LLMSpecV1",
716+
quantization: Optional[str] = None,
717+
) -> str:
718+
"""
719+
Cache model from openmind_hub. Return the cache directory.
720+
"""
721+
from openmind_hub import snapshot_download
722+
723+
cache_dir = _get_cache_dir(llm_family, llm_spec)
724+
if _skip_download(
725+
cache_dir,
726+
llm_spec.model_format,
727+
llm_spec.model_hub,
728+
llm_spec.model_revision,
729+
quantization,
730+
):
731+
return cache_dir
732+
733+
if llm_spec.model_format in ["pytorch", "mindspore"]:
734+
download_dir = retry_download(
735+
snapshot_download,
736+
llm_family.model_name,
737+
{
738+
"model_size": llm_spec.model_size_in_billions,
739+
"model_format": llm_spec.model_format,
740+
},
741+
llm_spec.model_id,
742+
revision=llm_spec.model_revision,
743+
)
744+
create_symlink(download_dir, cache_dir)
745+
746+
else:
747+
raise ValueError(f"Unsupported format: {llm_spec.model_format}")
748+
749+
meta_path = _get_meta_path(
750+
cache_dir, llm_spec.model_format, llm_spec.model_hub, quantization
751+
)
752+
_generate_meta_file(meta_path, llm_family, llm_spec, quantization)
753+
754+
return cache_dir
755+
756+
705757
def cache_from_huggingface(
706758
llm_family: LLMFamilyV1,
707759
llm_spec: "LLMSpecV1",
@@ -893,7 +945,9 @@ def match_llm(
893945
model_format: Optional[str] = None,
894946
model_size_in_billions: Optional[Union[int, str]] = None,
895947
quantization: Optional[str] = None,
896-
download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
948+
download_hub: Optional[
949+
Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
950+
] = None,
897951
) -> Optional[Tuple[LLMFamilyV1, LLMSpecV1, str]]:
898952
"""
899953
Find an LLM family, spec, and quantization that satisfy given criteria.
@@ -924,6 +978,12 @@ def _apply_format_to_model_id(spec: LLMSpecV1, q: str) -> LLMSpecV1:
924978
+ BUILTIN_LLM_FAMILIES
925979
+ user_defined_llm_families
926980
)
981+
elif download_hub == "openmind_hub":
982+
all_families = (
983+
BUILTIN_OPENMIND_HUB_LLM_FAMILIES
984+
+ BUILTIN_LLM_FAMILIES
985+
+ user_defined_llm_families
986+
)
927987
elif download_hub == "csghub":
928988
all_families = (
929989
BUILTIN_CSGHUB_LLM_FAMILIES
@@ -938,6 +998,12 @@ def _apply_format_to_model_id(spec: LLMSpecV1, q: str) -> LLMSpecV1:
938998
+ BUILTIN_LLM_FAMILIES
939999
+ user_defined_llm_families
9401000
)
1001+
elif download_from_openmind_hub():
1002+
all_families = (
1003+
BUILTIN_OPENMIND_HUB_LLM_FAMILIES
1004+
+ BUILTIN_LLM_FAMILIES
1005+
+ user_defined_llm_families
1006+
)
9411007
elif download_from_csghub():
9421008
all_families = (
9431009
BUILTIN_CSGHUB_LLM_FAMILIES

0 commit comments

Comments
 (0)