Skip to content

Commit

Permalink
Merge pull request #7 from L-jasmine/add_script
Browse files Browse the repository at this point in the history 
Add script
  • Loading branch information
@L-jasmine
L-jasmine authored Aug 27, 2024
2 parents 325a4ba + 891b729 commit e011d86
Show file tree
Hide file tree
Showing 3 changed files with 278 additions and 0 deletions.
106 changes: 106 additions & 0 deletions add-model.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import json
import sys

import urllib.request


def get_huggingface_models(repo_id):
url = f"https://huggingface.co/api/models/{repo_id}?blobs=1"
with urllib.request.urlopen(url) as response:
response_data = response.read()
response_json = json.loads(response_data)
return response_json


def parse_to_model_cards(
hugging_face_model,
featured,
model_type,
prompt_template="llama-3-chat",
reverse_prompt="",
context_size=4096,
):
org_name, model_name = hugging_face_model["id"].split("/")
model_index = {
"id": hugging_face_model["id"],
"name": model_name,
"architecture": "",
"featured": featured,
"model_type": model_type,
"like_count": hugging_face_model["likes"],
"download_count": hugging_face_model["downloads"],
}
files = []
for item in hugging_face_model["siblings"]:
if item["rfilename"].endswith(".gguf"):
model_id = hugging_face_model["id"]
file_size = item["size"]
file_id = item["rfilename"]

file = {
"name": item["rfilename"],
"url": f"https://huggingface.co/{hugging_face_model['id']}/resolve/main/{item['rfilename']}",
"size": f"{file_size}",
"quantization": "",
"tags": [],
"sha256": item["lfs"]["sha256"],
"download": {
"default": f"https://huggingface.co/{model_id}/resolve/main/{file_id}"
},
}
files.append(file)

model_card = {
"id": hugging_face_model["id"],
"name": model_name,
"author": {
"name": org_name,
"url": f"https://huggingface.co/{org_name}",
"description": "",
},
"summary": "",
"size": "",
"requires": "",
"architecture": "",
"released_at": hugging_face_model["createdAt"],
"files": files,
"prompt_template": prompt_template,
"reverse_prompt": reverse_prompt,
"context_size": context_size,
"metrics": {},
}
return model_index, model_card


def read_index(file_path):
with open(file_path, "r", encoding="utf-8") as file:
data = json.load(file)
return data


def rewrite_index(file_path, indexs):
text = json.dumps(indexs, indent=4)
with open(file_path, "w", encoding="utf-8") as file:
file.write(text)


def save_model_card(model_card):
file_path = f"./{model_card['id']}.json"
text = json.dumps(model_card, indent=4)
with open(file_path, "w", encoding="utf-8") as file:
file.write(text)
pass


file_path = "./index.json"
model_id = sys.argv[1]

indexs = read_index(file_path)

model_index, model_card = parse_to_model_cards(
get_huggingface_models(model_id), True, "chat", prompt_template="phi-3-chat"
)

indexs.append(model_index)
save_model_card(model_card)
rewrite_index(file_path, indexs)
9 changes: 9 additions & 0 deletions index.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -774,5 +774,14 @@
"name": "",
"architecture": "",
"model_type": "unknown"
},
{
"id": "second-state/Phi-3.5-mini-instruct-GGUF",
"name": "Phi-3.5-mini-instruct-GGUF",
"architecture": "",
"featured": true,
"model_type": "chat",
"like_count": 0,
"download_count": 179
}
]
163 changes: 163 additions & 0 deletions second-state/Phi-3.5-mini-instruct-GGUF.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
{
"id": "second-state/Phi-3.5-mini-instruct-GGUF",
"name": "Phi-3.5-mini-instruct-GGUF",
"author": {
"name": "second-state",
"url": "https://huggingface.co/second-state",
"description": ""
},
"summary": "",
"size": "",
"requires": "",
"architecture": "",
"released_at": "2024-08-21T15:43:32.000Z",
"files": [
{
"name": "Phi-3.5-mini-instruct-Q2_K.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q2_K.gguf",
"size": "1416204288",
"quantization": "",
"tags": [],
"sha256": "2e6564bea11e9447560fd9a83acac1b248293c2466ffee97683d3bf2288ce8e9",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q2_K.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q3_K_L.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q3_K_L.gguf",
"size": "2087597568",
"quantization": "",
"tags": [],
"sha256": "1cf63a5a22f926a97b895e60afd722f058844dd00817062d94fa3f7aad6cfcb6",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q3_K_L.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q3_K_M.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q3_K_M.gguf",
"size": "1955476992",
"quantization": "",
"tags": [],
"sha256": "e090e1ea535e89f4f18737dbf1c79bb5a168990159e9d8a6174374fe921096ca",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q3_K_M.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q3_K_S.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q3_K_S.gguf",
"size": "1681798656",
"quantization": "",
"tags": [],
"sha256": "91e2ea239d43ab6e15fb10412425c0d092ed03f7cd1483f7f354a3001f93d67c",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q3_K_S.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q4_0.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_0.gguf",
"size": "2176177152",
"quantization": "",
"tags": [],
"sha256": "cce3a577017b01ed37ae7cf25534c5fd59045d459e0a2249551ec5a2cac6882c",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_0.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q4_K_M.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_K_M.gguf",
"size": "2393232384",
"quantization": "",
"tags": [],
"sha256": "c389ea28dc7f10dfbe30fc5e05452f832b1adf85989253ba590e3620b9584f06",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_K_M.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q4_K_S.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_K_S.gguf",
"size": "2188760064",
"quantization": "",
"tags": [],
"sha256": "e2de4ce8a64658ea9906e3996a6da944d249a0564ac0fa306fdfc09c1253defd",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q4_K_S.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q5_0.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q5_0.gguf",
"size": "2641474560",
"quantization": "",
"tags": [],
"sha256": "48acfc895623cffb32a45432bbc1134ac5bea4d423fadfd42573088fd45d9840",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q5_0.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q5_K_M.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q5_K_M.gguf",
"size": "2815276032",
"quantization": "",
"tags": [],
"sha256": "aa6cd91e2f6cb05d1894764f78fe9454faacad19983301a01e84587c02d0af02",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q5_K_M.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q5_K_S.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q5_K_S.gguf",
"size": "2641474560",
"quantization": "",
"tags": [],
"sha256": "31e861ead1d275f332b24b7be5ac6ad8dcb41d323b45eaf5d6a6ced820fba2c2",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q5_K_S.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q6_K.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q6_K.gguf",
"size": "3135853056",
"quantization": "",
"tags": [],
"sha256": "be4239a20c9f0812d799130015a8fc109e1dd3a754245bbe1eb72bf03d700c8d",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q6_K.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-Q8_0.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q8_0.gguf",
"size": "4061222400",
"quantization": "",
"tags": [],
"sha256": "81fd0b829a6ff15b3df3cc792de3f3e75a9fb2f1f55c0059af55a698b1173150",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q8_0.gguf"
}
},
{
"name": "Phi-3.5-mini-instruct-f16.gguf",
"url": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-f16.gguf",
"size": "7643297280",
"quantization": "",
"tags": [],
"sha256": "04381ade8b69632b564f1dbc473dac52f7819349ceb3cc76d268dee91f762fe2",
"download": {
"default": "https://huggingface.co/second-state/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-f16.gguf"
}
}
],
"prompt_template": "phi-3-chat",
"reverse_prompt": "",
"context_size": 4096,
"metrics": {}
}

0 comments on commit e011d86

Please sign in to comment.