Upload Models directly from 'HuggingFace' and Google Colab

[tronizx]

Hey CivitAI I Want to push this idea to implement in your site, so that we don't have to upload model checkpoint file from our pc, rather than that we can push our model or checkpoint file directly from HuggingFace official site. Or directly push from Colab Notebooks.

[JustMaier]

Are you able to upload directly to HuggingFace from Colab? If so, can you send me the notebook or command that you're using to do that?

We've got HuggingFace integration on our roadmap, we've just been absolutely swamped with on-site things since launch.

[tronizx]

Either you can do this:
https://huggingface.co/transformers/v4.10.1/model_sharing.html
or
see this:

`from slugify import slugify
from huggingface_hub import HfApi, HfFolder, CommitOperationAdd
from huggingface_hub import create_repo
from IPython.display import display_markdown
from IPython.display import clear_output
from IPython.utils import capture
from google.colab import files
import shutil
import time
import os

Upload_sample_images = False #@param {type:"boolean"}
#@markdown - Upload showcase images of your trained model

Name_of_your_concept = "" #@param {type:"string"}
if(Name_of_your_concept == ""):
  Name_of_your_concept = Session_Name
Name_of_your_concept=Name_of_your_concept.replace(" ","-")  
  
Save_concept_to = "My_Profile" #@param ["Public_Library", "My_Profile"]


hf_token_write = "" #@param {type:"string"}
if hf_token_write =="":
  print('�[1;32mYour Hugging Face write access token : ')
  hf_token_write=input()

hf_token = hf_token_write

api = HfApi()
your_username = api.whoami(token=hf_token)["name"]

if(Save_concept_to == "Public_Library"):
  repo_id = f"sd-dreambooth-library/{slugify(Name_of_your_concept)}"
  #Join the Concepts Library organization if you aren't part of it already
  !curl -X POST -H 'Authorization: Bearer '$hf_token -H 'Content-Type: application/json' https://huggingface.co/organizations/sd-dreambooth-library/share/SSeOwppVCscfTEzFGQaqpfcjukVeNrKNHX
else:
  repo_id = f"{your_username}/{slugify(Name_of_your_concept)}"
output_dir = f'/content/models/'+INSTANCE_NAME

def bar(prg):
    br="�[1;33mUploading to HuggingFace : " '�[0m|'+'█' * prg + ' ' * (25-prg)+'| ' +str(prg*4)+ "%"
    return br

print("�[1;32mLoading...")

NM="False"
if os.path.getsize(OUTPUT_DIR+"/text_encoder/pytorch_model.bin") > 670901463:
  NM="True"

with capture.capture_output() as cap:
  if NM=="False":
    %cd $OUTPUT_DIR
    !rm -r safety_checker feature_extractor .git
    !rm model_index.json
    !git init
    !git lfs install --system --skip-repo
    !git remote add -f origin  "https://USER:{hf_token}@huggingface.co/runwayml/stable-diffusion-v1-5"
    !git config core.sparsecheckout true
    !echo -e "feature_extractor\nsafety_checker\nmodel_index.json" > .git/info/sparse-checkout
    !git pull origin main
    !rm -r .git
    %cd /content
  else:
    %cd $OUTPUT_DIR
    !rm -r feature_extractor .git
    !git init
    !git lfs install --system --skip-repo
    !git remote add -f origin  "https://USER:{hf_token}@huggingface.co/stabilityai/stable-diffusion-2-1"
    !git config core.sparsecheckout true
    !echo -e "feature_extractor" > .git/info/sparse-checkout
    !git pull origin main
    !rm -r .git
    %cd /content


image_string = ""

if os.path.exists('/content/sample_images'):
  !rm -r /content/sample_images
Samples="/content/sample_images"
!mkdir $Samples
clear_output()

if Upload_sample_images:

  print("�[1;32mUpload Sample images of the model")
  uploaded = files.upload()
  for filename in uploaded.keys():
    shutil.move(filename, Samples)
  %cd $Samples
  !find . -name "* *" -type f | rename 's/ /_/g'
  %cd /content
  clear_output()

  print(bar(1))

  images_upload = os.listdir(Samples)
  instance_prompt_list = []
  for i, image in enumerate(images_upload):
      image_string = f'''
  {image_string}![{i}](https://huggingface.co/{repo_id}/resolve/main/sample_images/{image})
      '''
    
readme_text = f'''---
license: creativeml-openrail-m
tags:
- text-to-image
- stable-diffusion
---

Sample pictures of this concept:
{image_string}
'''
#Save the readme to a file
readme_file = open("README.md", "w")
readme_file.write(readme_text)
readme_file.close()

operations = [
  CommitOperationAdd(path_in_repo="README.md", path_or_fileobj="README.md"),
  CommitOperationAdd(path_in_repo=f"{Session_Name}.ckpt",path_or_fileobj=MDLPTH)

]
create_repo(repo_id,private=True, token=hf_token)

api.create_commit(
  repo_id=repo_id,
  operations=operations,
  commit_message=f"Upload the concept {Name_of_your_concept} embeds and token",
  token=hf_token
)

api.upload_folder(
  folder_path=OUTPUT_DIR+"/feature_extractor",
  path_in_repo="feature_extractor",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(4))

if NM=="False":
  api.upload_folder(
    folder_path=OUTPUT_DIR+"/safety_checker",
    path_in_repo="safety_checker",
    repo_id=repo_id,
    token=hf_token
  )

clear_output()
print(bar(8))


api.upload_folder(
  folder_path=OUTPUT_DIR+"/scheduler",
  path_in_repo="scheduler",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(9))

api.upload_folder(
  folder_path=OUTPUT_DIR+"/text_encoder",
  path_in_repo="text_encoder",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(12))

api.upload_folder(
  folder_path=OUTPUT_DIR+"/tokenizer",
  path_in_repo="tokenizer",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(13))

api.upload_folder(
  folder_path=OUTPUT_DIR+"/unet",
  path_in_repo="unet",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(21))

api.upload_folder(
  folder_path=OUTPUT_DIR+"/vae",
  path_in_repo="vae",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(23))

api.upload_file(
  path_or_fileobj=OUTPUT_DIR+"/model_index.json",
  path_in_repo="model_index.json",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(24))

api.upload_folder(
  folder_path=Samples,
  path_in_repo="sample_images",
  repo_id=repo_id,
  token=hf_token
)

clear_output()
print(bar(25))

From https://github.com/TheLastBen/fast-stable-diffusion

[tronizx]

Right I need the API or a way to upload model directly to CivitAi model hub using programmatically.