diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1fb9ef5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tmp/* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..19ac1d1 --- /dev/null +++ b/Makefile @@ -0,0 +1,70 @@ +# tool macros +CXX := g++ +#CXXFLAGS := -funsafe-math-optimizations -Ofast -flto=auto -funroll-all-loops -pipe -march=native -std=c++20 -Wall -Wextra `pkg-config --cflags --libs opencv4` +CXXFLAGS := -funsafe-math-optimizations -Ofast -flto=auto -funroll-all-loops -pipe -march=native -std=c++20 -Wall -Wextra +DBGFLAGS := -g +CXXOBJFLAGS := $(CXXFLAGS) -c +LFLAGS := -lncnn -lstdc++ -pthread -Wl,--gc-sections -flto -fopt-info-vec-optimized + +# path macros +BIN_PATH := bin +OBJ_PATH := obj +SRC_PATH := src +DBG_PATH := debug + +# compile macros +TARGET_NAME := stablencnn +ifeq ($(OS),Windows_NT) + TARGET_NAME := $(addsuffix .exe,$(TARGET_NAME)) +endif +TARGET := $(BIN_PATH)/$(TARGET_NAME) +TARGET_DEBUG := $(DBG_PATH)/$(TARGET_NAME) + +# src files & obj files +SRC := $(foreach x, $(SRC_PATH), $(wildcard $(addprefix $(x)/*,.cpp))) +OBJ := $(addprefix $(OBJ_PATH)/, $(addsuffix .o, $(notdir $(basename $(SRC))))) +OBJ_DEBUG := $(addprefix $(DBG_PATH)/, $(addsuffix .o, $(notdir $(basename $(SRC))))) + +# clean files list +DISTCLEAN_LIST := $(OBJ) \ + $(OBJ_DEBUG) +CLEAN_LIST := $(TARGET) \ + $(TARGET_DEBUG) \ + $(DISTCLEAN_LIST) + +# default rule +default: makedir all + +# non-phony targets +$(TARGET): $(OBJ) + $(CXX) -o $@ $(OBJ) $(CXXFLAGS) $(LFLAGS) + +$(OBJ_PATH)/%.o: $(SRC_PATH)/%.cpp + $(CXX) $(CXXOBJFLAGS) -o $@ $< + +$(DBG_PATH)/%.o: $(SRC_PATH)/%.cpp + $(CXX) $(CXXOBJFLAGS) $(DBGFLAGS) -o $@ $< + +$(TARGET_DEBUG): $(OBJ_DEBUG) + $(CXX) $(CXXFLAGS) $(DBGFLAGS) $(OBJ_DEBUG) -o $@ $(LFLAGS) + +# phony rules +.PHONY: makedir +makedir: + @mkdir -p $(BIN_PATH) $(OBJ_PATH) $(DBG_PATH) + +.PHONY: all +all: $(TARGET) + +.PHONY: debug +debug: $(TARGET_DEBUG) + +.PHONY: clean +clean: + @echo CLEAN $(CLEAN_LIST) + @rm -f $(CLEAN_LIST) + +.PHONY: distclean +distclean: + @echo CLEAN $(DISTCLEAN_LIST) + @rm -f $(DISTCLEAN_LIST) diff --git a/README.md b/README.md new file mode 100644 index 0000000..35e0bc6 --- /dev/null +++ b/README.md @@ -0,0 +1,383 @@ +# Stable Diffusion NCNN + +**This is a public fork from [EdVince/Stable-Diffusion-NCNN](https://github.com/EdVince/Stable-Diffusion-NCNN)**. +All the credit goes to [EdVince](https://github.com/EdVince). +As I cannot push LFS object to a public fork, I make this repo standalone. + +Features appended: +1. remove OpenCV dependency +2. reduce memory usage from 7 GB to 5.5 GB +3. enable 4X super resolution from [Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN) + + +## Quick introduction +1. Checkout the [readme from EdVince's repo](https://github.com/EdVince/Stable-Diffusion-NCNN/blob/main/README.md) for implementation details. +2. Download and extract the assets from the replease of this repo, the layout should looks like: + +> +> YOUREXECUTABLE +> assets +> ├── AutoencoderKL-fp16.bin +> ├── AutoencoderKL-fp16.param +> ├── FrozenCLIPEmbedder-fp16.bin +> ├── FrozenCLIPEmbedder-fp16.param +> ├── log_sigmas.bin +> ├── RealESRGAN_x4plus_anime_6B.fp32-sim-sim-opt.bin +> ├── RealESRGAN_x4plus_anime_6B.fp32-sim-sim-opt.param +> ├── UNetModel-fp16.bin +> ├── UNetModel-fp16.param +> └── vocab.txt +> + +## Example usage: + +The C++ interface: + +```cpp +void sd( std::string positive_prompt=std::string{}, std::string output_png_path=std::string{}, std::string negative_prompt=std::string{}, int step=30, int seed=42 ); +``` + +The default parameters for the above interface can be found from [here](https://github.com/fengwang/Stable-Diffusion-NCNN/blob/56a00bb197dc66dc74ceb69334b09a3d184cbd72/src/stable_diffusion.cpp#L381) on. +Example usage can be found from file src/main.cpp. + +## Build + +This project only relies on [ncnn](https://github.com/Tencent/ncnn). If **ncnn** has been installed on your system, you can safely execute `make` to build this project. For Archlinux users, [ncnn-git](https://aur.archlinux.org/packages/ncnn-git) package from AUR is sufficient. For others, please install it manually. + +```bash +git clone https://github.com/fengwang/Stable-Diffusion-NCNN.git +cd Stable-Diffusion-NCNN +make +./bin/stablencnn +``` + +However, you need **5.5 GB** memory to run. + + +### Linking to NCNN + +If **ncnn** is not installed in the standard paths (such as `/usr/include/ncnn` and `/usr/lib/libncnn.so`, please update the makefile to comply. For example: + +Update CXXFLAGS + +``` +#CXXFLAGS := -funsafe-math-optimizations -Ofast -flto=auto -funroll-all-loops -pipe -march=native -std=c++20 -Wall -Wextra +CXXFLAGS := -funsafe-math-optimizations -Ofast -flto=auto -funroll-all-loops -pipe -march=native -std=c++20 -Wall -Wextra `pkg-config --cflags ncnn` +``` + +Update LFLAGS: + +``` +#LFLAGS := -lncnn -lstdc++ -pthread -Wl,--gc-sections -flto -fopt-info-vec-optimized +LFLAGS := `pkg-config --libs ncnn` -lstdc++ -pthread -Wl,--gc-sections -flto -fopt-info-vec-optimized +``` + + +## TODO +1. integrate super-resolution model in the pipeline +2. provide a GUI +3. provide a python binding (?) + + +## 99 Examples + +Generating using the default `positive_prompt` and default `negative_prompt` with different seed + + +### seed = 1 + +![1](./resources/result_1.png) +### seed = 2 + +![2](./resources/result_2.png) +### seed = 3 + +![3](./resources/result_3.png) +### seed = 4 + +![4](./resources/result_4.png) +### seed = 5 + +![5](./resources/result_5.png) +### seed = 6 + +![6](./resources/result_6.png) +### seed = 7 + +![7](./resources/result_7.png) +### seed = 8 + +![8](./resources/result_8.png) +### seed = 9 + +![9](./resources/result_9.png) +### seed = 10 + +![10](./resources/result_10.png) +### seed = 11 + +![11](./resources/result_11.png) +### seed = 12 + +![12](./resources/result_12.png) +### seed = 13 + +![13](./resources/result_13.png) +### seed = 14 + +![14](./resources/result_14.png) +### seed = 15 + +![15](./resources/result_15.png) +### seed = 16 + +![16](./resources/result_16.png) +### seed = 17 + +![17](./resources/result_17.png) +### seed = 18 + +![18](./resources/result_18.png) +### seed = 19 + +![19](./resources/result_19.png) +### seed = 20 + +![20](./resources/result_20.png) +### seed = 21 + +![21](./resources/result_21.png) +### seed = 22 + +![22](./resources/result_22.png) +### seed = 23 + +![23](./resources/result_23.png) +### seed = 24 + +![24](./resources/result_24.png) +### seed = 25 + +![25](./resources/result_25.png) +### seed = 26 + +![26](./resources/result_26.png) +### seed = 27 + +![27](./resources/result_27.png) +### seed = 28 + +![28](./resources/result_28.png) +### seed = 29 + +![29](./resources/result_29.png) +### seed = 30 + +![30](./resources/result_30.png) +### seed = 31 + +![31](./resources/result_31.png) +### seed = 32 + +![32](./resources/result_32.png) +### seed = 33 + +![33](./resources/result_33.png) +### seed = 34 + +![34](./resources/result_34.png) +### seed = 35 + +![35](./resources/result_35.png) +### seed = 36 + +![36](./resources/result_36.png) +### seed = 37 + +![37](./resources/result_37.png) +### seed = 38 + +![38](./resources/result_38.png) +### seed = 39 + +![39](./resources/result_39.png) +### seed = 40 + +![40](./resources/result_40.png) +### seed = 41 + +![41](./resources/result_41.png) +### seed = 42 + +![42](./resources/result_42.png) +### seed = 43 + +![43](./resources/result_43.png) +### seed = 44 + +![44](./resources/result_44.png) +### seed = 45 + +![45](./resources/result_45.png) +### seed = 46 + +![46](./resources/result_46.png) +### seed = 47 + +![47](./resources/result_47.png) +### seed = 48 + +![48](./resources/result_48.png) +### seed = 49 + +![49](./resources/result_49.png) +### seed = 50 + +![50](./resources/result_50.png) +### seed = 51 + +![51](./resources/result_51.png) +### seed = 52 + +![52](./resources/result_52.png) +### seed = 53 + +![53](./resources/result_53.png) +### seed = 54 + +![54](./resources/result_54.png) +### seed = 55 + +![55](./resources/result_55.png) +### seed = 56 + +![56](./resources/result_56.png) +### seed = 57 + +![57](./resources/result_57.png) +### seed = 58 + +![58](./resources/result_58.png) +### seed = 59 + +![59](./resources/result_59.png) +### seed = 60 + +![60](./resources/result_60.png) +### seed = 61 + +![61](./resources/result_61.png) +### seed = 62 + +![62](./resources/result_62.png) +### seed = 63 + +![63](./resources/result_63.png) +### seed = 64 + +![64](./resources/result_64.png) +### seed = 65 + +![65](./resources/result_65.png) +### seed = 66 + +![66](./resources/result_66.png) +### seed = 67 + +![67](./resources/result_67.png) +### seed = 68 + +![68](./resources/result_68.png) +### seed = 69 + +![69](./resources/result_69.png) +### seed = 70 + +![70](./resources/result_70.png) +### seed = 71 + +![71](./resources/result_71.png) +### seed = 72 + +![72](./resources/result_72.png) +### seed = 73 + +![73](./resources/result_73.png) +### seed = 74 + +![74](./resources/result_74.png) +### seed = 75 + +![75](./resources/result_75.png) +### seed = 76 + +![76](./resources/result_76.png) +### seed = 77 + +![77](./resources/result_77.png) +### seed = 78 + +![78](./resources/result_78.png) +### seed = 79 + +![79](./resources/result_79.png) +### seed = 80 + +![80](./resources/result_80.png) +### seed = 81 + +![81](./resources/result_81.png) +### seed = 82 + +![82](./resources/result_82.png) +### seed = 83 + +![83](./resources/result_83.png) +### seed = 84 + +![84](./resources/result_84.png) +### seed = 85 + +![85](./resources/result_85.png) +### seed = 86 + +![86](./resources/result_86.png) +### seed = 87 + +![87](./resources/result_87.png) +### seed = 88 + +![88](./resources/result_88.png) +### seed = 89 + +![89](./resources/result_89.png) +### seed = 90 + +![90](./resources/result_90.png) +### seed = 91 + +![91](./resources/result_91.png) +### seed = 92 + +![92](./resources/result_92.png) +### seed = 93 + +![93](./resources/result_93.png) +### seed = 94 + +![94](./resources/result_94.png) +### seed = 95 + +![95](./resources/result_95.png) +### seed = 96 + +![96](./resources/result_96.png) +### seed = 97 + +![97](./resources/result_97.png) +### seed = 98 + +![98](./resources/result_98.png) +### seed = 99 + +![99](./resources/result_99.png) diff --git a/resources/result_1.png b/resources/result_1.png new file mode 100644 index 0000000..e9a0a40 Binary files /dev/null and b/resources/result_1.png differ diff --git a/resources/result_10.png b/resources/result_10.png new file mode 100644 index 0000000..a8dd4d2 Binary files /dev/null and b/resources/result_10.png differ diff --git a/resources/result_11.png b/resources/result_11.png new file mode 100644 index 0000000..401ab8f Binary files /dev/null and b/resources/result_11.png differ diff --git a/resources/result_12.png b/resources/result_12.png new file mode 100644 index 0000000..1d65b96 Binary files /dev/null and b/resources/result_12.png differ diff --git a/resources/result_13.png b/resources/result_13.png new file mode 100644 index 0000000..3ee6a72 Binary files /dev/null and b/resources/result_13.png differ diff --git a/resources/result_14.png b/resources/result_14.png new file mode 100644 index 0000000..65f70f5 Binary files /dev/null and b/resources/result_14.png differ diff --git a/resources/result_15.png b/resources/result_15.png new file mode 100644 index 0000000..42d64fe Binary files /dev/null and b/resources/result_15.png differ diff --git a/resources/result_16.png b/resources/result_16.png new file mode 100644 index 0000000..cfca13d Binary files /dev/null and b/resources/result_16.png differ diff --git a/resources/result_17.png b/resources/result_17.png new file mode 100644 index 0000000..8924e88 Binary files /dev/null and b/resources/result_17.png differ diff --git a/resources/result_18.png b/resources/result_18.png new file mode 100644 index 0000000..a0d0df1 Binary files /dev/null and b/resources/result_18.png differ diff --git a/resources/result_19.png b/resources/result_19.png new file mode 100644 index 0000000..36bfc49 Binary files /dev/null and b/resources/result_19.png differ diff --git a/resources/result_2.png b/resources/result_2.png new file mode 100644 index 0000000..bc4fdaf Binary files /dev/null and b/resources/result_2.png differ diff --git a/resources/result_20.png b/resources/result_20.png new file mode 100644 index 0000000..c922865 Binary files /dev/null and b/resources/result_20.png differ diff --git a/resources/result_21.png b/resources/result_21.png new file mode 100644 index 0000000..eebfad6 Binary files /dev/null and b/resources/result_21.png differ diff --git a/resources/result_22.png b/resources/result_22.png new file mode 100644 index 0000000..217f6ec Binary files /dev/null and b/resources/result_22.png differ diff --git a/resources/result_23.png b/resources/result_23.png new file mode 100644 index 0000000..b02b260 Binary files /dev/null and b/resources/result_23.png differ diff --git a/resources/result_24.png b/resources/result_24.png new file mode 100644 index 0000000..0add97d Binary files /dev/null and b/resources/result_24.png differ diff --git a/resources/result_25.png b/resources/result_25.png new file mode 100644 index 0000000..750c674 Binary files /dev/null and b/resources/result_25.png differ diff --git a/resources/result_26.png b/resources/result_26.png new file mode 100644 index 0000000..3235e55 Binary files /dev/null and b/resources/result_26.png differ diff --git a/resources/result_27.png b/resources/result_27.png new file mode 100644 index 0000000..7b2b468 Binary files /dev/null and b/resources/result_27.png differ diff --git a/resources/result_28.png b/resources/result_28.png new file mode 100644 index 0000000..9adca24 Binary files /dev/null and b/resources/result_28.png differ diff --git a/resources/result_29.png b/resources/result_29.png new file mode 100644 index 0000000..89b4e70 Binary files /dev/null and b/resources/result_29.png differ diff --git a/resources/result_3.png b/resources/result_3.png new file mode 100644 index 0000000..34867fc Binary files /dev/null and b/resources/result_3.png differ diff --git a/resources/result_30.png b/resources/result_30.png new file mode 100644 index 0000000..27b3040 Binary files /dev/null and b/resources/result_30.png differ diff --git a/resources/result_31.png b/resources/result_31.png new file mode 100644 index 0000000..147d3b8 Binary files /dev/null and b/resources/result_31.png differ diff --git a/resources/result_32.png b/resources/result_32.png new file mode 100644 index 0000000..0823b7f Binary files /dev/null and b/resources/result_32.png differ diff --git a/resources/result_33.png b/resources/result_33.png new file mode 100644 index 0000000..21d0a4a Binary files /dev/null and b/resources/result_33.png differ diff --git a/resources/result_34.png b/resources/result_34.png new file mode 100644 index 0000000..6ddfcc2 Binary files /dev/null and b/resources/result_34.png differ diff --git a/resources/result_35.png b/resources/result_35.png new file mode 100644 index 0000000..54beb72 Binary files /dev/null and b/resources/result_35.png differ diff --git a/resources/result_36.png b/resources/result_36.png new file mode 100644 index 0000000..061be5f Binary files /dev/null and b/resources/result_36.png differ diff --git a/resources/result_37.png b/resources/result_37.png new file mode 100644 index 0000000..8887d14 Binary files /dev/null and b/resources/result_37.png differ diff --git a/resources/result_38.png b/resources/result_38.png new file mode 100644 index 0000000..cccc1dc Binary files /dev/null and b/resources/result_38.png differ diff --git a/resources/result_39.png b/resources/result_39.png new file mode 100644 index 0000000..9bbbaa9 Binary files /dev/null and b/resources/result_39.png differ diff --git a/resources/result_4.png b/resources/result_4.png new file mode 100644 index 0000000..206a319 Binary files /dev/null and b/resources/result_4.png differ diff --git a/resources/result_40.png b/resources/result_40.png new file mode 100644 index 0000000..ac7335c Binary files /dev/null and b/resources/result_40.png differ diff --git a/resources/result_41.png b/resources/result_41.png new file mode 100644 index 0000000..c552057 Binary files /dev/null and b/resources/result_41.png differ diff --git a/resources/result_42.png b/resources/result_42.png new file mode 100644 index 0000000..c81aaff Binary files /dev/null and b/resources/result_42.png differ diff --git a/resources/result_43.png b/resources/result_43.png new file mode 100644 index 0000000..d4e368c Binary files /dev/null and b/resources/result_43.png differ diff --git a/resources/result_44.png b/resources/result_44.png new file mode 100644 index 0000000..bf647cf Binary files /dev/null and b/resources/result_44.png differ diff --git a/resources/result_45.png b/resources/result_45.png new file mode 100644 index 0000000..9ddf96d Binary files /dev/null and b/resources/result_45.png differ diff --git a/resources/result_46.png b/resources/result_46.png new file mode 100644 index 0000000..b2d1122 Binary files /dev/null and b/resources/result_46.png differ diff --git a/resources/result_47.png b/resources/result_47.png new file mode 100644 index 0000000..ac74d80 Binary files /dev/null and b/resources/result_47.png differ diff --git a/resources/result_48.png b/resources/result_48.png new file mode 100644 index 0000000..5f5d64d Binary files /dev/null and b/resources/result_48.png differ diff --git a/resources/result_49.png b/resources/result_49.png new file mode 100644 index 0000000..c16a28b Binary files /dev/null and b/resources/result_49.png differ diff --git a/resources/result_5.png b/resources/result_5.png new file mode 100644 index 0000000..2920522 Binary files /dev/null and b/resources/result_5.png differ diff --git a/resources/result_50.png b/resources/result_50.png new file mode 100644 index 0000000..60ac568 Binary files /dev/null and b/resources/result_50.png differ diff --git a/resources/result_51.png b/resources/result_51.png new file mode 100644 index 0000000..9cd3154 Binary files /dev/null and b/resources/result_51.png differ diff --git a/resources/result_52.png b/resources/result_52.png new file mode 100644 index 0000000..83c83c5 Binary files /dev/null and b/resources/result_52.png differ diff --git a/resources/result_53.png b/resources/result_53.png new file mode 100644 index 0000000..4bad18b Binary files /dev/null and b/resources/result_53.png differ diff --git a/resources/result_54.png b/resources/result_54.png new file mode 100644 index 0000000..87c03d5 Binary files /dev/null and b/resources/result_54.png differ diff --git a/resources/result_55.png b/resources/result_55.png new file mode 100644 index 0000000..6b20b7f Binary files /dev/null and b/resources/result_55.png differ diff --git a/resources/result_56.png b/resources/result_56.png new file mode 100644 index 0000000..d5f4fc5 Binary files /dev/null and b/resources/result_56.png differ diff --git a/resources/result_57.png b/resources/result_57.png new file mode 100644 index 0000000..d22b46c Binary files /dev/null and b/resources/result_57.png differ diff --git a/resources/result_58.png b/resources/result_58.png new file mode 100644 index 0000000..2588ca1 Binary files /dev/null and b/resources/result_58.png differ diff --git a/resources/result_59.png b/resources/result_59.png new file mode 100644 index 0000000..93e5389 Binary files /dev/null and b/resources/result_59.png differ diff --git a/resources/result_6.png b/resources/result_6.png new file mode 100644 index 0000000..8fa61d0 Binary files /dev/null and b/resources/result_6.png differ diff --git a/resources/result_60.png b/resources/result_60.png new file mode 100644 index 0000000..36bbfbc Binary files /dev/null and b/resources/result_60.png differ diff --git a/resources/result_61.png b/resources/result_61.png new file mode 100644 index 0000000..2c1fac3 Binary files /dev/null and b/resources/result_61.png differ diff --git a/resources/result_62.png b/resources/result_62.png new file mode 100644 index 0000000..086ec40 Binary files /dev/null and b/resources/result_62.png differ diff --git a/resources/result_63.png b/resources/result_63.png new file mode 100644 index 0000000..ab07aba Binary files /dev/null and b/resources/result_63.png differ diff --git a/resources/result_64.png b/resources/result_64.png new file mode 100644 index 0000000..0a88292 Binary files /dev/null and b/resources/result_64.png differ diff --git a/resources/result_65.png b/resources/result_65.png new file mode 100644 index 0000000..9e4af3e Binary files /dev/null and b/resources/result_65.png differ diff --git a/resources/result_66.png b/resources/result_66.png new file mode 100644 index 0000000..8cd2f02 Binary files /dev/null and b/resources/result_66.png differ diff --git a/resources/result_67.png b/resources/result_67.png new file mode 100644 index 0000000..801dd1f Binary files /dev/null and b/resources/result_67.png differ diff --git a/resources/result_68.png b/resources/result_68.png new file mode 100644 index 0000000..27998af Binary files /dev/null and b/resources/result_68.png differ diff --git a/resources/result_69.png b/resources/result_69.png new file mode 100644 index 0000000..b121040 Binary files /dev/null and b/resources/result_69.png differ diff --git a/resources/result_7.png b/resources/result_7.png new file mode 100644 index 0000000..bfd5c31 Binary files /dev/null and b/resources/result_7.png differ diff --git a/resources/result_70.png b/resources/result_70.png new file mode 100644 index 0000000..cd63e8e Binary files /dev/null and b/resources/result_70.png differ diff --git a/resources/result_71.png b/resources/result_71.png new file mode 100644 index 0000000..5b81f66 Binary files /dev/null and b/resources/result_71.png differ diff --git a/resources/result_72.png b/resources/result_72.png new file mode 100644 index 0000000..9b738e9 Binary files /dev/null and b/resources/result_72.png differ diff --git a/resources/result_73.png b/resources/result_73.png new file mode 100644 index 0000000..48432a0 Binary files /dev/null and b/resources/result_73.png differ diff --git a/resources/result_74.png b/resources/result_74.png new file mode 100644 index 0000000..fee2705 Binary files /dev/null and b/resources/result_74.png differ diff --git a/resources/result_75.png b/resources/result_75.png new file mode 100644 index 0000000..91aa360 Binary files /dev/null and b/resources/result_75.png differ diff --git a/resources/result_76.png b/resources/result_76.png new file mode 100644 index 0000000..3d2c4d1 Binary files /dev/null and b/resources/result_76.png differ diff --git a/resources/result_77.png b/resources/result_77.png new file mode 100644 index 0000000..9ace961 Binary files /dev/null and b/resources/result_77.png differ diff --git a/resources/result_78.png b/resources/result_78.png new file mode 100644 index 0000000..6fce8d1 Binary files /dev/null and b/resources/result_78.png differ diff --git a/resources/result_79.png b/resources/result_79.png new file mode 100644 index 0000000..f7e5417 Binary files /dev/null and b/resources/result_79.png differ diff --git a/resources/result_8.png b/resources/result_8.png new file mode 100644 index 0000000..184cf56 Binary files /dev/null and b/resources/result_8.png differ diff --git a/resources/result_80.png b/resources/result_80.png new file mode 100644 index 0000000..7955dd6 Binary files /dev/null and b/resources/result_80.png differ diff --git a/resources/result_81.png b/resources/result_81.png new file mode 100644 index 0000000..b9df73a Binary files /dev/null and b/resources/result_81.png differ diff --git a/resources/result_82.png b/resources/result_82.png new file mode 100644 index 0000000..532d9de Binary files /dev/null and b/resources/result_82.png differ diff --git a/resources/result_83.png b/resources/result_83.png new file mode 100644 index 0000000..9d6a4b9 Binary files /dev/null and b/resources/result_83.png differ diff --git a/resources/result_84.png b/resources/result_84.png new file mode 100644 index 0000000..4be489f Binary files /dev/null and b/resources/result_84.png differ diff --git a/resources/result_85.png b/resources/result_85.png new file mode 100644 index 0000000..ea49915 Binary files /dev/null and b/resources/result_85.png differ diff --git a/resources/result_86.png b/resources/result_86.png new file mode 100644 index 0000000..f68c8cf Binary files /dev/null and b/resources/result_86.png differ diff --git a/resources/result_87.png b/resources/result_87.png new file mode 100644 index 0000000..c149afe Binary files /dev/null and b/resources/result_87.png differ diff --git a/resources/result_88.png b/resources/result_88.png new file mode 100644 index 0000000..88b018f Binary files /dev/null and b/resources/result_88.png differ diff --git a/resources/result_89.png b/resources/result_89.png new file mode 100644 index 0000000..1480eaa Binary files /dev/null and b/resources/result_89.png differ diff --git a/resources/result_9.png b/resources/result_9.png new file mode 100644 index 0000000..0932a00 Binary files /dev/null and b/resources/result_9.png differ diff --git a/resources/result_90.png b/resources/result_90.png new file mode 100644 index 0000000..6bfeb2d Binary files /dev/null and b/resources/result_90.png differ diff --git a/resources/result_91.png b/resources/result_91.png new file mode 100644 index 0000000..19f8382 Binary files /dev/null and b/resources/result_91.png differ diff --git a/resources/result_92.png b/resources/result_92.png new file mode 100644 index 0000000..8f41d32 Binary files /dev/null and b/resources/result_92.png differ diff --git a/resources/result_93.png b/resources/result_93.png new file mode 100644 index 0000000..6bc5003 Binary files /dev/null and b/resources/result_93.png differ diff --git a/resources/result_94.png b/resources/result_94.png new file mode 100644 index 0000000..8dc941e Binary files /dev/null and b/resources/result_94.png differ diff --git a/resources/result_95.png b/resources/result_95.png new file mode 100644 index 0000000..82b45d6 Binary files /dev/null and b/resources/result_95.png differ diff --git a/resources/result_96.png b/resources/result_96.png new file mode 100644 index 0000000..0c1166c Binary files /dev/null and b/resources/result_96.png differ diff --git a/resources/result_97.png b/resources/result_97.png new file mode 100644 index 0000000..6fb887f Binary files /dev/null and b/resources/result_97.png differ diff --git a/resources/result_98.png b/resources/result_98.png new file mode 100644 index 0000000..bfe40b0 Binary files /dev/null and b/resources/result_98.png differ diff --git a/resources/result_99.png b/resources/result_99.png new file mode 100644 index 0000000..1c05102 Binary files /dev/null and b/resources/result_99.png differ diff --git a/src/decoder_slover.cpp b/src/decoder_slover.cpp new file mode 100644 index 0000000..12d50d4 --- /dev/null +++ b/src/decoder_slover.cpp @@ -0,0 +1,30 @@ +#include "decoder_slover.h" + +DecodeSlover::DecodeSlover() +{ + net.opt.use_vulkan_compute = false; + net.opt.use_winograd_convolution = false; + net.opt.use_sgemm_convolution = false; + net.opt.use_fp16_packed = true; + net.opt.use_fp16_storage = true; + net.opt.use_fp16_arithmetic = true; + net.opt.use_packing_layout = true; + net.load_param( "assets/AutoencoderKL-fp16.param" ); + net.load_model( "assets/AutoencoderKL-fp16.bin" ); +} + +ncnn::Mat DecodeSlover::decode( ncnn::Mat sample ) +{ + ncnn::Mat x_samples_ddim; + { + sample.substract_mean_normalize( 0, factor ); + { + ncnn::Extractor ex = net.create_extractor(); + ex.set_light_mode( true ); + ex.input( "input.1", sample ); + ex.extract( "815", x_samples_ddim ); + } + x_samples_ddim.substract_mean_normalize( _mean_, _norm_ ); + } + return x_samples_ddim; +} \ No newline at end of file diff --git a/src/decoder_slover.h b/src/decoder_slover.h new file mode 100644 index 0000000..ee63e94 --- /dev/null +++ b/src/decoder_slover.h @@ -0,0 +1,31 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +//#include +#include +#include +using namespace std; + +class DecodeSlover +{ +public: + DecodeSlover(); + + ncnn::Mat decode(ncnn::Mat sample); + +private: + + const float factor[4] = { 5.48998f, 5.48998f, 5.48998f, 5.48998f }; + + const float _mean_[3] = { -1.0f, -1.0f, -1.0f }; + const float _norm_[3] = { 127.5f, 127.5f, 127.5f }; + + ncnn::Net net; +}; \ No newline at end of file diff --git a/src/diffusion_slover.cpp b/src/diffusion_slover.cpp new file mode 100644 index 0000000..791cf38 --- /dev/null +++ b/src/diffusion_slover.cpp @@ -0,0 +1,164 @@ +#include "./diffusion_slover.h" +#include +#include +#include + +DiffusionSlover::DiffusionSlover() +{ + net.opt.use_vulkan_compute = false; + net.opt.use_winograd_convolution = false; + net.opt.use_sgemm_convolution = false; + net.opt.use_fp16_packed = true; + net.opt.use_fp16_storage = true; + net.opt.use_fp16_arithmetic = true; + net.opt.use_packing_layout = true; + net.load_param( "assets/UNetModel-fp16.param" ); + net.load_model( "assets/UNetModel-fp16.bin" ); + ifstream in( "assets/log_sigmas.bin", ios::in | ios::binary ); + in.read( ( char* )&log_sigmas, sizeof log_sigmas ); + in.close(); +} + +ncnn::Mat DiffusionSlover::randn_4_64_64( int seed ) +{ + std::vector arr; + { + std::mt19937 gen{ static_cast( seed ) }; + std::normal_distribution d{0.0f, 1.0f}; + arr.resize( 64 * 64 * 4 ); + std::for_each( arr.begin(), arr.end(), [&]( float & x ) + { + x = d( gen ); + } ); + } + ncnn::Mat x_mat( 64, 64, 4, reinterpret_cast( arr.data() ) ); + return x_mat.clone(); +} + +ncnn::Mat DiffusionSlover::CFGDenoiser_CompVisDenoiser( ncnn::Mat& input, float sigma, ncnn::Mat cond, ncnn::Mat uncond ) +{ + // get_scalings + float c_out = -1.0 * sigma; + float c_in = 1.0 / sqrt( sigma * sigma + 1 ); + // sigma_to_t + float log_sigma = log( sigma ); + vector dists( 1000 ); + + for ( int i = 0; i < 1000; i++ ) + { + if ( log_sigma - log_sigmas[i] >= 0 ) + dists[i] = 1; + + else + dists[i] = 0; + + if ( i == 0 ) continue; + + dists[i] += dists[i - 1]; + } + + int low_idx = min( int( max_element( dists.begin(), dists.end() ) - dists.begin() ), 1000 - 2 ); + int high_idx = low_idx + 1; + float low = log_sigmas[low_idx]; + float high = log_sigmas[high_idx]; + float w = ( low - log_sigma ) / ( low - high ); + w = max( 0.f, min( 1.f, w ) ); + float t = ( 1 - w ) * low_idx + w * high_idx; + ncnn::Mat t_mat( 1 ); + t_mat[0] = t; + ncnn::Mat c_in_mat( 1 ); + c_in_mat[0] = c_in; + ncnn::Mat c_out_mat( 1 ); + c_out_mat[0] = c_out; + ncnn::Mat denoised_cond; + { + ncnn::Extractor ex = net.create_extractor(); + ex.set_light_mode( true ); + ex.input( "in0", input ); + ex.input( "in1", t_mat ); + ex.input( "in2", cond ); + ex.input( "c_in", c_in_mat ); + ex.input( "c_out", c_out_mat ); + ex.extract( "outout", denoised_cond ); + } + ncnn::Mat denoised_uncond; + { + ncnn::Extractor ex = net.create_extractor(); + ex.set_light_mode( true ); + ex.input( "in0", input ); + ex.input( "in1", t_mat ); + ex.input( "in2", uncond ); + ex.input( "c_in", c_in_mat ); + ex.input( "c_out", c_out_mat ); + ex.extract( "outout", denoised_uncond ); + } + + for ( int c = 0; c < 4; c++ ) + { + float* u_ptr = denoised_uncond.channel( c ); + float* c_ptr = denoised_cond.channel( c ); + + for ( int hw = 0; hw < 64 * 64; hw++ ) + { + ( *u_ptr ) = ( *u_ptr ) + 7 * ( ( *c_ptr ) - ( *u_ptr ) ); + u_ptr++; + c_ptr++; + } + } + + return denoised_uncond; +} + +ncnn::Mat DiffusionSlover::sampler( int seed, int step, ncnn::Mat& c, ncnn::Mat& uc ) +{ + ncnn::Mat x_mat = randn_4_64_64( seed % 1000 ); + // t_to_sigma + vector sigma( step ); + float delta = 0.0 - 999.0 / ( step - 1 ); + + for ( int i = 0; i < step; i++ ) + { + float t = 999.0 + i * delta; + int low_idx = floor( t ); + int high_idx = ceil( t ); + float w = t - low_idx; + sigma[i] = exp( ( 1 - w ) * log_sigmas[low_idx] + w * log_sigmas[high_idx] ); + } + + sigma.push_back( 0.f ); + float _norm_[4] = { sigma[0], sigma[0], sigma[0], sigma[0] }; + x_mat.substract_mean_normalize( 0, _norm_ ); + // sample_euler_ancestral + { + for ( int i = 0; i < static_cast(sigma.size()) - 1; i++ ) + { + cout << "step:" << i << "\t\t"; + double t1 = ncnn::get_current_time(); + ncnn::Mat denoised = CFGDenoiser_CompVisDenoiser( x_mat, sigma[i], c, uc ); + double t2 = ncnn::get_current_time(); + cout << t2 - t1 << "ms" << endl; + float sigma_up = min( sigma[i + 1], sqrt( sigma[i + 1] * sigma[i + 1] * ( sigma[i] * sigma[i] - sigma[i + 1] * sigma[i + 1] ) / ( sigma[i] * sigma[i] ) ) ); + float sigma_down = sqrt( sigma[i + 1] * sigma[i + 1] - sigma_up * sigma_up ); + srand( time( NULL ) ); + ncnn::Mat randn = randn_4_64_64( rand() % 1000 ); + + for ( int c = 0; c < 4; c++ ) + { + float* x_ptr = x_mat.channel( c ); + float* d_ptr = denoised.channel( c ); + float* r_ptr = randn.channel( c ); + + for ( int hw = 0; hw < 64 * 64; hw++ ) + { + *x_ptr = *x_ptr + ( ( *x_ptr - *d_ptr ) / sigma[i] ) * ( sigma_down - sigma[i] ) + *r_ptr * sigma_up; + x_ptr++; + d_ptr++; + r_ptr++; + } + } + } + } + ncnn::Mat fuck_x; + fuck_x.clone_from( x_mat ); + return fuck_x; +} diff --git a/src/diffusion_slover.h b/src/diffusion_slover.h new file mode 100644 index 0000000..bbaa5c9 --- /dev/null +++ b/src/diffusion_slover.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +//#include +#include +#include +#include +#include +using namespace std; + +class DiffusionSlover +{ +public: + DiffusionSlover(); + + ncnn::Mat sampler(int seed, int step, ncnn::Mat& c, ncnn::Mat& uc); + +private: + ncnn::Mat randn_4_64_64(int seed); + ncnn::Mat CFGDenoiser_CompVisDenoiser(ncnn::Mat& input, float sigma, ncnn::Mat cond, ncnn::Mat uncond); + +private: + float log_sigmas[1000] = { 0 }; + + ncnn::Net net; +}; \ No newline at end of file diff --git a/src/esr4x.hpp b/src/esr4x.hpp new file mode 100644 index 0000000..cb0dc22 --- /dev/null +++ b/src/esr4x.hpp @@ -0,0 +1,46 @@ +#ifndef FNBXWWCFGKJITOKYXRMKNCJASNTJMUSUVRVIVADDNNHPXPCEVEYRYVBKQJBNABBDVUBDPUWTU +#define FNBXWWCFGKJITOKYXRMKNCJASNTJMUSUVRVIVADDNNHPXPCEVEYRYVBKQJBNABBDVUBDPUWTU + +#include + +struct Esr4x +{ + ncnn::Net net; + + Esr4x() + { + net.opt.use_vulkan_compute = false; + net.opt.use_winograd_convolution = false; + net.opt.use_sgemm_convolution = false; + net.opt.use_fp16_packed = false; + net.opt.use_fp16_storage = false; + net.opt.use_fp16_arithmetic = false; + net.opt.use_packing_layout = true; + net.load_param( "assets/RealESRGAN_x4plus_anime_6B.fp32-sim-sim-opt.param" ); + net.load_model( "assets/RealESRGAN_x4plus_anime_6B.fp32-sim-sim-opt.bin" ); + } + + ncnn::Mat inference( ncnn::Mat& input ) + { + ncnn::Extractor ex = net.create_extractor(); + ex.set_light_mode( true ); + { + constexpr float mean[] = {0.0f, 0.0f, 0.0f}; + constexpr float norm[] = {1.0f/255.0f, 1.0f/255.0f, 1.0f/255.0f}; + input.substract_mean_normalize( mean, norm ); + } + ex.input( "data", input ); + ncnn::Mat ans; + ex.extract( "output", ans ); + { + constexpr float mean[] = {0.0f, 0.0f, 0.0f}; + constexpr float norm[] = {255.0f, 255.0f, 255.0f}; + ans.substract_mean_normalize( mean, norm ); + } + return ans; + } + +}; // struct Esr4x + +#endif//FNBXWWCFGKJITOKYXRMKNCJASNTJMUSUVRVIVADDNNHPXPCEVEYRYVBKQJBNABBDVUBDPUWTU + diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..18d6627 --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,18 @@ +#include "stable_diffusion.h" +#include + +int main() +{ + std::string positive_prompt = "floating hair, portrait, ((loli)), ((one girl)), cute face, hidden hands, asymmetrical bangs, beautiful detailed eyes, eye shadow, hair ornament, ribbons, bowties, buttons, pleated skirt, (((masterpiece))), ((best quality)), colorful"; + + std::string negative_prompt = "sleeveless, amputeebad feetbad handsbad duplicate, bad anatomy, bad anatomy disfigured malformed mutated, bad feet, bad hands, bad legs, blurry, bodysuit, cloned face, cropped, deformed, disfigured, dismemberment, error, extra arms, extra digit, extra digits, extra feet, extra fingers, extra legs, extra limbs, fewer digits, full, fused fingers, futa, gross proportions, hermaphrodite, jpeg artifacts, latex, leather, long body, long neck, low quality, lowres, malformed hands, malformed limbs, missing arms, missing fingers, missing legs, morbid, mutated hands, mutated hands and fingers, mutilated, normal quality, ((nsfw)), out of frame, signature, text, text font ui, too many fingers, trannsexual, trans, ugly, username, watermark, worst quality, yaoi, wall, door, bed, desk, tail, butt_plug, zettai_ryouiki, thick_thighs, wide_hips, thick_thighs, (((skirt_lift))), undressing, naked, nude, tattoo, sofa, on_sofa, chair, on_chair,multiple breasts, (((mutated hands and fingers))), ((long body)), (((mutation, poorly drawn))) , black-white, bad anatomy, liquid body, liquid tongue, disfigured, malformed, mutated, anatomical nonsense, text font ui, error, malformed hands, long neck, blurred, lowers, lowres, bad anatomy, bad proportions, bad shadow, uncoordinated body, unnatural body, fused breasts, bad breasts, huge breasts, poorly drawn breasts, extra breasts, liquid breasts, heavy breasts, missing breasts, huge haunch, huge thighs, huge calf, bad hands, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, fused ears, bad ears, poorly drawn ears, extra ears, liquid ears, heavy ears, missing ears, fused animal ears, bad animal ears, poorly drawn animal ears, extra animal ears, liquid animal ears, heavy animal ears, missing animal ears, text, ui, error, missing fingers, missing limb, fused fingers, one hand with more than 5 fingers, one hand with less than 5 fingers, one hand with more than 5 digit, one hand with less than 5 digit, extra digit, fewer digits, fused digit, missing digit, bad digit, liquid digit, colorful tongue, black tongue, cropped, watermark, username, blurry, JPEG artifacts, signature, 3D, 3D game, 3D game scene, 3D character, malformed feet, extra feet, bad feet, poorly drawn feet, fused feet, missing feet, extra shoes, bad shoes, fused shoes, more than two shoes, poorly drawn shoes, bad gloves, poorly drawn gloves, fused gloves, bad cum, poorly drawn cum, fused cum, bad hairs, poorly drawn hairs, fused hairs, big muscles, ugly, bad face, fused face, poorly drawn face, cloned face, big face, long face, bad eyes, fused eyes poorly drawn eyes, extra eyes, malformed limbs, more than 2 nipples, missing nipples, different nipples, fused nipples, bad nipples, poorly drawn nipples, black nipples, colorful nipples, gross proportions. short arm, (((((missing arms))))), missing thighs, missing calf, missing legs, mutation, duplicate, morbid, mutilated, poorly drawn hands, more than 1 left hand, more than 1 right hand, deformed, (blurry), disfigured, missing legs, extra arms, extra thighs, more than 2 thighs, extra calf, fused calf, extra legs, bad knee, extra knee, more than 2 legs, bad tails, bad mouth, fused mouth, poorly drawn mouth, bad tongue, tongue within mouth, too long tongue, black tongue, big mouth, cracked mouth, bad mouth, dirty face, dirty teeth, dirty pantie, fused pantie, poorly drawn pantie, fused cloth, poorly drawn cloth, bad pantie, yellow teeth, thick lips, bad cameltoe, colorful cameltoe, bad asshole, poorly drawn asshole, fused asshole, missing asshole, bad anus, bad pussy, bad crotch, bad crotch seam, fused anus, fused pussy, fused anus, fused crotch, poorly drawn crotch, fused seam, poorly drawn anus, poorly drawn pussy, poorly drawn crotch, poorly drawn crotch seam, bad thigh gap, missing thigh gap, fused thigh gap, liquid thigh gap, poorly drawn thigh gap, poorly drawn anus, bad collarbone, fused collarbone, missing collarbone, liquid collarbone, strong girl, obesity, worst quality, low quality, normal quality, liquid tentacles, bad tentacles, poorly drawn tentacles, split tentacles, fused tentacles, missing clit, bad clit, fused clit, colorful clit, black clit, liquid clit, QR code, bar code, censored, safety panties, safety knickers, beard, furry ,pony, pubic hair, mosaic, excrement, faeces, shit, futa, testis, shoes, glasses, ((((eyewear))))"; + + for ( int seed = 1; seed < 100; ++seed ) + { + std::string output_png_path = std::string{"./result_"} + std::to_string( seed ) + std::string{".png"}; + sd( positive_prompt, output_png_path, negative_prompt, 30, seed ); + } + + return 0; +} + diff --git a/src/prompt_slover.cpp b/src/prompt_slover.cpp new file mode 100644 index 0000000..659b238 --- /dev/null +++ b/src/prompt_slover.cpp @@ -0,0 +1,304 @@ +#include "prompt_slover.h" + +PromptSlover::PromptSlover() +{ + // 加载CLIP模型 + net.opt.use_vulkan_compute = false; + net.opt.use_winograd_convolution = false; + net.opt.use_sgemm_convolution = false; + net.opt.use_fp16_packed = true; + net.opt.use_fp16_storage = true; + net.opt.use_fp16_arithmetic = true; + net.opt.use_packing_layout = true; + net.load_param( "assets/FrozenCLIPEmbedder-fp16.param" ); + net.load_model( "assets/FrozenCLIPEmbedder-fp16.bin" ); + // 读取tokenizer字典 + std::ifstream infile; + std::string pathname = "assets/vocab.txt"; + infile.open( pathname.data() ); + std::string s; + int idx = 0; + + while ( getline( infile, s ) ) + { + tokenizer_token2idx.insert( pair( s, idx ) ); + tokenizer_idx2token.insert( pair( idx, s ) ); + idx++; + } + + infile.close(); +} + +ncnn::Mat PromptSlover::get_conditioning( string& prompt ) +{ + // 重要度计算可以匹配“()”和“[]”，圆括号是加重要度，方括号是减重要度 + vector> parsed = parse_prompt_attention( prompt ); + // token转ids + vector> tokenized; + { + for ( auto p : parsed ) + { + vector tokens = split( p.first ); + vector ids; + + for ( string token : tokens ) + { + ids.push_back( tokenizer_token2idx[token] ); + } + + tokenized.push_back( ids ); + } + } + // 一些处理 + vector remade_tokens; + vector multipliers; + { + int last_comma = -1; + + for ( unsigned long it_tokenized = 0; it_tokenized < tokenized.size(); it_tokenized++ ) + { + vector tokens = tokenized[it_tokenized]; + float weight = parsed[it_tokenized].second; + unsigned long i = 0; + + while ( i < tokens.size() ) + { + int token = tokens[i]; + + if ( token == 267 ) + { + last_comma = remade_tokens.size(); + } + + else if ( ( max( int( remade_tokens.size() ), 1 ) % 75 == 0 ) && ( last_comma != -1 ) && ( remade_tokens.size() - last_comma <= 20 ) ) + { + last_comma += 1; + vector reloc_tokens( remade_tokens.begin() + last_comma, remade_tokens.end() ); + vector reloc_mults( multipliers.begin() + last_comma, multipliers.end() ); + vector _remade_tokens_( remade_tokens.begin(), remade_tokens.begin() + last_comma ); + remade_tokens = _remade_tokens_; + int length = remade_tokens.size(); + int rem = ceil( length / 75.0 ) * 75 - length; + vector tmp_token( rem, 49407 ); + remade_tokens.insert( remade_tokens.end(), tmp_token.begin(), tmp_token.end() ); + remade_tokens.insert( remade_tokens.end(), reloc_tokens.begin(), reloc_tokens.end() ); + vector _multipliers_( multipliers.begin(), multipliers.end() + last_comma ); + vector tmp_multipliers( rem, 1.0f ); + _multipliers_.insert( _multipliers_.end(), tmp_multipliers.begin(), tmp_multipliers.end() ); + _multipliers_.insert( _multipliers_.end(), reloc_mults.begin(), reloc_mults.end() ); + multipliers = _multipliers_; + } + + remade_tokens.push_back( token ); + multipliers.push_back( weight ); + i += 1; + } + } + + int prompt_target_length = ceil( max( int( remade_tokens.size() ), 1 ) / 75.0 ) * 75; + int tokens_to_add = prompt_target_length - remade_tokens.size(); + vector tmp_token( tokens_to_add, 49407 ); + remade_tokens.insert( remade_tokens.end(), tmp_token.begin(), tmp_token.end() ); + vector tmp_multipliers( tokens_to_add, 1.0f ); + multipliers.insert( multipliers.end(), tmp_multipliers.begin(), tmp_multipliers.end() ); + } + // 切分 + ncnn::Mat conds( 768, 0 ); + { + while ( remade_tokens.size() > 0 ) + { + vector rem_tokens( remade_tokens.begin() + 75, remade_tokens.end() ); + vector rem_multipliers( multipliers.begin() + 75, multipliers.end() ); + vector current_tokens; + vector current_multipliers; + + if ( remade_tokens.size() > 0 ) + { + current_tokens.insert( current_tokens.end(), remade_tokens.begin(), remade_tokens.begin() + 75 ); + current_multipliers.insert( current_multipliers.end(), multipliers.begin(), multipliers.begin() + 75 ); + } + + else + { + vector tmp_token( 75, 49407 ); + current_tokens.insert( current_tokens.end(), tmp_token.begin(), tmp_token.end() ); + vector tmp_multipliers( 75, 1.0f ); + current_multipliers.insert( current_multipliers.end(), tmp_multipliers.begin(), tmp_multipliers.end() ); + } + + { + ncnn::Mat token_mat = ncnn::Mat( 77 ); + token_mat.fill( int( 49406 ) ); + ncnn::Mat multiplier_mat = ncnn::Mat( 77 ); + multiplier_mat.fill( 1.0f ); + int* token_ptr = token_mat; + float* multiplier_ptr = multiplier_mat; + + for ( int i = 0; i < 75; i++ ) + { + token_ptr[i + 1] = int( current_tokens[i] ); + multiplier_ptr[i + 1] = current_multipliers[i]; + } + + ncnn::Extractor ex = net.create_extractor(); + ex.set_light_mode( true ); + ex.input( "token", token_mat ); + ex.input( "multiplier", multiplier_mat ); + ex.input( "cond", conds ); + ncnn::Mat new_conds; + ex.extract( "conds", new_conds ); + conds = new_conds; + } + + remade_tokens = rem_tokens; + multipliers = rem_multipliers; + } + } + return conds; +} + +vector> PromptSlover::parse_prompt_attention( string& texts ) +{ + vector> res; + stack round_brackets; + stack square_brackets; + const float round_bracket_multiplier = 1.1; + const float square_bracket_multiplier = 1 / 1.1; + vector ms; + + for ( char c : texts ) + { + string s = string( 1, c ); + + if ( s == "(" || s == "[" || s == ")" || s == "]" ) + { + ms.push_back( s ); + } + + else + { + if ( ms.size() < 1 ) + ms.push_back( "" ); + + string last = ms[ms.size() - 1]; + + if ( last == "(" || last == "[" || last == ")" || last == "]" ) + { + ms.push_back( "" ); + } + + ms[ms.size() - 1] += s; + } + } + + for ( string text : ms ) + { + if ( text == "(" ) + { + round_brackets.push( res.size() ); + } + + else if ( text == "[" ) + { + square_brackets.push( res.size() ); + } + + else if ( text == ")" && round_brackets.size() > 0 ) + { + for ( unsigned long p = round_brackets.top(); p < res.size(); p++ ) + { + res[p].second *= round_bracket_multiplier; + } + + round_brackets.pop(); + } + + else if ( text == "]" and square_brackets.size() > 0 ) + { + for ( unsigned long p = square_brackets.top(); p < res.size(); p++ ) + { + res[p].second *= square_bracket_multiplier; + } + + square_brackets.pop(); + } + + else + { + res.push_back( make_pair( text, 1.0 ) ); + } + } + + while ( !round_brackets.empty() ) + { + for ( unsigned long p = round_brackets.top(); p < res.size(); p++ ) + { + res[p].second *= round_bracket_multiplier; + } + + round_brackets.pop(); + } + + while ( !square_brackets.empty() ) + { + for ( unsigned long p = square_brackets.top(); p < res.size(); p++ ) + { + res[p].second *= square_bracket_multiplier; + } + + square_brackets.pop(); + } + + unsigned long i = 0; + + while ( i + 1 < res.size() ) + { + if ( res[i].second == res[i + 1].second ) + { + res[i].first += res[i + 1].first; + auto it = res.begin(); + res.erase( it + i + 1 ); + } + + else + { + i += 1; + } + } + + return res; +} + +string PromptSlover::whitespace_clean( string& text ) +{ + return regex_replace( text, regex( "\\s+" ), " " ); +} + +std::vector PromptSlover::split( std::string str ) +{ + std::string::size_type pos; + std::vector result; + str += " "; + int size = str.size(); + + for ( int i = 0; i < size; i++ ) + { + pos = min( str.find( " ", i ), str.find( ",", i ) ); + + if ( pos < str.size() ) + { + std::string s = str.substr( i, pos - i ); + string pat = string( 1, str[pos] ); + + if ( s.length() > 0 ) + result.push_back( s + "" ); + + if ( pat != " " ) + result.push_back( pat + "" ); + + i = pos; + } + } + + return result; +} \ No newline at end of file diff --git a/src/prompt_slover.h b/src/prompt_slover.h new file mode 100644 index 0000000..bf23b11 --- /dev/null +++ b/src/prompt_slover.h @@ -0,0 +1,29 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +class PromptSlover +{ +public: + PromptSlover(); + + ncnn::Mat get_conditioning(string& prompt); + +private: + std::vector split(std::string str); + string whitespace_clean(string& text); + vector> parse_prompt_attention(string& texts); + + map tokenizer_token2idx; + map tokenizer_idx2token; + + ncnn::Net net; +}; \ No newline at end of file diff --git a/src/save_png.hpp b/src/save_png.hpp new file mode 100644 index 0000000..f20fcea --- /dev/null +++ b/src/save_png.hpp @@ -0,0 +1,260 @@ +#ifndef OCOBWIPQFPFHWOCXYSRQMKIHKVBTJPNKGAHBYVGKCDYHIWWNHASRNQHEUNSVISJTNCIOGEBLK +#define OCOBWIPQFPFHWOCXYSRQMKIHKVBTJPNKGAHBYVGKCDYHIWWNHASRNQHEUNSVISJTNCIOGEBLK + +// adapted from https://github.com/miloyip/svpng/blob/master/svpng.inc +inline void save_png( std::uint8_t* img, unsigned w, unsigned h, int alpha, char const* const file_name ) noexcept +{ + constexpr unsigned t[] = { 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, 0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c }; + unsigned a = 1, b = 0, c, p = w * ( alpha ? 4 : 3 ) + 1, x, y, i; + FILE* fp = fopen( file_name, "wb" ); + + for ( i = 0; i < 8; i++ ) + fputc( ( "\x89PNG\r\n\32\n" )[i], fp );; + + { + { + fputc( ( 13 ) >> 24, fp ); + fputc( ( ( 13 ) >> 16 ) & 255, fp ); + fputc( ( ( 13 ) >> 8 ) & 255, fp ); + fputc( ( 13 ) & 255, fp ); + } + c = ~0U; + + for ( i = 0; i < 4; i++ ) + { + fputc( ( "IHDR" )[i], fp ); + c ^= ( ( "IHDR" )[i] ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + { + { + fputc( ( w ) >> 24, fp ); + c ^= ( ( w ) >> 24 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( w ) >> 16 ) & 255, fp ); + c ^= ( ( ( w ) >> 16 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( w ) >> 8 ) & 255, fp ); + c ^= ( ( ( w ) >> 8 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( w ) & 255, fp ); + c ^= ( ( w ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + { + { + fputc( ( h ) >> 24, fp ); + c ^= ( ( h ) >> 24 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( h ) >> 16 ) & 255, fp ); + c ^= ( ( ( h ) >> 16 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( h ) >> 8 ) & 255, fp ); + c ^= ( ( ( h ) >> 8 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( h ) & 255, fp ); + c ^= ( ( h ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + { + fputc( 8, fp ); + c ^= ( 8 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( alpha ? 6 : 2, fp ); + c ^= ( alpha ? 6 : 2 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + + for ( i = 0; i < 3; i++ ) + { + fputc( ( "\0\0\0" )[i], fp ); + c ^= ( ( "\0\0\0" )[i] ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + + { + fputc( ( ~c ) >> 24, fp ); + fputc( ( ( ~c ) >> 16 ) & 255, fp ); + fputc( ( ( ~c ) >> 8 ) & 255, fp ); + fputc( ( ~c ) & 255, fp ); + } + + { + { + fputc( ( 2 + h * ( 5 + p ) + 4 ) >> 24, fp ); + fputc( ( ( 2 + h * ( 5 + p ) + 4 ) >> 16 ) & 255, fp ); + fputc( ( ( 2 + h * ( 5 + p ) + 4 ) >> 8 ) & 255, fp ); + fputc( ( 2 + h * ( 5 + p ) + 4 ) & 255, fp ); + } + c = ~0U; + + for ( i = 0; i < 4; i++ ) + { + fputc( ( "IDAT" )[i], fp ); + c ^= ( ( "IDAT" )[i] ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + + for ( i = 0; i < 2; i++ ) + { + fputc( ( "\x78\1" )[i], fp ); + c ^= ( ( "\x78\1" )[i] ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + + for ( y = 0; y < h; y++ ) + { + { + fputc( y == h - 1, fp ); + c ^= ( y == h - 1 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + { + fputc( ( p ) & 255, fp ); + c ^= ( ( p ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( p ) >> 8 ) & 255, fp ); + c ^= ( ( ( p ) >> 8 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + { + { + fputc( ( ~p ) & 255, fp ); + c ^= ( ( ~p ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( ~p ) >> 8 ) & 255, fp ); + c ^= ( ( ( ~p ) >> 8 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + { + { + fputc( 0, fp ); + c ^= ( 0 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + a = ( a + ( 0 ) ) % 65521; + b = ( b + a ) % 65521; + } + + for ( x = 0; x < p - 1; x++, img++ ) + { + { + fputc( *img, fp ); + c ^= ( *img ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + a = ( a + ( *img ) ) % 65521; + b = ( b + a ) % 65521; + } + } + + { + { + fputc( ( ( b << 16 ) | a ) >> 24, fp ); + c ^= ( ( ( b << 16 ) | a ) >> 24 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( ( b << 16 ) | a ) >> 16 ) & 255, fp ); + c ^= ( ( ( ( b << 16 ) | a ) >> 16 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( ( b << 16 ) | a ) >> 8 ) & 255, fp ); + c ^= ( ( ( ( b << 16 ) | a ) >> 8 ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + { + fputc( ( ( b << 16 ) | a ) & 255, fp ); + c ^= ( ( ( b << 16 ) | a ) & 255 ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + + { + fputc( ( ~c ) >> 24, fp ); + fputc( ( ( ~c ) >> 16 ) & 255, fp ); + fputc( ( ( ~c ) >> 8 ) & 255, fp ); + fputc( ( ~c ) & 255, fp ); + } + + { + { + fputc( ( 0 ) >> 24, fp ); + fputc( ( ( 0 ) >> 16 ) & 255, fp ); + fputc( ( ( 0 ) >> 8 ) & 255, fp ); + fputc( ( 0 ) & 255, fp ); + } + c = ~0U; + + for ( i = 0; i < 4; i++ ) + { + fputc( ( "IEND" )[i], fp ); + c ^= ( ( "IEND" )[i] ); + c = ( c >> 4 ) ^ t[c & 15]; + c = ( c >> 4 ) ^ t[c & 15]; + } + } + + { + fputc( ( ~c ) >> 24, fp ); + fputc( ( ( ~c ) >> 16 ) & 255, fp ); + fputc( ( ( ~c ) >> 8 ) & 255, fp ); + fputc( ( ~c ) & 255, fp ); + } + + fclose( fp ); +} + +#endif//OCOBWIPQFPFHWOCXYSRQMKIHKVBTJPNKGAHBYVGKCDYHIWWNHASRNQHEUNSVISJTNCIOGEBLK + diff --git a/src/stable_diffusion.cpp b/src/stable_diffusion.cpp new file mode 100644 index 0000000..b699546 --- /dev/null +++ b/src/stable_diffusion.cpp @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "./prompt_slover.h" +#include "./decoder_slover.h" +#include "./diffusion_slover.h" +#include "./esr4x.hpp" +#include "./save_png.hpp" +#include +#include +#include +#include +#include +#include + + +void sd( std::string positive_prompt = std::string{}, std::string output_png_path = std::string{}, std::string negative_prompt = std::string{}, int step = 30, int seed = 42 ) +{ + if ( positive_prompt.empty() ) + positive_prompt = + "floating hair, portrait, ((loli)), ((one girl)), cute face, hidden hands, asymmetrical bangs, beautiful detailed eyes, eye shaw, hair ornament, ribbons, bowties, buttons, pleated skirt, (((masterpiece))), ((best quality)), colorful"; + + if ( negative_prompt.empty() ) + negative_prompt = + "sleeveless, amputeebad feetbad handsbad duplicate, bad anatomy, bad anatomy disfigured malformed mutated, bad feet, bad hands, bad legs, blurry, bodysuit, cloned face, cropped, deformed, disfigured, dismemberment, error, extra arms, extra digit, extra digits, extra feet, extra fingers, extra legs, extra limbs, fewer digits, full, fused fingers, futa, gross proportions, hermaphrodite, jpeg artifacts, latex, leather, long body, long neck, low quality, lowres, malformed hands, malformed limbs, missing arms, missing fingers, missing legs, morbid, mutated hands, mutated hands and fingers, mutilated, normal quality, ((nsfw)), out of frame, signature, text, text font ui, too many fingers, trannsexual, trans, ugly, username, watermark, worst quality, yaoi, wall, or, bed, desk, tail, butt_plug, zettai_ryouiki, thick_thighs, wide_hips, thick_thighs, (((skirt_lift))), undressing, naked, nude, tattoo, sofa, on_sofa, chair, on_chair,multiple breasts, (((mutated hands and fingers))), ((long body)), (((mutation, poorly drawn))) , black-white, bad anatomy, liquid body, liquid tongue, disfigured, malformed, mutated, anatomical nonsense, text font ui, error, malformed hands, long neck, blurred, lowers, lowres, bad anatomy, bad proportions, bad shaw, uncoordinated body, unnatural body, fused breasts, bad breasts, huge breasts, poorly drawn breasts, extra breasts, liquid breasts, heavy breasts, missing breasts, huge haunch, huge thighs, huge calf, bad hands, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, fused ears, bad ears, poorly drawn ears, extra ears, liquid ears, heavy ears, missing ears, fused animal ears, bad animal ears, poorly drawn animal ears, extra animal ears, liquid animal ears, heavy animal ears, missing animal ears, text, ui, error, missing fingers, missing limb, fused fingers, one hand with more than 5 fingers, one hand with less than 5 fingers, one hand with more than 5 digit, one hand with less than 5 digit, extra digit, fewer digits, fused digit, missing digit, bad digit, liquid digit, colorful tongue, black tongue, cropped, watermark, username, blurry, JPEG artifacts, signature, 3D, 3D game, 3D game scene, 3D character, malformed feet, extra feet, bad feet, poorly drawn feet, fused feet, missing feet, extra shoes, bad shoes, fused shoes, more than two shoes, poorly drawn shoes, bad gloves, poorly drawn gloves, fused gloves, bad cum, poorly drawn cum, fused cum, bad hairs, poorly drawn hairs, fused hairs, big muscles, ugly, bad face, fused face, poorly drawn face, cloned face, big face, long face, bad eyes, fused eyes poorly drawn eyes, extra eyes, malformed limbs, more than 2 nipples, missing nipples, different nipples, fused nipples, bad nipples, poorly drawn nipples, black nipples, colorful nipples, gross proportions. short arm, (((((missing arms))))), missing thighs, missing calf, missing legs, mutation, duplicate, morbid, mutilated, poorly drawn hands, more than 1 left hand, more than 1 right hand, deformed, (blurry), disfigured, missing legs, extra arms, extra thighs, more than 2 thighs, extra calf, fused calf, extra legs, bad knee, extra knee, more than 2 legs, bad tails, bad mouth, fused mouth, poorly drawn mouth, bad tongue, tongue within mouth, too long tongue, black tongue, big mouth, cracked mouth, bad mouth, dirty face, dirty teeth, dirty pantie, fused pantie, poorly drawn pantie, fused cloth, poorly drawn cloth, bad pantie, yellow teeth, thick lips, bad cameltoe, colorful cameltoe, bad asshole, poorly drawn asshole, fused asshole, missing asshole, bad anus, bad pussy, bad crotch, bad crotch seam, fused anus, fused pussy, fused anus, fused crotch, poorly drawn crotch, fused seam, poorly drawn anus, poorly drawn pussy, poorly drawn crotch, poorly drawn crotch seam, bad thigh gap, missing thigh gap, fused thigh gap, liquid thigh gap, poorly drawn thigh gap, poorly drawn anus, bad collarbone, fused collarbone, missing collarbone, liquid collarbone, strong girl, obesity, worst quality, low quality, normal quality, liquid tentacles, bad tentacles, poorly drawn tentacles, split tentacles, fused tentacles, missing clit, bad clit, fused clit, colorful clit, black clit, liquid clit, QR code, bar code, censored, safety panties, safety knickers, beard, furry ,pony, pubic hair, mosaic, excrement, faeces, shit, futa, testis, shoes, glasses, ((((eyewear))))"; + + if ( output_png_path.empty() ) + output_png_path = std::string{"./result_"} + std::to_string( step ) + std::string{"_"} + std::to_string( seed ) + std::string{".png"}; + std::cout << "----------------[start]------------------" << std::endl; + std::cout << "positive_prompt: " << positive_prompt << std::endl; + std::cout << "output_png_path: " << output_png_path << std::endl; + std::cout << "negative_prompt: " << negative_prompt << std::endl; + std::cout << "step: " << step << std::endl; + std::cout << "seed: " << seed << std::endl; + std::cout << "----------------[prompt]------------------" << std::endl; + ncnn::Mat cond; + ncnn::Mat uncond; + { + PromptSlover prompt_slover; + cond = prompt_slover.get_conditioning( positive_prompt ); + uncond = prompt_slover.get_conditioning( negative_prompt ); + } + std::cout << "----------------[diffusion]---------------" << std::endl; + ncnn::Mat sample; + { + DiffusionSlover diffusion_slover; + sample = diffusion_slover.sampler( seed, step, cond, uncond ); + } + std::cout << "----------------[decode]------------------" << std::endl; + ncnn::Mat x_samples_ddim; + { + DecodeSlover decode_slover; + x_samples_ddim = decode_slover.decode( sample ); + } + std::cout << "----------------[4x]--------------------" << std::endl; + { + Esr4x esr4x; + x_samples_ddim = esr4x.inference( x_samples_ddim ); + } + std::cout << "----------------[save]--------------------" << std::endl; + { + std::vector buffer; + //buffer.resize( 512 * 512 * 3 ); + buffer.resize( 4*512 * 4*512 * 3 ); + x_samples_ddim.to_pixels( buffer.data(), ncnn::Mat::PIXEL_RGB ); + save_png( buffer.data(), 4*512, 4*512, 0, output_png_path.c_str() ); + } + std::cout << "----------------[close]-------------------" << std::endl; +} + diff --git a/src/stable_diffusion.h b/src/stable_diffusion.h new file mode 100644 index 0000000..3fee3f5 --- /dev/null +++ b/src/stable_diffusion.h @@ -0,0 +1,10 @@ +#ifndef STABLE_DIFFUSION_H_INCLUDED_SOASFLKJASLKJSLKJSLKJAFLKJAFDS +#define STABLE_DIFFUSION_H_INCLUDED_SOASFLKJASLKJSLKJSLKJAFLKJAFDS + +#include + +void sd( std::string positive_prompt=std::string{}, std::string output_png_path=std::string{}, std::string negative_prompt=std::string{}, int step=30, int seed=42 ); + + +#endif//STABLE_DIFFUSION_H_INCLUDED_SOASFLKJASLKJSLKJSLKJAFLKJAFDS +