From 9899ebd41d1ceeba9f22f06d4b85038d250d039a Mon Sep 17 00:00:00 2001 From: RaymondWang0 Date: Sun, 3 Mar 2024 03:54:32 +0000 Subject: [PATCH] deploy: 6fb886674776ac5ac34f38fb4078f504ef23c9d0 --- classmatmul_1_1MatmulOperator-members.html | 17 +- classmatmul_1_1MatmulOperator.html | 9 + dir_6a23dd2f4b74efa88df1553988330e41.html | 2 + doxygen_crawl.html | 1 + files.html | 15 +- index.html | 12 +- interface_8h_source.html | 104 ++++++++++++ linear_8h_source.html | 187 +++++++++++---------- matmul_8h_source.html | 83 ++++----- profiler_8h_source.html | 2 +- structa8w4__thread__args-members.html | 7 +- structa8w4__thread__args.html | 9 + 12 files changed, 294 insertions(+), 154 deletions(-) create mode 100644 interface_8h_source.html diff --git a/classmatmul_1_1MatmulOperator-members.html b/classmatmul_1_1MatmulOperator-members.html index 9529af2e..0806caaf 100644 --- a/classmatmul_1_1MatmulOperator-members.html +++ b/classmatmul_1_1MatmulOperator-members.html @@ -81,7 +81,9 @@

This is the complete list of members for matmul::MatmulOperator, including all inherited members.

- + + + @@ -101,12 +103,13 @@ - - - - - - + + + + + + +
gemm_accelerator_int8_int4_fast_no_offset(struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
cblas_gemm_accelerator_no_offset(struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
gemm_accelerator_int8_int4_fast_no_offset(struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
gemm_accelerator_int8_int4_fast_no_offset_v2(struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
gemm_forward_cuda(const struct matmul_params *params, int split_k_iters) (defined in matmul::MatmulOperator)matmul::MatmulOperator
gemm_forward_cuda_8splits(const struct matmul_params *params, float16_t *split_8_buffer) (defined in matmul::MatmulOperator)matmul::MatmulOperator
gemm_forward_cuda_half(const struct matmul_params *params, int split_k_iters) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_accelerator_int8_int4_fast_no_offset(struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_accelerator_transposed_fastover_column(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_accelerator_transposed_fastover_column_bias(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_cuda(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_transposed(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_fp16_int4(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_int4(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_int4_with_offset(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_int8(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_accelerator_untransposed_fastover_column(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_cuda(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
mat_mul_transposed(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_fp16_int4(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_int4(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_int4_with_offset(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
naive_mat_mul_int8(const struct matmul_params *params) (defined in matmul::MatmulOperator)matmul::MatmulOperator
diff --git a/classmatmul_1_1MatmulOperator.html b/classmatmul_1_1MatmulOperator.html index 9b2a9522..13a4cc87 100644 --- a/classmatmul_1_1MatmulOperator.html +++ b/classmatmul_1_1MatmulOperator.html @@ -93,6 +93,9 @@ void mat_mul_accelerator_transposed_fastover_column_bias (const struct matmul_params *params)   + +void mat_mul_accelerator_untransposed_fastover_column (const struct matmul_params *params) +  void naive_mat_mul_int8 (const struct matmul_params *params)   @@ -135,6 +138,12 @@ void gemm_accelerator_int8_int4_fast_no_offset (struct matmul_params *params)   + +void gemm_accelerator_int8_int4_fast_no_offset_v2 (struct matmul_params *params) +  + +void cblas_gemm_accelerator_no_offset (struct matmul_params *params) +  void naive_mat_mul_int4 (const struct matmul_params *params)   diff --git a/dir_6a23dd2f4b74efa88df1553988330e41.html b/dir_6a23dd2f4b74efa88df1553988330e41.html index 2627ff29..91f22152 100644 --- a/dir_6a23dd2f4b74efa88df1553988330e41.html +++ b/dir_6a23dd2f4b74efa88df1553988330e41.html @@ -94,6 +94,8 @@    GPTBigCodeTokenizer.h   + interface.h +   LLaMATokenizer.h    model.h diff --git a/doxygen_crawl.html b/doxygen_crawl.html index c92e90e3..e6f34930 100644 --- a/doxygen_crawl.html +++ b/doxygen_crawl.html @@ -16,6 +16,7 @@ + diff --git a/files.html b/files.html index 92c0ebfc..0d81d241 100644 --- a/files.html +++ b/files.html @@ -138,13 +138,14 @@  common.h  Generate.h  GPTBigCodeTokenizer.h - LLaMATokenizer.h - model.h - operators.h - OPTTokenizer.h - profiler.h - stb_image.h - utils.h + interface.h + LLaMATokenizer.h + model.h + operators.h + OPTTokenizer.h + profiler.h + stb_image.h + utils.h diff --git a/index.html b/index.html index 5aa2db41..2757bd89 100644 --- a/index.html +++ b/index.html @@ -199,13 +199,13 @@

  • Run the shell script to set up the environment for speech-to-speech chatbot.
    cd llm
    ./voicechat_setup.sh
    • -
  • Start the speech-to-speech chat locally.
    ./chat -v # chat.exe -v on Windows
    +
  • Start the speech-to-speech chat locally.
    ./voicechat # chat.exe -v on Windows
  • If you encounter any issues or errors during setup, please explore here to follow the step-by-step guide to debug.

    • Deploy vision language model (VLM) chatbot with TinyChatEngine

    -

    TinyChatEngine supports not only LLM but also VLM. We introduce a sophisticated text/voice chatbot for VLM. Here, we provide very easy-to-follow instructions to deploy vision language model chatbot (VILA-7B) with TinyChatEngine.

    +

    TinyChatEngine supports not only LLM but also VLM. We introduce a sophisticated text/voice chatbot for VLM. Here, we provide easy-to-follow instructions to deploy vision language model chatbot (VILA-7B) with TinyChatEngine. We recommend using M1/M2 MacBooks for this VLM feature.

    • Follow the instructions above to setup the basic environment, i.e., Prerequisites and Step-by-step to Deploy LLaMA2-7B-chat with TinyChatEngine.
    • To demonstrate images in the terminal, please download and install the following toolkit.
        @@ -214,7 +214,9 @@

      • (For other OS) Please refer to here to get the appropriate terminal ready.

      • -
    • (Optional) To enable the speech-to-speech chatbot for VLM, please follow the instruction above to run the shell script to set up the environment.
      • +
    • (Optional) To enable the speech-to-speech chatbot for VLM, please follow the instruction above to run the shell script to set up the environment.
      cd llm
      +
      ./voicechat_setup.sh
      +
    • Download the quantized VILA-7B model from our model zoo.
      • On an x86 device (e.g., Intel/AMD laptop)
        python tools/download_model.py --model VILA_7B_awq_int4_CLIP_ViT-L --QM QM_x86
        • @@ -223,9 +225,9 @@

    • (For MacOS) Start the chatbot locally. Please use an appropriate terminal (e.g., iTerm2).
        -
      • Image/Text to text
        ./scripts/vila.sh ../assets/figures/vlm_demo/pedestrian.png
        +
      • Image/Text to text
        ./vila ../assets/figures/vlm_demo/pedestrian.png
        • -
      • Image/Speech to speech bash ./scripts/voice_vila.sh ../assets/figures/vlm_demo/pedestrian.png
          +
        • Image/Speech to speech bash ./voice_vila ../assets/figures/vlm_demo/pedestrian.png
          • There are several images under the path ../assets/figures/vlm_demo. Feel free to try different images with VILA on your device!
          • diff --git a/interface_8h_source.html b/interface_8h_source.html new file mode 100644 index 00000000..a6cdfc7d --- /dev/null +++ b/interface_8h_source.html @@ -0,0 +1,104 @@ + + + + + + + +TinyChatEngine: llm/include/interface.h Source File + + + + + + + + + + + +
          +
          + + + + + + +
          +
          TinyChatEngine +
          +
          +
          + + + + + + + + + +
          +
          + + +
          +
          +
          +
          +
          +
          Loading...
          +
          Searching...
          +
          No Matches
          +
          +
          +
          +
          + + +
          +
          +
          interface.h
          +
          +
          +
          1#ifndef INTERFACE_H
          +
          2#define INTERFACE_H
          +
          3
          +
          4void set_print_black();
          +
          5void set_print_red();
          +
          6void set_print_yellow();
          +
          7void set_print_bold_yellow();
          +
          8void set_print_blue();
          +
          9void set_print_white();
          +
          10void set_print_reset();
          +
          11
          +
          12#endif
          +
          + +
          +Generated by doxygen 1.10.0 +
          + + diff --git a/linear_8h_source.html b/linear_8h_source.html index c9710269..cd037f9b 100644 --- a/linear_8h_source.html +++ b/linear_8h_source.html @@ -215,99 +215,102 @@
          127#ifdef USE_INT8_INT4_PRODUCT
          128 static void initialize_memory(const int block_size);
          129#endif
          -
          130 Matrix3D<uint8_t> weight;
          -
          131 Matrix3D<float> scale, zero_point;
          -
          132 Matrix3D<float> offset;
          -
          133 Matrix3D<float> bias;
          -
          134 bool has_bias = false;
          -
          135#ifdef PACK_QK
          -
          136 struct pack_q4_tensor *packed_weights;
          -
          137#endif
          -
          138
          -
          139 private:
          -
          140 std::string profile_name = "Linear_FP_int4";
          -
          141};
          +
          130#ifdef QM_ARM
          +
          131 static void initialize_weight_memory();
          +
          132#endif
          +
          133 Matrix3D<uint8_t> weight;
          +
          134 Matrix3D<float> scale, zero_point;
          +
          135 Matrix3D<float> offset;
          +
          136 Matrix3D<float> bias;
          +
          137 bool has_bias = false;
          +
          138#ifdef PACK_QK
          +
          139 struct pack_q4_tensor *packed_weights;
          +
          140#endif
          +
          141
          +
          142 private:
          +
          143 std::string profile_name = "Linear_FP_int4";
          +
          144};
          -
          142
          -
          143#ifdef QM_CUDA
          -
          144#include <cuda.h>
          -
          145#include <cuda_fp16.h>
          -
          146#include <cuda_runtime.h>
          -
          147
          -
          148class Linear_FP16_int4_ref {
          -
          149 public:
          -
          150 Linear_FP16_int4_ref(Matrix3D<int> weight_, std::string weight_path) : weight(weight_) {
          -
          151 naive_float16_t *scale_ptr;
          -
          152 // naive_float16_t *offset_ptr; // TODO: Currently, we don't need offset
          -
          153 int *zero_point_ptr;
          -
          154 // length of int8_t weight = elements / 2
          -
          155 // length of scales/offset = elements / QK = weight / (QK/2)
          -
          156 // length of zero_point = 1
          -
          157 // assert((weight.m_dim_z * 8) % (QK) == 0);
          -
          158 allocate_aligned_memory_gpu(scale_ptr, (this->weight.length() * 8 * sizeof(naive_float16_t)) / QK);
          -
          159 // allocate_aligned_memory_gpu(offset_ptr, (this->weight.length() * 8 * sizeof(naive_float16_t)) / QK); //
          -
          160 // TODO: Currently, we don't need offset
          -
          161 allocate_aligned_memory_gpu(zero_point_ptr, (this->weight.length() * sizeof(int)) / QK);
          -
          162
          -
          163 int x = this->weight.m_dim_x, y = this->weight.m_dim_y, z = (this->weight.m_dim_z * 8) / QK;
          -
          164 scale = Matrix3D<naive_float16_t>(scale_ptr, x, y, z);
          -
          165 // offset = Matrix3D<naive_float16_t>(offset_ptr, x, y, z); // TODO: Currently, we don't need offset
          -
          166 zero_point = Matrix3D<int>(zero_point_ptr, x, y, z / 8);
          -
          167 weight.load((weight_path + "/weight_int4.bin").c_str());
          -
          168 // offset.load((weight_path + "/offset_int4.bin").c_str()); // TODO: Currently, we don't need offset
          -
          169 scale.load((weight_path + "/scaling_factor_int4.bin").c_str());
          -
          170 zero_point.load((weight_path + "/zero_point_int4.bin").c_str());
          -
          171 };
          -
          172 Linear_FP16_int4_ref(){};
          -
          173 void forward_ref(const Matrix3D<naive_float16_t> &x, Matrix3D<naive_float16_t> &output);
          -
          174 Matrix3D<int> weight;
          -
          175 Matrix3D<naive_float16_t> scale;
          -
          176 Matrix3D<naive_float16_t> offset; // TODO: Currently, we don't need offset
          -
          177 Matrix3D<int> zero_point;
          -
          178
          -
          179 private:
          -
          180 std::string profile_name = "Linear_FP16_int4_ref";
          -
          181};
          -
          182
          -
          183class Linear_half_int4 {
          -
          184 public:
          -
          185 Linear_half_int4(Matrix3D<int> weight_, std::string weight_path) : weight(weight_) {
          -
          186 int output_channel = this->weight.m_dim_y, input_channel = this->weight.m_dim_z * 8;
          -
          187
          -
          188 float16_t *scale_ptr;
          -
          189 // float16_t *offset_ptr; // TODO: Currently, we don't need offset
          -
          190 int *zero_point_ptr;
          -
          191 // length of int8_t weight = elements / 2
          -
          192 // length of scales/offset = elements / QK = weight / (QK/2)
          -
          193 // length of zero_point = 1
          -
          194 // assert((weight.m_dim_z * 8) % (QK) == 0);
          -
          195 allocate_aligned_memory_gpu(scale_ptr, output_channel * calculate_zeros_width(input_channel, QK) * 8 * sizeof(float16_t));
          -
          196 // allocate_aligned_memory(offset_ptr, (this->weight.length() * 8 * sizeof(float16_t)) / QK); // TODO: Currently, we don't need offset
          -
          197 // Currently, we don't need offset
          -
          198 allocate_aligned_memory_gpu(zero_point_ptr, output_channel * calculate_zeros_width(input_channel, QK) * sizeof(int));
          -
          199
          -
          200 scale = Matrix3D<float16_t>(scale_ptr, 1, output_channel, calculate_zeros_width(input_channel, QK) * 8);
          -
          201 // offset = Matrix3D<float16_t>(offset_ptr, x, y, z); // TODO: Currently, we don't need offset
          -
          202 zero_point = Matrix3D<int>(zero_point_ptr, 1, output_channel, calculate_zeros_width(input_channel, QK));
          -
          203 weight.load((weight_path + "/weight_int4.bin").c_str());
          -
          204 // offset.load((weight_path + "/offset_int4.bin").c_str()); // TODO: Currently, we don't need offset
          -
          205 scale.load((weight_path + "/scaling_factor_int4.bin").c_str());
          -
          206 zero_point.load((weight_path + "/zero_point_int4.bin").c_str());
          -
          207 };
          -
          208 Linear_half_int4(){};
          -
          209 // void forward(const Matrix3D<float16_t> &x, Matrix3D<float16_t> &output);
          -
          210 void forward(const Matrix3D<float16_t> &x, Matrix3D<float16_t> &output);
          -
          211 Matrix3D<int> weight;
          -
          212 Matrix3D<float16_t> scale;
          -
          213 Matrix3D<float16_t> offset; // TODO: Currently, we don't need offset
          -
          214 Matrix3D<int> zero_point;
          -
          215
          -
          216 private:
          -
          217 std::string profile_name = "Linear_half_int4";
          -
          218};
          -
          219#endif
          -
          220
          -
          221#endif
          +
          145
          +
          146#ifdef QM_CUDA
          +
          147#include <cuda.h>
          +
          148#include <cuda_fp16.h>
          +
          149#include <cuda_runtime.h>
          +
          150
          +
          151class Linear_FP16_int4_ref {
          +
          152 public:
          +
          153 Linear_FP16_int4_ref(Matrix3D<int> weight_, std::string weight_path) : weight(weight_) {
          +
          154 naive_float16_t *scale_ptr;
          +
          155 // naive_float16_t *offset_ptr; // TODO: Currently, we don't need offset
          +
          156 int *zero_point_ptr;
          +
          157 // length of int8_t weight = elements / 2
          +
          158 // length of scales/offset = elements / QK = weight / (QK/2)
          +
          159 // length of zero_point = 1
          +
          160 // assert((weight.m_dim_z * 8) % (QK) == 0);
          +
          161 allocate_aligned_memory_gpu(scale_ptr, (this->weight.length() * 8 * sizeof(naive_float16_t)) / QK);
          +
          162 // allocate_aligned_memory_gpu(offset_ptr, (this->weight.length() * 8 * sizeof(naive_float16_t)) / QK); //
          +
          163 // TODO: Currently, we don't need offset
          +
          164 allocate_aligned_memory_gpu(zero_point_ptr, (this->weight.length() * sizeof(int)) / QK);
          +
          165
          +
          166 int x = this->weight.m_dim_x, y = this->weight.m_dim_y, z = (this->weight.m_dim_z * 8) / QK;
          +
          167 scale = Matrix3D<naive_float16_t>(scale_ptr, x, y, z);
          +
          168 // offset = Matrix3D<naive_float16_t>(offset_ptr, x, y, z); // TODO: Currently, we don't need offset
          +
          169 zero_point = Matrix3D<int>(zero_point_ptr, x, y, z / 8);
          +
          170 weight.load((weight_path + "/weight_int4.bin").c_str());
          +
          171 // offset.load((weight_path + "/offset_int4.bin").c_str()); // TODO: Currently, we don't need offset
          +
          172 scale.load((weight_path + "/scaling_factor_int4.bin").c_str());
          +
          173 zero_point.load((weight_path + "/zero_point_int4.bin").c_str());
          +
          174 };
          +
          175 Linear_FP16_int4_ref(){};
          +
          176 void forward_ref(const Matrix3D<naive_float16_t> &x, Matrix3D<naive_float16_t> &output);
          +
          177 Matrix3D<int> weight;
          +
          178 Matrix3D<naive_float16_t> scale;
          +
          179 Matrix3D<naive_float16_t> offset; // TODO: Currently, we don't need offset
          +
          180 Matrix3D<int> zero_point;
          +
          181
          +
          182 private:
          +
          183 std::string profile_name = "Linear_FP16_int4_ref";
          +
          184};
          +
          185
          +
          186class Linear_half_int4 {
          +
          187 public:
          +
          188 Linear_half_int4(Matrix3D<int> weight_, std::string weight_path) : weight(weight_) {
          +
          189 int output_channel = this->weight.m_dim_y, input_channel = this->weight.m_dim_z * 8;
          +
          190
          +
          191 float16_t *scale_ptr;
          +
          192 // float16_t *offset_ptr; // TODO: Currently, we don't need offset
          +
          193 int *zero_point_ptr;
          +
          194 // length of int8_t weight = elements / 2
          +
          195 // length of scales/offset = elements / QK = weight / (QK/2)
          +
          196 // length of zero_point = 1
          +
          197 // assert((weight.m_dim_z * 8) % (QK) == 0);
          +
          198 allocate_aligned_memory_gpu(scale_ptr, output_channel * calculate_zeros_width(input_channel, QK) * 8 * sizeof(float16_t));
          +
          199 // allocate_aligned_memory(offset_ptr, (this->weight.length() * 8 * sizeof(float16_t)) / QK); // TODO: Currently, we don't need offset
          +
          200 // Currently, we don't need offset
          +
          201 allocate_aligned_memory_gpu(zero_point_ptr, output_channel * calculate_zeros_width(input_channel, QK) * sizeof(int));
          +
          202
          +
          203 scale = Matrix3D<float16_t>(scale_ptr, 1, output_channel, calculate_zeros_width(input_channel, QK) * 8);
          +
          204 // offset = Matrix3D<float16_t>(offset_ptr, x, y, z); // TODO: Currently, we don't need offset
          +
          205 zero_point = Matrix3D<int>(zero_point_ptr, 1, output_channel, calculate_zeros_width(input_channel, QK));
          +
          206 weight.load((weight_path + "/weight_int4.bin").c_str());
          +
          207 // offset.load((weight_path + "/offset_int4.bin").c_str()); // TODO: Currently, we don't need offset
          +
          208 scale.load((weight_path + "/scaling_factor_int4.bin").c_str());
          +
          209 zero_point.load((weight_path + "/zero_point_int4.bin").c_str());
          +
          210 };
          +
          211 Linear_half_int4(){};
          +
          212 // void forward(const Matrix3D<float16_t> &x, Matrix3D<float16_t> &output);
          +
          213 void forward(const Matrix3D<float16_t> &x, Matrix3D<float16_t> &output);
          +
          214 Matrix3D<int> weight;
          +
          215 Matrix3D<float16_t> scale;
          +
          216 Matrix3D<float16_t> offset; // TODO: Currently, we don't need offset
          +
          217 Matrix3D<int> zero_point;
          +
          218
          +
          219 private:
          +
          220 std::string profile_name = "Linear_half_int4";
          +
          221};
          +
          222#endif
          +
          223
          +
          224#endif
          Linear_FP_int4
          Definition linear.h:27
          Linear_FP
          Definition linear.h:6
          Matrix3D
          Definition common.h:34
          diff --git a/matmul_8h_source.html b/matmul_8h_source.html index 66736672..f90fb51e 100644 --- a/matmul_8h_source.html +++ b/matmul_8h_source.html @@ -194,9 +194,9 @@
          100};
          101
          -
          102
          -
          103#define MAX(A, B) ((A) > (B) ? (A) : (B))
          -
          104#define MIN(A, B) ((A) < (B) ? (A) : (B))
          +
          102#define MAX(A, B) ((A) > (B) ? (A) : (B))
          +
          103#define MIN(A, B) ((A) < (B) ? (A) : (B))
          +
          104
          105namespace matmul {
          106class MatmulOperator {
          @@ -204,44 +204,47 @@
          108 void mat_mul_transposed(const struct matmul_params *params);
          109 void mat_mul_accelerator_transposed_fastover_column(const struct matmul_params *params);
          110 void mat_mul_accelerator_transposed_fastover_column_bias(const struct matmul_params *params);
          -
          111 // int8
          -
          112 void naive_mat_mul_int8(const struct matmul_params *params);
          -
          113 void mat_mul_accelerator_int8_fast_32unroll_over_column(const struct matmul_params *params);
          -
          114 void mat_mul_accelerator_int8_fast_2x2_32unroll(const struct matmul_params *params);
          -
          115 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias(const struct matmul_params *params);
          -
          116 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias_batch(const struct matmul_params *params);
          -
          117 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias_ofp32(const struct matmul_params *params);
          -
          118 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias_ofp32_batch(const struct matmul_params *params);
          -
          119 void mat_mul_accelerator_int8_fast_2x2_32unroll_bfp32_ofp32(const struct matmul_params *params);
          -
          120 void mat_mul_accelerator_int8_fast_2x2_32unroll_bfp32_ofp32_over_column(const struct matmul_params *params);
          -
          121 // void mat_mul_accelerator_int8_fast_2x2_omp(const struct matmul_params *params);
          -
          122 // int4
          -
          123 void mat_mul_accelerator_int4_fast(const struct matmul_params *params);
          -
          124 void mat_mul_accelerator_int4_fast_no_offset(const struct matmul_params *params);
          -
          125 void mat_mul_accelerator_int8_int4_fast_no_offset(struct matmul_params *params);
          -
          126 void gemv_accelerator_int8_int4_fast_no_offset(struct matmul_params *params);
          -
          127 void gemm_accelerator_int8_int4_fast_no_offset(struct matmul_params *params);
          -
          128 void naive_mat_mul_int4(const struct matmul_params *params);
          -
          129 void naive_mat_mul_int4_with_offset(const struct matmul_params *params);
          -
          130 // cuda
          -
          131 void naive_mat_mul_fp16_int4(const struct matmul_params *params);
          -
          132 // void naive_mat_mul_fp16_int4_gemv(const struct matmul_params *params);
          -
          133 void mat_mul_cuda(const struct matmul_params *params);
          -
          135 void gemm_forward_cuda(const struct matmul_params *params, int split_k_iters);
          -
          136 void gemm_forward_cuda_8splits(const struct matmul_params *params, float16_t *split_8_buffer);
          -
          137 void gemm_forward_cuda_half(const struct matmul_params *params, int split_k_iters);
          -
          138 void gemm_forward_cuda_half_test(const struct matmul_params *params, int split_k_iters);
          -
          140 void gemv_forward_cuda(const struct matmul_params *params);
          -
          141
          -
          142 private:
          -
          143 float interval_to_us(struct timeval *start, struct timeval *end);
          -
          144 void CHECK_MATRICES(const struct matrix *A, const struct matrix *B, const struct matrix *C);
          -
          145 void CHECK_MATRICES_int4weight(const struct matrix *A, const struct matrix *B, const struct matrix *C);
          -
          146};
          +
          111 void mat_mul_accelerator_untransposed_fastover_column(const struct matmul_params *params);
          +
          112 // int8
          +
          113 void naive_mat_mul_int8(const struct matmul_params *params);
          +
          114 void mat_mul_accelerator_int8_fast_32unroll_over_column(const struct matmul_params *params);
          +
          115 void mat_mul_accelerator_int8_fast_2x2_32unroll(const struct matmul_params *params);
          +
          116 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias(const struct matmul_params *params);
          +
          117 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias_batch(const struct matmul_params *params);
          +
          118 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias_ofp32(const struct matmul_params *params);
          +
          119 void mat_mul_accelerator_int8_fast_2x2_32unroll_nobias_ofp32_batch(const struct matmul_params *params);
          +
          120 void mat_mul_accelerator_int8_fast_2x2_32unroll_bfp32_ofp32(const struct matmul_params *params);
          +
          121 void mat_mul_accelerator_int8_fast_2x2_32unroll_bfp32_ofp32_over_column(const struct matmul_params *params);
          +
          122 // void mat_mul_accelerator_int8_fast_2x2_omp(const struct matmul_params *params);
          +
          123 // int4
          +
          124 void mat_mul_accelerator_int4_fast(const struct matmul_params *params);
          +
          125 void mat_mul_accelerator_int4_fast_no_offset(const struct matmul_params *params);
          +
          126 void mat_mul_accelerator_int8_int4_fast_no_offset(struct matmul_params *params);
          +
          127 void gemv_accelerator_int8_int4_fast_no_offset(struct matmul_params *params);
          +
          128 void gemm_accelerator_int8_int4_fast_no_offset(struct matmul_params *params);
          +
          129 void gemm_accelerator_int8_int4_fast_no_offset_v2(struct matmul_params *params);
          +
          130 void cblas_gemm_accelerator_no_offset(struct matmul_params *params);
          +
          131 void naive_mat_mul_int4(const struct matmul_params *params);
          +
          132 void naive_mat_mul_int4_with_offset(const struct matmul_params *params);
          +
          133 // cuda
          +
          134 void naive_mat_mul_fp16_int4(const struct matmul_params *params);
          +
          135 // void naive_mat_mul_fp16_int4_gemv(const struct matmul_params *params);
          +
          136 void mat_mul_cuda(const struct matmul_params *params);
          +
          138 void gemm_forward_cuda(const struct matmul_params *params, int split_k_iters);
          +
          139 void gemm_forward_cuda_8splits(const struct matmul_params *params, float16_t *split_8_buffer);
          +
          140 void gemm_forward_cuda_half(const struct matmul_params *params, int split_k_iters);
          +
          141 void gemm_forward_cuda_half_test(const struct matmul_params *params, int split_k_iters);
          +
          143 void gemv_forward_cuda(const struct matmul_params *params);
          +
          144
          +
          145 private:
          +
          146 float interval_to_us(struct timeval *start, struct timeval *end);
          +
          147 void CHECK_MATRICES(const struct matrix *A, const struct matrix *B, const struct matrix *C);
          +
          148 void CHECK_MATRICES_int4weight(const struct matrix *A, const struct matrix *B, const struct matrix *C);
          +
          149};
          -
          147} // namespace matmul
          -
          148
          -
          149#endif
          +
          150} // namespace matmul
          +
          151
          +
          152#endif
          matmul::MatmulOperator
          Definition matmul.h:106
          matmul_params
          Definition matmul.h:78
          matrix
          Definition matmul.h:59
          diff --git a/profiler_8h_source.html b/profiler_8h_source.html index 5c838f54..39b555d4 100644 --- a/profiler_8h_source.html +++ b/profiler_8h_source.html @@ -128,7 +128,7 @@
          42 std::cout << entry.first + ", ";
          43 float s = (float)(entry.second) / 1000000;
          44 float ts = (float)counts.at(entry.first);
          -
          45 printf("Total time: %.1f s, %.1f ms/token, %.1f token/s, %d tokens\n", s, s / ts * 1000, ts / s,
          +
          45 printf("Total time: %.1f s, %.1f ms/token, %.1f token/s, %d tokens\n\n", s, s / ts * 1000, ts / s,
          46 counts.at(entry.first));
          47 }
          48 } else {
          diff --git a/structa8w4__thread__args-members.html b/structa8w4__thread__args-members.html index 7b2155c9..fce625b4 100644 --- a/structa8w4__thread__args-members.html +++ b/structa8w4__thread__args-members.html @@ -77,9 +77,12 @@

          This is the complete list of members for a8w4_thread_args, including all inherited members.

          - - + + + + +
          end_j (defined in a8w4_thread_args)a8w4_thread_args
          params (defined in a8w4_thread_args)a8w4_thread_args
          end_i (defined in a8w4_thread_args)a8w4_thread_args
          end_j (defined in a8w4_thread_args)a8w4_thread_args
          params (defined in a8w4_thread_args)a8w4_thread_args
          start_i (defined in a8w4_thread_args)a8w4_thread_args
          start_j (defined in a8w4_thread_args)a8w4_thread_args
          tile_size (defined in a8w4_thread_args)a8w4_thread_args
          diff --git a/structa8w4__thread__args.html b/structa8w4__thread__args.html index dc95b636..3f1e6a31 100644 --- a/structa8w4__thread__args.html +++ b/structa8w4__thread__args.html @@ -80,12 +80,21 @@ + + + + + +

          Public Attributes

          +int start_i
           
          +int end_i
           
          int start_j
           
          int end_j
           
          +int tile_size
           
          const struct matmul_params * params