@@ -16386,8 +16386,8 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
1638616386 // original formula use_more_bits :
1638716387 // return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;
1638816388 // The intervals of 3 are replaced by a broad bump in the central layers.
16389- // In the case of a 32 layers model, layers 5-7 and layers 12-16 are always skipped.
16390- // In the case of a 40 layers model, layers 6-9 and layers 15-20 are always skipped.
16389+ // In the case of a 32 layers model, layers 5-7 and layers 12-16 are always skipped.
16390+ // In the case of a 40 layers model, layers 6-9 and layers 15-20 are always skipped.
1639116391 // difquant_half_tensors replaces it and keeps the broad 50% bump to the upper quant. Ex : 16/32
1639216392 auto difquant_half_tensors = [](int i_layer, int n_layers) -> bool {
1639316393 return i_layer <= n_layers/8 || i_layer > 6*n_layers/8 || (i_layer >= 2*n_layers/8 && i_layer < 3*n_layers/8);
@@ -16448,7 +16448,7 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
1644816448 else new_type = GGML_TYPE_IQ4_XS;
1644916449 }
1645016450 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_XL || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||
16451- ftype == LLAMA_FTYPE_MOSTLY_IQ2_S) {
16451+ ftype == LLAMA_FTYPE_MOSTLY_IQ2_S) {
1645216452 if (qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q6_K;
1645316453 else if (qs.model.hparams.n_head() <= 20) new_type = GGML_TYPE_IQ4_XS;
1645416454 else new_type = GGML_TYPE_Q4_K;
@@ -16478,7 +16478,7 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
1647816478 if (qs.model.hparams.n_expert >= 4) {
1647916479 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q2_K_L ||
1648016480 ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ||
16481- ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XL) {
16481+ ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XL) {
1648216482 new_type = GGML_TYPE_Q4_K;
1648316483 }
1648416484 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS ||
@@ -16611,20 +16611,20 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
1661116611 }
1661216612 else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_L) {
1661316613 if (qs.model.hparams.n_gqa() >= 2 || qs.model.hparams.n_expert >= 2) new_type = GGML_TYPE_Q4_K;
16614- else new_type = GGML_TYPE_Q3_K;
16614+ else new_type = GGML_TYPE_Q3_K;
1661516615 }
1661616616 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S) {
1661716617 if (qs.model.hparams.n_gqa() >= 2 || qs.model.hparams.n_expert >= 2) new_type = GGML_TYPE_Q4_K;
16618- else new_type = GGML_TYPE_Q3_K;
16618+ else new_type = GGML_TYPE_Q3_K;
1661916619 }
1662016620 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {
1662116621 if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 2) new_type = GGML_TYPE_Q5_K;
1662216622 else if (qs.model.hparams.n_gqa() >= 2) new_type = GGML_TYPE_Q4_K;
16623- else new_type = GGML_TYPE_Q3_K;
16623+ else new_type = GGML_TYPE_Q3_K;
1662416624 }
1662516625 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XL) {
1662616626 if (qs.model.hparams.n_gqa() >= 2 || qs.model.hparams.n_expert >= 2) new_type = GGML_TYPE_Q5_K;
16627- else new_type = GGML_TYPE_Q4_K;
16627+ else new_type = GGML_TYPE_Q4_K;
1662816628 }
1662916629 else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S ||
1663016630 ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) && (qs.model.hparams.n_gqa() >= 2 || qs.model.hparams.n_expert >= 2)) {
@@ -16722,7 +16722,7 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
1672216722 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S) {
1672316723 if (qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;
1672416724 else if (qs.model.hparams.n_gqa() >= 2 || qs.model.hparams.n_expert >= 2) new_type = GGML_TYPE_Q3_K;
16725- else new_type = GGML_TYPE_Q2_K;
16725+ else new_type = GGML_TYPE_Q2_K;
1672616726 }
1672716727 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XL) {
1672816728 if (qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;
0 commit comments