[CPU][fix] fix matmul decompress test case for migration v3.8 (#1)

* fix matmul decompress test case

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* save tmp

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* [FORK][FIX] IP weights compression: scalar scale
[FORK][FEATURE] InnerProduct primitive: squashed weight decompression

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* [FORK][FIX] IP weights compression: max bcast blocking computation
[FORK][FEATURE] InnerProduct primitive: squashed weight decompression

* fix compile issue

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* fix crash issue

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* try to fix compare issue

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* contiue fix some accrucy issue

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* fix f4_e2m1

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* continue to fix f4e2m1

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* fix confict on smoke_FC_(2|3)D_I8_sparse

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* clean debug and unused code

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

* revert this change, should affect test case

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

---------

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>
Co-authored-by: dmitrygo <dmitry.gorokhov@intel.com>

[FORK][FIX] add missing override

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

fix compilation error

move quant functions to hpp file

[FORK][Fix] Fix condition compilation

Signed-off-by: HU Yuan2 <yuan2.hu@intel.com>

Author: tiger100256-hu

src/common/memory_desc_wrapper.hpp

@@ -393,8 +393,7 @@ struct memory_desc_wrapper : public c_compatible {
         if (utils::one_of(format_kind(), format_kind::undef, format_kind::any))
             return false;
         if (has_runtime_dims_or_strides() || has_broadcast()) return false;
-        return nelems(with_padding) * data_type_size()
-                / sub_byte_data_type_multiplier()
+        return utils::div_up(nelems(with_padding)* data_type_size(), sub_byte_data_type_multiplier())
                 == size(0, /* include_additional_size = */ false);
     }
 
@@ -714,7 +713,7 @@ inline bool memory_desc_wrapper::similar_to(const memory_desc_wrapper &rhs,
                                     rhs.padded_dims() + ds, ndims() - ds)
                             && custom_cpm(padded_offsets() + ds,
                                     rhs.padded_offsets() + ds, ndims() - ds))
-            && IMPLICATION(check_off0, (offset0() == DNNL_RUNTIME_DIM_VAL || rhs.offset0() ==DNNL_RUNTIME_DIM_VAL || offset0() == rhs.offset0()));           
+            && IMPLICATION(check_off0, (offset0() == DNNL_RUNTIME_DIM_VAL || rhs.offset0() ==DNNL_RUNTIME_DIM_VAL || offset0() == rhs.offset0()));
 }
 
 inline bool memory_desc_wrapper::consistent_with(

src/common/primitive_attr_quant.cpp

@@ -55,14 +55,22 @@ quant_entry_t quant_entry_t::deserialize(deserializer_t &d) {
 
 std::string quant_entry_t::get_verbose() const {
     std::string s;
-    s.append(std::to_string(mask_));
-    s.append(":").append(dnnl_dt2str(data_type_));
+    s.append(std::to_string(get_mask()));
+    s.append(":").append(dnnl_dt2str(get_data_type()));
+    s.append(":").append(std::to_string(type_));
+    s.append(":");
     if (group_ndims_ > 0) {
-        s.append(":")
-                .append(std::to_string(group_dims_[0]))
+                s.append(std::to_string(group_dims_[0]))
                 .append("x")
                 .append(std::to_string(group_dims_[1]));
     }
+    s.append(":");
+    if (get_ndims() > 0) {
+            s.append(std::to_string(get_dims()[0]))
+            .append("x")
+            .append(std::to_string(get_dims()[1]));
+    }
+
     return s;
 }
 

src/common/primitive_attr_quant.hpp

@@ -53,6 +53,8 @@ struct quant_entry_t : public c_compatible {
     }
     status_t set(int mask, data_type_t data_type, int group_ndims,
             const dims_t group_dims) {
+        type_ = type_ | DNNL;
+        is_set_ = true;
         mask_ = mask;
         data_type_ = data_type;
         group_ndims_ = group_ndims;
@@ -61,25 +63,98 @@ struct quant_entry_t : public c_compatible {
         }
         return status::success;
     }
+    status_t set_scales(const dims_t dims, int ndims, data_type_t data_type = data_type::f32, int mask = 1) {
+        type_ = type_ | OV_SCALES;
+        is_set_scale = true;
+        ndims_scale = ndims;
+        mask_scale = mask;
+        data_type_scale = data_type;
+        if (ndims_scale > 0) {
+            utils::array_copy(dims_scale, dims, ndims_scale);
+        }
+        return status::success;
+    }
+    status_t set_zero_points(const dims_t dims, int ndims, data_type_t data_type) {
+        type_ = type_ | OV_ZERO_POINTS;
+        is_set_wei = true;
+        ndims_wei = ndims;
+        mask_wei = 1;
+        if (ndims_wei > 0) {
+            utils::array_copy(dims_wei, dims, ndims_wei);
+        }
+        data_type_wei = data_type;
+        return status::success;
+    }
+    status_t set_zero_points(const dims_t dims, int ndims, data_type_t data_type, int mask) {
+        type_ = type_ | DNNL;
+        is_set_wei = true;
+        ndims_wei = ndims;
+        mask_wei = mask;
+        if (ndims_wei > 0) {
+            utils::array_copy(dims_wei, dims, ndims_wei);
+            group_ndims_ = ndims;
+            utils::array_copy(group_dims_, dims, group_ndims_);
+        }
+        data_type_wei = data_type;
+        return status::success;
+    }
     status_t set(const quant_entry_t &other) {
-        return set(other.mask_, other.data_type_, other.group_ndims_,
-                other.group_dims_);
+        type_ = other.type_;
+        is_set_ = other.is_set_;
+        mask_ = other.mask_;
+        data_type_ = other.data_type_;
+        group_ndims_ = other.group_ndims_;
+        if(group_ndims_ > 0)
+            utils::array_copy(group_dims_, other.group_dims_, group_ndims_);
+        is_set_scale = other.is_set_scale;
+        mask_scale = other.mask_scale;
+        data_type_scale = other.data_type_scale;
+        ndims_scale = other.ndims_scale;
+        if (ndims_scale > 0)
+            utils::array_cmp(dims_scale, other.dims_scale, ndims_scale);
+        is_set_wei = other.is_set_wei;
+        mask_wei = other.mask_wei;
+        data_type_wei = other.data_type_wei;
+        ndims_wei = other.ndims_wei;
+        if(ndims_wei > 0)
+            utils::array_cmp(dims_wei, other.dims_wei, ndims_wei);
+        return status::success;
     }
-
     quant_entry_t &operator=(const quant_entry_t &rhs) {
         auto st = this->set(rhs);
         assert(st == status::success);
         UNUSED(st);
         return *this;
     }
-
     bool has_default_values() const { return *this == default_quant_entry(); }
     bool has_default_groups() const {
         return this->group_ndims_ == default_quant_entry().group_ndims_;
     }
-
-    int get_mask() const { return mask_; }
-    data_type_t get_data_type() const { return data_type_; }
+    int get_mask() const {
+        if (is_set_wei) return mask_wei;
+        if (is_set_) return mask_;
+        if (is_set_scale) return mask_scale;
+        return INT_MIN;
+    }
+    data_type_t get_data_type() const {
+        if (is_set_wei) return data_type_wei;
+        if (is_set_) return data_type_;
+        if (is_set_scale) return data_type_scale;
+        return data_type::undef;
+    }
+    const dims_t& get_dims() const {
+        if (is_set_wei) return dims_wei;
+        if (is_set_) return group_dims_;
+        if (is_set_scale) return dims_scale;
+        static const dims_t result = {};
+        return result;
+    }
+    int get_ndims() const {
+        if (is_set_wei) return ndims_wei;
+        if (is_set_) return group_ndims_;
+        if (is_set_scale) return ndims_scale;
+        return 0;
+    }
     dim_t get_group(int d) const {
         // If groups were not requested, return `1` for convenience.
         if (group_ndims_ == default_quant_entry().group_ndims_) return 1;
@@ -93,13 +168,33 @@ struct quant_entry_t : public c_compatible {
     // `gtests/internals/test_comparison_operators` linking requirements which
     // mandates bodies to be in the header file.
     bool operator==(const quant_entry_t &rhs) const {
-        return mask_ == rhs.mask_ && data_type_ == rhs.data_type_
+        bool result = (type_ == rhs.type_ && is_set_ == rhs.is_set_
+                && mask_ == rhs.mask_
+                && data_type_ == rhs.data_type_
                 && group_ndims_ == rhs.group_ndims_
                 && IMPLICATION(group_ndims_ > 0,
-                        utils::array_cmp(
-                                group_dims_, rhs.group_dims_, group_ndims_));
+                    utils::array_cmp(
+                        group_dims_, rhs.group_dims_, group_ndims_)));
+
+        if (!result) return false;
+        result = (is_set_scale == rhs.is_set_scale
+                && mask_scale == rhs.mask_scale
+                && data_type_scale == rhs.data_type_scale
+                && ndims_scale == rhs.ndims_scale
+                && IMPLICATION(ndims_scale > 0,
+                    utils::array_cmp(
+                        dims_scale, rhs.dims_scale, ndims_scale)));
+
+        if (!result) return false;
+        result = (is_set_wei == rhs.is_set_wei
+                && mask_wei == rhs.mask_wei
+                && data_type_wei == rhs.data_type_wei
+                && ndims_wei == rhs.ndims_wei
+                && IMPLICATION(ndims_wei > 0,
+                    utils::array_cmp(
+                        dims_wei, rhs.dims_wei, ndims_wei)));
+        return result;
     }
-
     size_t get_hash() const;
 
     void serialize(serialization_stream_t &sstream) const;
@@ -109,23 +204,32 @@ struct quant_entry_t : public c_compatible {
     std::string get_verbose() const;
 
 private:
+    data_type_t data_type_ = data_type::undef;
     int group_ndims_ = 0;
     dims_t group_dims_ {};
-public:
     // Note: INT_MIN is used on purpose to avoid potential issues when
     // `(mask & bit)` expression will return `true`. `INT_MIN` is represented
     // as `10...0` in bits and will avoid such situations.
     int mask_ = INT_MIN;
-    data_type_t data_type_ = data_type::undef;
+    bool is_set_ = false;
     // openvino extension
+    enum entry_type {
+        NONE = 0,
+        DNNL = 1,
+        OV_SCALES = 2,
+        OV_ZERO_POINTS = 4
+    };
+    int type_ = NONE;
     // scale
-    bool is_set_ = false;
-    int ndims_ = 0;
-    dims_t dims_ {};
+    bool is_set_scale = false;
+    int ndims_scale = 0;
+    int mask_scale = INT_MIN;
+    dims_t dims_scale {};
+    data_type_t data_type_scale = data_type::undef;
     // zero_point
     bool is_set_wei = false;
     int ndims_wei = 0;
-    int mask_wei = 0;
+    int mask_wei = INT_MIN;
     dims_t dims_wei {};
     data_type_t data_type_wei = data_type::s32;
 };
@@ -144,57 +248,35 @@ struct quant_entries_t : public c_compatible {
 
     // See `set(...)` comment for `quant_entry_t` for a design choice
     // explanation.
-    status_t set(int arg, int mask) {
+    virtual status_t set(int arg, int mask) {
         return set(arg, mask, default_data_type_, 0, {});
     }
-    status_t set_scales(int arg, const dims_t dims, int ndims, data_type_t data_type = data_type::f32) {
-        if (!check_arg(arg)) return status::invalid_arguments;
-        entries_[arg].is_set_ = true;
-        entries_[arg].ndims_ = ndims;
-        entries_[arg].mask_ = 1;
-        entries_[arg].data_type_ = data_type;
-        utils::array_copy(entries_[arg].dims_, dims, entries_[arg].ndims_);
-        return status::success;
-    }
-    status_t set_zero_points(int arg, const dims_t dims, int ndims, data_type_t data_type) {
-        const bool supported_arg = utils::one_of(arg, DNNL_ARG_WEIGHTS);
-        if (!supported_arg) return status::unimplemented;
-
-        switch (arg) {
-            case DNNL_ARG_WEIGHTS:
-                entries_[arg].is_set_wei = true;
-                entries_[arg].ndims_wei = ndims;
-                entries_[arg].mask_wei = 1;
-                utils::array_copy(entries_[arg].dims_wei, dims, ndims);
-                entries_[arg].data_type_wei = data_type;
-                break;
-        }
-        return status::success;
-    }
     const dims_t & get_dims(int arg) const {
-        return get(arg).dims_wei;
+        return get(arg).get_dims();
     }
     int get_ndims(int arg) const {
-        const bool supported_arg = utils::one_of(arg, DNNL_ARG_WEIGHTS);
-        if (!supported_arg) return status::unimplemented;
-        switch (arg) {
-            case DNNL_ARG_WEIGHTS: return get(arg).ndims_wei; break;
-            default: return 0;
-        }
+        return get(arg).get_ndims();
     }
-    status_t set(int arg, int mask, data_type_t data_type, int group_ndims,
+    virtual status_t set(int arg, int mask, data_type_t data_type, int group_ndims,
             const dims_t group_dims) {
         if (!check_arg(arg)) return status::invalid_arguments;
         CHECK(entries_[arg].set(mask, data_type, group_ndims, group_dims));
-        if (arg == DNNL_ARG_WEIGHTS) {
-            utils::array_copy(entries_[arg].dims_wei, group_dims, group_ndims);
-            entries_[arg].ndims_wei = group_ndims;
-        }
         return status::success;
     }
+    status_t set_scales(int arg, const dims_t dims, int ndims, data_type_t data_type = data_type::f32) {
+        if (!check_arg(arg)) return status::invalid_arguments;
+        CHECK(entries_[arg].set_scales(dims, ndims, data_type));
+        return status::success;
+    }
+    status_t set_zero_points(int arg, const dims_t dims, int ndims, data_type_t data_type) {
+        if (arg != DNNL_ARG_WEIGHTS) return status::unimplemented;
+        CHECK(entries_[arg].set_zero_points(dims, ndims, data_type));
+        return status::success;
+    }
+
     // Use this interface with `default_quant_entry` when need to remove a
     // specific entry.
-    status_t set(int arg, const quant_entry_t &other) {
+    virtual status_t set(int arg, const quant_entry_t &other) {
         return entries_[arg].set(other);
     }
 
@@ -356,7 +438,23 @@ struct zero_points_t : public quant_entries_t {
     }
 
     static zero_points_t deserialize(deserializer_t &d);
+    status_t set(int arg, int mask) override {
+        return quant_entries_t::set(arg, mask, default_data_type_, 0, {});
+    }
+    status_t set(int arg, int mask, data_type_t data_type, int group_ndims,
+            const dims_t group_dims) override {
+        if (!check_arg(arg)) return status::invalid_arguments;
+        if (arg == DNNL_ARG_WEIGHTS) {
+            CHECK(entries_[arg].set_zero_points(group_dims, group_ndims, data_type, mask));
+        } else {
+            CHECK(entries_[arg].set(mask, data_type, group_ndims, group_dims));
+        }
+        return status::success;
+    }
 
+    status_t set(int arg, const quant_entry_t &other) override {
+        return quant_entries_t::set(arg, other);
+    }
 private:
     static constexpr data_type_t default_data_type_ = data_type::s32;
 
@@ -388,7 +486,7 @@ struct src_dyn_quant_params_t : public c_compatible {
         return status::success;
     }
 
-    uint64_t get() {
+    uint64_t get() const {
         return group_size_;
     }
 
@@ -397,6 +495,7 @@ struct src_dyn_quant_params_t : public c_compatible {
         return group_size_ == rhs.group_size_;
     }
 
+private:
     uint64_t group_size_;
 };
 

src/common/primitive_hashing_utils.cpp

@@ -192,7 +192,7 @@ size_t get_attr_hash(const primitive_attr_t &attr) {
         seed = hash_combine(
                 seed, get_md_hash(attr.dropout_.user_dropout_desc_));
     }
-    seed = hash_combine(seed, attr.src_dyn_quant_params_.group_size_);
+    seed = hash_combine(seed, attr.src_dyn_quant_params_.get());
     // Combined hash for attributes
     return seed;
 }

src/common/primitive_serialization.cpp

@@ -232,7 +232,7 @@ void serialize(serialization_stream_t &sstream, const primitive_attr_t &attr) {
         int zero = 0;
         sstream.append(zero);
     }
-    sstream.append(attr.src_dyn_quant_params_.group_size_);
+    sstream.append(attr.src_dyn_quant_params_.get());
 }
 
 void serialize(serialization_stream_t &sstream, const concat_desc_t &desc) {

src/common/verbose.cpp

@@ -833,7 +833,7 @@ std::ostream &operator<<(std::ostream &ss, const primitive_attr_t *attr) {
 
     const src_dyn_quant_params_t &dyn_qp = attr->src_dyn_quant_params_;
     if (!dyn_qp.has_default_values()) {
-        ss << "src_dyn_quant_group_size:" << dyn_qp.group_size_ << ";";
+        ss << "src_dyn_quant_group_size:" << dyn_qp.get() << ";";
     }
 
     if (!attr->dropout_.has_default_values()) {