Skip to content

Commit 70fbe73

Browse files
benwtrentvaleriy42
benwtrent
and
valeriy42
authored
[ML] Activate model metadata output (#1456) (#1457)
Activate the output of the model metadata and the corresponding unit tests for total feature importance. The implementation itself was introduced in #1387 however, I need to fix the documentation, it was originally attributed to v7.10. Hence, I mark this PR as enhancement to rectify the docs. Co-authored-by: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
1 parent c71e761 commit 70fbe73

File tree

3 files changed

+56
-59
lines changed

3 files changed

+56
-59
lines changed

‎docs/CHANGELOG.asciidoc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@
3030

3131
== {es} version 7.10.0
3232

33+
=== Enhancements
34+
35+
* Calculate total feature importance to store with model metadata. (See {ml-pull}1387[#1387].)
36+
3337
=== Bug Fixes
3438

3539
* Fix progress on resume after final training has completed for classification and regression.
@@ -64,7 +68,6 @@
6468
regression. (See {ml-pull}1340[#1340].)
6569
* Improvement in handling large inference model definitions. (See {ml-pull}1349[#1349].)
6670
* Add a peak_model_bytes field to model_size_stats. (See {ml-pull}1389[#1389].)
67-
* Calculate total feature importance as a new result type. (See {ml-pull}1387[#1387].)
6871

6972
=== Bug Fixes
7073

‎lib/api/CDataFrameAnalyzer.cc‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,7 @@ void CDataFrameAnalyzer::run() {
144144
analysisRunner->waitToFinish();
145145
this->writeInferenceModel(*analysisRunner, outputWriter);
146146
this->writeResultsOf(*analysisRunner, outputWriter);
147-
// TODO reactivate once Java parsing is ready
148-
// this->writeInferenceModelMetadata(*analysisRunner, outputWriter);
147+
this->writeInferenceModelMetadata(*analysisRunner, outputWriter);
149148
}
150149
}
151150

‎lib/api/unittest/CDataFrameAnalyzerFeatureImportanceTest.cc‎

Lines changed: 51 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -537,16 +537,15 @@ BOOST_FIXTURE_TEST_CASE(testRegressionFeatureImportanceAllShap, SFixture) {
537537
BOOST_REQUIRE_CLOSE(c3Sum, c4Sum, 5.0); // c3 and c4 within 5% of each other
538538
// make sure the local approximation differs from the prediction always by the same bias (up to a numeric error)
539539
BOOST_REQUIRE_SMALL(maths::CBasicStatistics::variance(bias), 1e-6);
540-
// TODO reactivate once Java parsing is ready
541-
// BOOST_TEST_REQUIRE(hasTotalFeatureImportance);
542-
// BOOST_REQUIRE_CLOSE(c1TotalShapActual,
543-
// maths::CBasicStatistics::mean(c1TotalShapExpected), 1.0);
544-
// BOOST_REQUIRE_CLOSE(c2TotalShapActual,
545-
// maths::CBasicStatistics::mean(c2TotalShapExpected), 1.0);
546-
// BOOST_REQUIRE_CLOSE(c3TotalShapActual,
547-
// maths::CBasicStatistics::mean(c3TotalShapExpected), 1.0);
548-
// BOOST_REQUIRE_CLOSE(c4TotalShapActual,
549-
// maths::CBasicStatistics::mean(c4TotalShapExpected), 1.0);
540+
BOOST_TEST_REQUIRE(hasTotalFeatureImportance);
541+
BOOST_REQUIRE_CLOSE(c1TotalShapActual,
542+
maths::CBasicStatistics::mean(c1TotalShapExpected), 1.0);
543+
BOOST_REQUIRE_CLOSE(c2TotalShapActual,
544+
maths::CBasicStatistics::mean(c2TotalShapExpected), 1.0);
545+
BOOST_REQUIRE_CLOSE(c3TotalShapActual,
546+
maths::CBasicStatistics::mean(c3TotalShapExpected), 1.0);
547+
BOOST_REQUIRE_CLOSE(c4TotalShapActual,
548+
maths::CBasicStatistics::mean(c4TotalShapExpected), 1.0);
550549
}
551550

552551
BOOST_FIXTURE_TEST_CASE(testRegressionFeatureImportanceNoImportance, SFixture) {
@@ -629,7 +628,6 @@ BOOST_FIXTURE_TEST_CASE(testClassificationFeatureImportanceAllShap, SFixture) {
629628
if (result["model_metadata"].HasMember("total_feature_importance")) {
630629
hasTotalFeatureImportance = true;
631630
}
632-
// TODO reactivate once Java parsing is ready
633631
c1FooTotalShapActual = readTotalShapValue(result, "c1", "foo");
634632
c2FooTotalShapActual = readTotalShapValue(result, "c2", "foo");
635633
c3FooTotalShapActual = readTotalShapValue(result, "c3", "foo");
@@ -650,24 +648,23 @@ BOOST_FIXTURE_TEST_CASE(testClassificationFeatureImportanceAllShap, SFixture) {
650648
BOOST_REQUIRE_CLOSE(c3Sum, c4Sum, 40.0); // c3 and c4 within 40% of each other
651649
// make sure the local approximation differs from the prediction always by the same bias (up to a numeric error)
652650
BOOST_REQUIRE_SMALL(maths::CBasicStatistics::variance(bias), 1e-6);
653-
// TODO reactivate once Java parsing is ready
654-
// BOOST_TEST_REQUIRE(hasTotalFeatureImportance);
655-
// BOOST_REQUIRE_CLOSE(c1FooTotalShapActual,
656-
// maths::CBasicStatistics::mean(c1TotalShapExpected), 1.0);
657-
// BOOST_REQUIRE_CLOSE(c2FooTotalShapActual,
658-
// maths::CBasicStatistics::mean(c2TotalShapExpected), 1.0);
659-
// BOOST_REQUIRE_CLOSE(c3FooTotalShapActual,
660-
// maths::CBasicStatistics::mean(c3TotalShapExpected), 1.0);
661-
// BOOST_REQUIRE_CLOSE(c4FooTotalShapActual,
662-
// maths::CBasicStatistics::mean(c4TotalShapExpected), 1.0);
663-
// BOOST_REQUIRE_CLOSE(c1BarTotalShapActual,
664-
// maths::CBasicStatistics::mean(c1TotalShapExpected), 1.0);
665-
// BOOST_REQUIRE_CLOSE(c2BarTotalShapActual,
666-
// maths::CBasicStatistics::mean(c2TotalShapExpected), 1.0);
667-
// BOOST_REQUIRE_CLOSE(c3BarTotalShapActual,
668-
// maths::CBasicStatistics::mean(c3TotalShapExpected), 1.0);
669-
// BOOST_REQUIRE_CLOSE(c4BarTotalShapActual,
670-
// maths::CBasicStatistics::mean(c4TotalShapExpected), 1.0);
651+
BOOST_TEST_REQUIRE(hasTotalFeatureImportance);
652+
BOOST_REQUIRE_CLOSE(c1FooTotalShapActual,
653+
maths::CBasicStatistics::mean(c1TotalShapExpected), 1.0);
654+
BOOST_REQUIRE_CLOSE(c2FooTotalShapActual,
655+
maths::CBasicStatistics::mean(c2TotalShapExpected), 1.0);
656+
BOOST_REQUIRE_CLOSE(c3FooTotalShapActual,
657+
maths::CBasicStatistics::mean(c3TotalShapExpected), 1.0);
658+
BOOST_REQUIRE_CLOSE(c4FooTotalShapActual,
659+
maths::CBasicStatistics::mean(c4TotalShapExpected), 1.0);
660+
BOOST_REQUIRE_CLOSE(c1BarTotalShapActual,
661+
maths::CBasicStatistics::mean(c1TotalShapExpected), 1.0);
662+
BOOST_REQUIRE_CLOSE(c2BarTotalShapActual,
663+
maths::CBasicStatistics::mean(c2TotalShapExpected), 1.0);
664+
BOOST_REQUIRE_CLOSE(c3BarTotalShapActual,
665+
maths::CBasicStatistics::mean(c3TotalShapExpected), 1.0);
666+
BOOST_REQUIRE_CLOSE(c4BarTotalShapActual,
667+
maths::CBasicStatistics::mean(c4TotalShapExpected), 1.0);
671668
}
672669

673670
BOOST_FIXTURE_TEST_CASE(testMultiClassClassificationFeatureImportanceAllShap, SFixture) {
@@ -734,7 +731,6 @@ BOOST_FIXTURE_TEST_CASE(testMultiClassClassificationFeatureImportanceAllShap, SF
734731
if (result["model_metadata"].HasMember("total_feature_importance")) {
735732
hasTotalFeatureImportance = true;
736733
}
737-
// TODO reactivate once Java parsing is ready
738734
c1FooTotalShapActual = readTotalShapValue(result, "c1", "foo");
739735
c2FooTotalShapActual = readTotalShapValue(result, "c2", "foo");
740736
c3FooTotalShapActual = readTotalShapValue(result, "c3", "foo");
@@ -749,32 +745,31 @@ BOOST_FIXTURE_TEST_CASE(testMultiClassClassificationFeatureImportanceAllShap, SF
749745
c4BazTotalShapActual = readTotalShapValue(result, "c4", "baz");
750746
}
751747
}
752-
// TODO reactivate once Java parsing is ready
753-
// BOOST_TEST_REQUIRE(hasTotalFeatureImportance);
754-
// BOOST_REQUIRE_CLOSE(c1FooTotalShapActual,
755-
// maths::CBasicStatistics::mean(c1FooTotalShapExpected), 1.0);
756-
// BOOST_REQUIRE_CLOSE(c2FooTotalShapActual,
757-
// maths::CBasicStatistics::mean(c2FooTotalShapExpected), 1.0);
758-
// BOOST_REQUIRE_CLOSE(c3FooTotalShapActual,
759-
// maths::CBasicStatistics::mean(c3FooTotalShapExpected), 1.0);
760-
// BOOST_REQUIRE_CLOSE(c4FooTotalShapActual,
761-
// maths::CBasicStatistics::mean(c4FooTotalShapExpected), 1.0);
762-
// BOOST_REQUIRE_CLOSE(c1BarTotalShapActual,
763-
// maths::CBasicStatistics::mean(c1BarTotalShapExpected), 1.0);
764-
// BOOST_REQUIRE_CLOSE(c2BarTotalShapActual,
765-
// maths::CBasicStatistics::mean(c2BarTotalShapExpected), 1.0);
766-
// BOOST_REQUIRE_CLOSE(c3BarTotalShapActual,
767-
// maths::CBasicStatistics::mean(c3BarTotalShapExpected), 1.0);
768-
// BOOST_REQUIRE_CLOSE(c4BarTotalShapActual,
769-
// maths::CBasicStatistics::mean(c4BarTotalShapExpected), 1.0);
770-
// BOOST_REQUIRE_CLOSE(c1BazTotalShapActual,
771-
// maths::CBasicStatistics::mean(c1BazTotalShapExpected), 1.0);
772-
// BOOST_REQUIRE_CLOSE(c2BazTotalShapActual,
773-
// maths::CBasicStatistics::mean(c2BazTotalShapExpected), 1.0);
774-
// BOOST_REQUIRE_CLOSE(c3BazTotalShapActual,
775-
// maths::CBasicStatistics::mean(c3BazTotalShapExpected), 1.0);
776-
// BOOST_REQUIRE_CLOSE(c4BazTotalShapActual,
777-
// maths::CBasicStatistics::mean(c4BazTotalShapExpected), 1.0);
748+
BOOST_TEST_REQUIRE(hasTotalFeatureImportance);
749+
BOOST_REQUIRE_CLOSE(c1FooTotalShapActual,
750+
maths::CBasicStatistics::mean(c1FooTotalShapExpected), 1.0);
751+
BOOST_REQUIRE_CLOSE(c2FooTotalShapActual,
752+
maths::CBasicStatistics::mean(c2FooTotalShapExpected), 1.0);
753+
BOOST_REQUIRE_CLOSE(c3FooTotalShapActual,
754+
maths::CBasicStatistics::mean(c3FooTotalShapExpected), 1.0);
755+
BOOST_REQUIRE_CLOSE(c4FooTotalShapActual,
756+
maths::CBasicStatistics::mean(c4FooTotalShapExpected), 1.0);
757+
BOOST_REQUIRE_CLOSE(c1BarTotalShapActual,
758+
maths::CBasicStatistics::mean(c1BarTotalShapExpected), 1.0);
759+
BOOST_REQUIRE_CLOSE(c2BarTotalShapActual,
760+
maths::CBasicStatistics::mean(c2BarTotalShapExpected), 1.0);
761+
BOOST_REQUIRE_CLOSE(c3BarTotalShapActual,
762+
maths::CBasicStatistics::mean(c3BarTotalShapExpected), 1.0);
763+
BOOST_REQUIRE_CLOSE(c4BarTotalShapActual,
764+
maths::CBasicStatistics::mean(c4BarTotalShapExpected), 1.0);
765+
BOOST_REQUIRE_CLOSE(c1BazTotalShapActual,
766+
maths::CBasicStatistics::mean(c1BazTotalShapExpected), 1.0);
767+
BOOST_REQUIRE_CLOSE(c2BazTotalShapActual,
768+
maths::CBasicStatistics::mean(c2BazTotalShapExpected), 1.0);
769+
BOOST_REQUIRE_CLOSE(c3BazTotalShapActual,
770+
maths::CBasicStatistics::mean(c3BazTotalShapExpected), 1.0);
771+
BOOST_REQUIRE_CLOSE(c4BazTotalShapActual,
772+
maths::CBasicStatistics::mean(c4BazTotalShapExpected), 1.0);
778773
}
779774

780775
BOOST_FIXTURE_TEST_CASE(testRegressionFeatureImportanceNoShap, SFixture) {

0 commit comments

Comments
 (0)