Development baseline3 largedatasets

autosklearn/data/target_validator.py

@@ -166,6 +166,7 @@ def _fit(
             # The label encoder makes sure data is, and remains
             # 1 dimensional
             self.encoder = preprocessing.OrdinalEncoder(handle_unknown='use_encoded_value',
+                                                        dtype=np.int32,
                                                         unknown_value=-1)
         else:
             # We should not reach this if statement as we check for type of targets before

autosklearn/ensemble_builder.py

@@ -1260,8 +1260,7 @@ def fit_ensemble(self, selected_keys: list):
 
         try:
             self.logger.debug(
-                "Fitting the ensemble on %d models.",
-                len(predictions_train),
+                f"Fitting the ensemble on {len(predictions_train)} models: {include_num_runs}"
             )
             start_time = time.time()
             ensemble.fit(predictions_train, self.y_true_ensemble,

autosklearn/evaluation/__init__.py

@@ -130,6 +130,7 @@ def __init__(
         init_params: Optional[Dict[str, Any]] = None,
         budget_type: Optional[str] = None,
         ta: Optional[Callable] = None,
+        compute_train_loss: bool = False,
         **resampling_strategy_args: Any,
     ):
 
@@ -190,6 +191,7 @@ def __init__(
         self.disable_file_output = disable_file_output
         self.init_params = init_params
         self.budget_type = budget_type
+        self.compute_train_loss = compute_train_loss
 
         if memory_limit is not None:
             memory_limit = int(math.ceil(memory_limit))
@@ -204,6 +206,7 @@ def __init__(
             self._get_test_loss = True
         else:
             self._get_test_loss = False
+        del dm
 
         self.port = port
         self.pynisher_context = pynisher_context
@@ -461,9 +464,11 @@ def run(
                 additional_run_info['learning_curve'] = learning_curve
                 additional_run_info['learning_curve_runtime'] = learning_curve_runtime
 
-            train_learning_curve = autosklearn.evaluation.util.extract_learning_curve(
-                info, 'train_loss'
-            )
+            train_learning_curve = []
+            if self.compute_train_loss:
+                train_learning_curve = autosklearn.evaluation.util.extract_learning_curve(
+                    info, 'train_loss'
+                )
             if len(train_learning_curve) > 1:
                 additional_run_info['train_learning_curve'] = train_learning_curve
                 additional_run_info['learning_curve_runtime'] = learning_curve_runtime
@@ -498,5 +503,11 @@ def run(
         runtime = float(obj.wall_clock_time)
 
         autosklearn.evaluation.util.empty_queue(queue)
-        self.logger.info("Finished evaluating configuration %d" % config_id)
+        self.logger.info(
+            "Finished evaluating configuration c:{}/i:{} with status {}".format(
+                config_id,
+                instance,
+                status
+            )
+        )
         return status, cost, runtime, additional_run_info

autosklearn/metalearning/files/accuracy_binary.classification_dense/algorithm_runs.arff

@@ -11,8 +11,8 @@ twonorm,1.0,1,0.015561015561015523,ok
 autouniv-au1-1000,1.0,7,0.18787878787878787,ok
 hill-valley,1.0,9,0.0,ok
 eye_movements,1.0,14,0.17655210643015518,ok
-abalone,1.0,17,0.7670537010159652,ok
-tamilnadu-electricity,1.0,19,0.9385715231349705,ok
+abalone,1.0,709,0.05224963715529751,ok
+tamilnadu-electricity,1.0,709,0.05924405904547558,ok
 xd6,1.0,22,0.0,ok
 colleges_usnews,1.0,25,0.2517482517482518,ok
 pc4,1.0,28,0.079002079002079,ok
@@ -34,7 +34,7 @@ cardiotocography,1.0,83,0.0,ok
 kick,1.0,86,0.09483474505896028,ok
 microaggregation2,1.0,88,0.3684848484848485,ok
 monks-problems-2,1.0,91,0.0,ok
-autouniv-au6-750,1.0,97,0.6720647773279352,ok
+autouniv-au6-750,1.0,709,0.1578947368421053,ok
 meta_stream_intervals.arff,1.0,99,0.004898013955984992,ok
 diabetes130us,1.0,102,0.3863676969805253,ok
 steel-plates-fault,1.0,105,0.21562499999999996,ok
@@ -46,18 +46,18 @@ isolet,1.0,121,0.03342401865526623,ok
 led24,1.0,124,0.2547348484848485,ok
 satimage,1.0,129,0.07873644507307875,ok
 pbcseq,1.0,131,0.14352574102964122,ok
-bachchoralharmony,1.0,134,0.8801498127340824,ok
+bachchoralharmony,1.0,709,0.7019796682718031,ok
 compas-two-years,1.0,138,0.3256748994830557,ok
 volcanoes-b6,1.0,142,0.030520646319569078,ok
 philippine,1.0,145,0.16943866943866948,ok
 irish,1.0,147,0.0,ok
 monks-problems-1,1.0,149,0.0,ok
 porto-seguro,1.0,150,0.03627449482992995,ok
 arsenic-female-lung,1.0,154,0.005434782608695676,ok
-autouniv-au7-1100,1.0,160,0.5371900826446281,ok
+autouniv-au7-1100,1.0,709,0.26997245179063367,ok
 steel-plates-fault,1.0,162,0.0,ok
 speeddating,1.0,165,0.12590448625180894,ok
-devnagari-script,1.0,171,0.03764822134387347,ok
+devnagari-script,1.0,709,0.02154150197628457,ok
 arsenic-male-lung,1.0,172,0.0,ok
 fars,1.0,173,0.19925567994237525,ok
 wdbc,1.0,176,0.010695187165775444,ok
@@ -66,7 +66,7 @@ rmftsa_ladata,1.0,184,0.09580838323353291,ok
 banknote-authentication,1.0,188,0.0,ok
 visualizing_soil,1.0,191,0.00035075412136087447,ok
 wine-quality-white,1.0,193,0.31311881188118806,ok
-analcatdata_dmft,1.0,197,0.7832699619771863,ok
+analcatdata_dmft,1.0,709,0.2433460076045627,ok
 ova_colon,1.0,199,0.03339882121807469,ok
 volcanoes-e1,1.0,204,0.07435897435897432,ok
 cpu_small,1.0,206,0.06992230854605996,ok
@@ -100,7 +100,7 @@ mc1,1.0,302,0.004803073967339144,ok
 ova_prostate,1.0,304,0.0,ok
 volcanoes-b2,1.0,310,0.02840909090909094,ok
 hypothyroid,1.0,313,0.002411575562700996,ok
-autouniv-au7-700,1.0,315,0.48051948051948057,ok
+autouniv-au7-700,1.0,709,0.37662337662337664,ok
 ova_lung,1.0,317,0.013752455795677854,ok
 pokerhand,1.0,322,0.0009647853352628966,ok
 quake,1.0,327,0.42618384401114207,ok
@@ -150,7 +150,7 @@ micro-mass,1.0,480,0.07446808510638303,ok
 indian_pines,1.0,485,0.0553529996685449,ok
 miceprotein,1.0,486,0.0,ok
 diabetes,1.0,492,0.23320158102766797,ok
-collins,1.0,494,0.696969696969697,ok
+collins,1.0,709,0.2212121212121212,ok
 internet-advertisements,1.0,499,0.020332717190388205,ok
 ova_endometrium,1.0,503,0.037328094302554016,ok
 phishingwebsites,1.0,506,0.023300438596491224,ok
@@ -164,7 +164,7 @@ gtsrb-hog01,1.0,528,0.005962820063135754,ok
 puma32h,1.0,532,0.0828708842027377,ok
 bioresponse,1.0,536,0.20856911883589324,ok
 cjs,1.0,538,0.0032537960954447387,ok
-spoken-arabic-digit,1.0,543,0.7427538734258812,ok
+spoken-arabic-digit,1.0,709,0.12207334760687893,ok
 fri_c4_500_100,1.0,546,0.08484848484848484,ok
 ova_kidney,1.0,548,0.00982318271119842,ok
 ldpa,1.0,553,0.005643071154164292,ok

autosklearn/metalearning/files/accuracy_binary.classification_dense/configurations.csv

@@ -204,3 +204,4 @@ idx,balancing:strategy,classifier:__choice__,classifier:adaboost:algorithm,class
 701,none,liblinear_svc,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17045.7732372673,False,True,1,squared_hinge,ovr,l2,0.0008192814934567822,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,no_encoding,minority_coalescer,0.008405913574040906,mean,robust_scaler,,,0.9598706457974426,0.1882826974837794,select_percentile_classification,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,63.57218134514263,mutual_info,,,
 702,weighting,gradient_boosting,,,,,,,,,,,,,,,,,,,,,,,,train,3.387912939529945e-10,0.30755227194768237,auto,255,None,60,39,18,loss,1e-07,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,one_hot_encoding,minority_coalescer,0.010000000000000004,most_frequent,none,,,,,select_percentile_classification,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,93.39844669585806,f_classif,,,
 707,weighting,lda,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,auto,,0.00018030860519654287,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,no_encoding,minority_coalescer,0.041538950281903686,mean,robust_scaler,,,0.8898452660666816,0.1704082739702074,kitchen_sinks,,,,,,,,,,,,,,,,,,,,,,,,0.0005845623820571637,8606,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+709,weighting,mlp,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,relu,0.0001,auto,0.9,0.999,train,1E-08,2,0.0003,32,128,True,adam,0.0001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,one_hot_encoding,minority_coalescer,0.004071801722749603,median,quantile_transformer,1000,normal,,,no_preprocessing,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

autosklearn/metalearning/files/accuracy_multiclass.classification_dense/algorithm_runs.arff

@@ -11,8 +11,8 @@ twonorm,1.0,1,0.015561015561015523,ok
 autouniv-au1-1000,1.0,7,0.18787878787878787,ok
 hill-valley,1.0,9,0.0,ok
 eye_movements,1.0,14,0.17655210643015518,ok
-abalone,1.0,17,0.7670537010159652,ok
-tamilnadu-electricity,1.0,19,0.9385715231349705,ok
+abalone,1.0,709,0.05224963715529751,ok
+tamilnadu-electricity,1.0,709,0.05924405904547558,ok
 xd6,1.0,22,0.0,ok
 colleges_usnews,1.0,25,0.2517482517482518,ok
 pc4,1.0,28,0.079002079002079,ok
@@ -34,7 +34,7 @@ cardiotocography,1.0,83,0.0,ok
 kick,1.0,86,0.09483474505896028,ok
 microaggregation2,1.0,88,0.3684848484848485,ok
 monks-problems-2,1.0,91,0.0,ok
-autouniv-au6-750,1.0,97,0.6720647773279352,ok
+autouniv-au6-750,1.0,709,0.1578947368421053,ok
 meta_stream_intervals.arff,1.0,99,0.004898013955984992,ok
 diabetes130us,1.0,102,0.3863676969805253,ok
 steel-plates-fault,1.0,105,0.21562499999999996,ok
@@ -46,18 +46,18 @@ isolet,1.0,121,0.03342401865526623,ok
 led24,1.0,124,0.2547348484848485,ok
 satimage,1.0,129,0.07873644507307875,ok
 pbcseq,1.0,131,0.14352574102964122,ok
-bachchoralharmony,1.0,134,0.8801498127340824,ok
+bachchoralharmony,1.0,709,0.7019796682718031,ok
 compas-two-years,1.0,138,0.3256748994830557,ok
 volcanoes-b6,1.0,142,0.030520646319569078,ok
 philippine,1.0,145,0.16943866943866948,ok
 irish,1.0,147,0.0,ok
 monks-problems-1,1.0,149,0.0,ok
 porto-seguro,1.0,150,0.03627449482992995,ok
 arsenic-female-lung,1.0,154,0.005434782608695676,ok
-autouniv-au7-1100,1.0,160,0.5371900826446281,ok
+autouniv-au7-1100,1.0,709,0.26997245179063367,ok
 steel-plates-fault,1.0,162,0.0,ok
 speeddating,1.0,165,0.12590448625180894,ok
-devnagari-script,1.0,171,0.03764822134387347,ok
+devnagari-script,1.0,709,0.02154150197628457,ok
 arsenic-male-lung,1.0,172,0.0,ok
 fars,1.0,173,0.19925567994237525,ok
 wdbc,1.0,176,0.010695187165775444,ok
@@ -66,7 +66,7 @@ rmftsa_ladata,1.0,184,0.09580838323353291,ok
 banknote-authentication,1.0,188,0.0,ok
 visualizing_soil,1.0,191,0.00035075412136087447,ok
 wine-quality-white,1.0,193,0.31311881188118806,ok
-analcatdata_dmft,1.0,197,0.7832699619771863,ok
+analcatdata_dmft,1.0,709,0.2433460076045627,ok
 ova_colon,1.0,199,0.03339882121807469,ok
 volcanoes-e1,1.0,204,0.07435897435897432,ok
 cpu_small,1.0,206,0.06992230854605996,ok
@@ -100,7 +100,7 @@ mc1,1.0,302,0.004803073967339144,ok
 ova_prostate,1.0,304,0.0,ok
 volcanoes-b2,1.0,310,0.02840909090909094,ok
 hypothyroid,1.0,313,0.002411575562700996,ok
-autouniv-au7-700,1.0,315,0.48051948051948057,ok
+autouniv-au7-700,1.0,709,0.37662337662337664,ok
 ova_lung,1.0,317,0.013752455795677854,ok
 pokerhand,1.0,322,0.0009647853352628966,ok
 quake,1.0,327,0.42618384401114207,ok
@@ -150,7 +150,7 @@ micro-mass,1.0,480,0.07446808510638303,ok
 indian_pines,1.0,485,0.0553529996685449,ok
 miceprotein,1.0,486,0.0,ok
 diabetes,1.0,492,0.23320158102766797,ok
-collins,1.0,494,0.696969696969697,ok
+collins,1.0,709,0.2212121212121212,ok
 internet-advertisements,1.0,499,0.020332717190388205,ok
 ova_endometrium,1.0,503,0.037328094302554016,ok
 phishingwebsites,1.0,506,0.023300438596491224,ok
@@ -164,7 +164,7 @@ gtsrb-hog01,1.0,528,0.005962820063135754,ok
 puma32h,1.0,532,0.0828708842027377,ok
 bioresponse,1.0,536,0.20856911883589324,ok
 cjs,1.0,538,0.0032537960954447387,ok
-spoken-arabic-digit,1.0,543,0.7427538734258812,ok
+spoken-arabic-digit,1.0,709,0.12207334760687893,ok
 fri_c4_500_100,1.0,546,0.08484848484848484,ok
 ova_kidney,1.0,548,0.00982318271119842,ok
 ldpa,1.0,553,0.005643071154164292,ok

autosklearn/metalearning/files/accuracy_multiclass.classification_dense/configurations.csv

@@ -204,3 +204,4 @@ idx,balancing:strategy,classifier:__choice__,classifier:adaboost:algorithm,class
 701,none,liblinear_svc,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17045.7732372673,False,True,1,squared_hinge,ovr,l2,0.0008192814934567822,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,no_encoding,minority_coalescer,0.008405913574040906,mean,robust_scaler,,,0.9598706457974426,0.1882826974837794,select_percentile_classification,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,63.57218134514263,mutual_info,,,
 702,weighting,gradient_boosting,,,,,,,,,,,,,,,,,,,,,,,,train,3.387912939529945e-10,0.30755227194768237,auto,255,None,60,39,18,loss,1e-07,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,one_hot_encoding,minority_coalescer,0.010000000000000004,most_frequent,none,,,,,select_percentile_classification,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,93.39844669585806,f_classif,,,
 707,weighting,lda,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,auto,,0.00018030860519654287,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,no_encoding,minority_coalescer,0.041538950281903686,mean,robust_scaler,,,0.8898452660666816,0.1704082739702074,kitchen_sinks,,,,,,,,,,,,,,,,,,,,,,,,0.0005845623820571637,8606,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+709,weighting,mlp,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,relu,0.0001,auto,0.9,0.999,train,1E-08,2,0.0003,32,128,True,adam,0.0001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,one_hot_encoding,minority_coalescer,0.004071801722749603,median,quantile_transformer,1000,normal,,,no_preprocessing,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,