From b176d88dad640e2251c20ebae959cabfbd1fa0b0 Mon Sep 17 00:00:00 2001 From: pagutierrez Date: Thu, 9 Nov 2017 18:49:00 +0100 Subject: [PATCH] Updated OrBoost for new versions. Deleted extra files. (Issue #5) --- src/Algorithms/orensemble/orensemble/Makefile | 50 - src/Algorithms/orensemble/orensemble/README | 24 - .../orensemble/orensemble/aggrank.cpp | 285 -- .../orensemble/orensemble/aggrank.h | 86 - .../orensemble/boostrank-predict.cpp | 150 - .../orensemble/boostrank-train-crossval.cpp | 211 -- .../orensemble/orensemble/boostrank-train.cpp | 177 - src/Algorithms/orensemble/orensemble/h | 801 ----- .../orensemble/lemga-20060516/lemga/LEMGA.dox | 39 - .../orensemble/lemga-20060516/lemga/Makefile | 258 -- .../lemga-20060516/lemga/adaboost.cpp | 67 - .../lemga-20060516/lemga/adaboost.h | 49 - .../lemga-20060516/lemga/adaboost_ecoc.cpp | 257 -- .../lemga-20060516/lemga/adaboost_ecoc.h | 76 - .../lemga-20060516/lemga/adaboost_erp.cpp | 142 - .../lemga-20060516/lemga/adaboost_erp.h | 49 - .../lemga-20060516/lemga/aggregating.cpp | 130 - .../lemga-20060516/lemga/aggregating.h | 86 - .../lemga-20060516/lemga/bagging.cpp | 55 - .../orensemble/lemga-20060516/lemga/bagging.h | 43 - .../lemga-20060516/lemga/boosting.cpp | 277 -- .../lemga-20060516/lemga/boosting.h | 295 -- .../lemga-20060516/lemga/cascade.cpp | 65 - .../orensemble/lemga-20060516/lemga/cascade.h | 76 - .../lemga-20060516/lemga/cgboost.cpp | 189 - .../orensemble/lemga-20060516/lemga/cgboost.h | 101 - .../orensemble/lemga-20060516/lemga/cost.h | 95 - .../lemga-20060516/lemga/crossval.cpp | 240 -- .../lemga-20060516/lemga/crossval.h | 149 - .../lemga-20060516/lemga/datafeeder.cpp | 164 - .../lemga-20060516/lemga/datafeeder.h | 71 - .../orensemble/lemga-20060516/lemga/dataset.h | 176 - .../lemga-20060516/lemga/feedforwardnn.cpp | 286 -- .../lemga-20060516/lemga/feedforwardnn.h | 108 - .../lemga-20060516/lemga/kernel.cpp | 67 - .../orensemble/lemga-20060516/lemga/kernel.h | 204 -- .../lemga-20060516/lemga/learnmodel.cpp | 225 -- .../lemga-20060516/lemga/learnmodel.h | 204 -- .../lemga/libsvm-2.81-weight.patch | 356 -- .../lemga/libsvm-infkernels.patch | 211 -- .../lemga-20060516/lemga/libsvm-kernel.patch | 28 - .../lemga-20060516/lemga/lpboost.cpp | 127 - .../orensemble/lemga-20060516/lemga/lpboost.h | 87 - .../lemga-20060516/lemga/mgnboost.cpp | 25 - .../lemga-20060516/lemga/mgnboost.h | 94 - .../lemga-20060516/lemga/multiclass_ecoc.cpp | 415 --- .../lemga-20060516/lemga/multiclass_ecoc.h | 136 - .../lemga-20060516/lemga/nnlayer.cpp | 115 - .../orensemble/lemga-20060516/lemga/nnlayer.h | 110 - .../lemga-20060516/lemga/object.cpp | 152 - .../orensemble/lemga-20060516/lemga/object.h | 182 - .../lemga-20060516/lemga/optimize.h | 380 -- .../lemga-20060516/lemga/ordinal_ble.cpp | 320 -- .../lemga-20060516/lemga/ordinal_ble.h | 91 - .../lemga-20060516/lemga/perceptron.cpp | 644 ---- .../lemga-20060516/lemga/perceptron.h | 136 - .../orensemble/lemga-20060516/lemga/pulse.cpp | 184 - .../orensemble/lemga-20060516/lemga/pulse.h | 68 - .../lemga-20060516/lemga/quickfun.c | 50 - .../lemga-20060516/lemga/quickfun.h | 24 - .../orensemble/lemga-20060516/lemga/random.c | 40 - .../orensemble/lemga-20060516/lemga/random.h | 36 - .../lemga-20060516/lemga/shared_ptr.h | 103 - .../orensemble/lemga-20060516/lemga/stump.cpp | 207 -- .../orensemble/lemga-20060516/lemga/stump.h | 61 - .../orensemble/lemga-20060516/lemga/svm.cpp | 428 --- .../orensemble/lemga-20060516/lemga/svm.h | 76 - .../lemga-20060516/lemga/utility.cpp | 59 - .../orensemble/lemga-20060516/lemga/utility.h | 56 - .../lemga-20060516/lemga/vectorop.h | 88 - .../orensemble/lemga-20060516/test/Makefile | 130 - .../orensemble/lemga-20060516/test/README | 20 - .../orensemble/lemga-20060516/test/adabst.cpp | 89 - .../orensemble/lemga-20060516/test/bag.cpp | 69 - .../orensemble/lemga-20060516/test/lpbst.cpp | 70 - .../orensemble/lemga-20060516/test/multi.cpp | 70 - .../orensemble/lemga-20060516/test/nnet.cpp | 88 - .../orensemble/lemga-20060516/test/showlm.cpp | 28 - .../lemga-20060516/test/testsvm.cpp | 84 - .../orensemble/libsvm-2.81/COPYRIGHT | 31 - .../orensemble/libsvm-2.81/FAQ.html | 1154 ------ .../orensemble/libsvm-2.81/Makefile | 15 - .../orensemble/libsvm-2.81/Makefile.win | 39 - .../orensemble/orensemble/libsvm-2.81/README | 524 --- .../orensemble/libsvm-2.81/heart_scale | 270 -- .../orensemble/libsvm-2.81/python/Makefile | 28 - .../orensemble/libsvm-2.81/python/README | 140 - .../libsvm-2.81/python/cross_validation.py | 31 - .../orensemble/libsvm-2.81/python/svm.py | 282 -- .../orensemble/libsvm-2.81/python/svm_test.py | 49 - .../orensemble/libsvm-2.81/python/svmc.i | 101 - .../orensemble/libsvm-2.81/python/svmc_wrap.c | 2084 ----------- .../python/test_cross_validation.py | 27 - .../orensemble/libsvm-2.81/svm-predict.c | 177 - .../orensemble/libsvm-2.81/svm-scale.c | 308 -- .../libsvm-2.81/svm-toy/gtk/Makefile | 22 - .../libsvm-2.81/svm-toy/gtk/callbacks.cpp | 419 --- .../libsvm-2.81/svm-toy/gtk/callbacks.h | 54 - .../libsvm-2.81/svm-toy/gtk/interface.c | 164 - .../libsvm-2.81/svm-toy/gtk/interface.h | 14 - .../orensemble/libsvm-2.81/svm-toy/gtk/main.c | 23 - .../libsvm-2.81/svm-toy/gtk/svm-toy.glade | 238 -- .../libsvm-2.81/svm-toy/qt/Makefile | 19 - .../libsvm-2.81/svm-toy/qt/svm-toy.cpp | 430 --- .../libsvm-2.81/svm-toy/windows/svm-toy.cpp | 453 --- .../orensemble/libsvm-2.81/svm-train.c | 301 -- .../orensemble/orensemble/libsvm-2.81/svm.cpp | 3164 ----------------- .../orensemble/orensemble/libsvm-2.81/svm.h | 73 - .../orensemble/libsvm-2.81/tools/README | 116 - .../orensemble/libsvm-2.81/tools/easy.py | 78 - .../orensemble/libsvm-2.81/tools/grid.py | 349 -- .../orensemble/libsvm-2.81/tools/subset.py | 144 - .../orensemble/orensemble/orboost.cpp | 140 - .../orensemble/orensemble/orboost.h | 50 - .../orensemble/orboostallProcessFolder.sh | 78 - .../orensemble/orensemble/pyrimdines.dense | 50 - .../orensemble/orensemble/pyrimdines.model | 410 --- .../orensemble/orensemble/rankboost.cpp | 149 - .../orensemble/orensemble/rankboost.h | 41 - .../orensemble/orensemble/softperc.cpp | 58 - .../orensemble/orensemble/softperc.h | 45 - .../orensemble/train_automobile.0.model | 110 - 122 files changed, 24216 deletions(-) delete mode 100644 src/Algorithms/orensemble/orensemble/Makefile delete mode 100644 src/Algorithms/orensemble/orensemble/README delete mode 100644 src/Algorithms/orensemble/orensemble/aggrank.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/aggrank.h delete mode 100644 src/Algorithms/orensemble/orensemble/boostrank-predict.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/boostrank-train-crossval.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/boostrank-train.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/LEMGA.dox delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/Makefile delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cost.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/dataset.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-2.81-weight.patch delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-infkernels.patch delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-kernel.patch delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/optimize.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.c delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.c delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/shared_ptr.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/vectorop.h delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/Makefile delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/README delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/adabst.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/bag.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/lpbst.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/multi.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/nnet.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/showlm.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/lemga-20060516/test/testsvm.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/COPYRIGHT delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/FAQ.html delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile.win delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/README delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/heart_scale delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/Makefile delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/README delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/cross_validation.py delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/svm.py delete mode 100755 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/svm_test.py delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/svmc.i delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/svmc_wrap.c delete mode 100755 src/Algorithms/orensemble/orensemble/libsvm-2.81/python/test_cross_validation.py delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-predict.c delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-scale.c delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/gtk/Makefile delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/gtk/callbacks.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/gtk/callbacks.h delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/gtk/interface.c delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/gtk/interface.h delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/gtk/main.c delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/gtk/svm-toy.glade delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/qt/Makefile delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/qt/svm-toy.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-toy/windows/svm-toy.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm-train.c delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/svm.h delete mode 100644 src/Algorithms/orensemble/orensemble/libsvm-2.81/tools/README delete mode 100755 src/Algorithms/orensemble/orensemble/libsvm-2.81/tools/easy.py delete mode 100755 src/Algorithms/orensemble/orensemble/libsvm-2.81/tools/grid.py delete mode 100755 src/Algorithms/orensemble/orensemble/libsvm-2.81/tools/subset.py delete mode 100644 src/Algorithms/orensemble/orensemble/orboost.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/orboost.h delete mode 100755 src/Algorithms/orensemble/orensemble/orboostallProcessFolder.sh delete mode 100644 src/Algorithms/orensemble/orensemble/pyrimdines.dense delete mode 100644 src/Algorithms/orensemble/orensemble/pyrimdines.model delete mode 100644 src/Algorithms/orensemble/orensemble/rankboost.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/rankboost.h delete mode 100644 src/Algorithms/orensemble/orensemble/softperc.cpp delete mode 100644 src/Algorithms/orensemble/orensemble/softperc.h delete mode 100644 src/Algorithms/orensemble/orensemble/train_automobile.0.model diff --git a/src/Algorithms/orensemble/orensemble/Makefile b/src/Algorithms/orensemble/orensemble/Makefile deleted file mode 100644 index 556dada..0000000 --- a/src/Algorithms/orensemble/orensemble/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -LEMGA = lemga-20060516/lemga - -objs = boostrank-train boostrank-predict - -all: $(objs) - -.PHONY: clean $(addsuffix _clean, $(objs)) -clean: $(addsuffix _clean, $(objs)) - -include ${LEMGA}/Makefile - -VPATH = ${LEMGA} -CC_FLAGS := -I${LEMGA} ${CC_FLAGS} - -softperc.o: softperc.cpp softperc.h - $(CC) $(CC_FLAGS) -O3 -c -o $@ $< -aggrank.o: aggrank.cpp aggrank.h - $(CC) $(CC_FLAGS) -O3 -c -o $@ $< -orboost.o: orboost.cpp orboost.h aggrank.h - $(CC) $(CC_FLAGS) -O3 -c -o $@ $< -rankboost.o: rankboost.cpp rankboost.h aggrank.h - $(CC) $(CC_FLAGS) -O3 -c -o $@ $< - -boostrank_o := $(stump_o) $(perceptron_o) $(aggregating_o) $(nnlayer_o) $(feedforwardnn_o) $(crossval_o) $(datafeeder_o) -boostrank_ro := $(addprefix r, $(boostrank_o)) softperc.o \ - aggrank.o orboost.o rankboost.o - -boostrank-train_src = boostrank-train.cpp aggrank.h orboost.h rankboost.h \ - $(stump_h) $(perceptron_h) $(nnlayer_h) $(feedforwardnn_h) $(crossval_h) $(datafeeder_h)\ - softperc.h - -boostrank-train.o: $(boostrank-train_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -boostrank-train: $(boostrank_ro) boostrank-train.o - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -boostrank-train_clean: - $(RM) $(boostrank_ro) \ - boostrank-train.o boostrank-train - -boostrank-predict_src = boostrank-predict.cpp aggrank.h orboost.h rankboost.h \ - $(object_h) $(stump_h) $(perceptron_h) \ - softperc.h - -boostrank-predict.o: $(boostrank-predict_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -boostrank-predict: $(boostrank_ro) boostrank-predict.o - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -boostrank-predict_clean: - $(RM) $(boostrank_ro) \ - boostrank-predict.o boostrank-predict diff --git a/src/Algorithms/orensemble/orensemble/README b/src/Algorithms/orensemble/orensemble/README deleted file mode 100644 index 30b887b..0000000 --- a/src/Algorithms/orensemble/orensemble/README +++ /dev/null @@ -1,24 +0,0 @@ -The program implements three large-margin thresholded ensemble -algorithms for ordinal regression. It includes an improved RankBoost -algorithm, the ORBoost-LR algorithm, and the ORBoost-All algorithm. -Details can be found in - -H.-T. Lin and L. Li. Large-Margin Thresholded Ensembles for -Ordinal Regression: Theory and Practice. In J. Balcazar et al., eds., -Algorithmic Learning Theory: ALT '06, vol. 4264 of Lecture Notes in -Artificial Intelligence, 319-333, Springer-Verlag. Barcelona, Spain, 2006. - -===Installation=== -The program needs to be linked to the LEMGA library by Ling Li. LEMGA -can be downloaded at - -http://www.work.caltech.edu/ling/lemga/index.html - -The Makefile entry "LEMGA" should be modified for correct linking. -The program has been tested to work with the 2006/05/16 snapshot of LEMGA. - -===Copyright Disclaimer=== -Copyright (c) 2006-2007 Hsuan-Tien Lin (htlin at caltech.edu) -All rights reserved. Redistribution and use in source and binary forms, -with or without modification, are only permitted for non-commercial purposes. -The author takes no responsibility on damages or other problems caused. diff --git a/src/Algorithms/orensemble/orensemble/aggrank.cpp b/src/Algorithms/orensemble/orensemble/aggrank.cpp deleted file mode 100644 index 0249cfb..0000000 --- a/src/Algorithms/orensemble/orensemble/aggrank.cpp +++ /dev/null @@ -1,285 +0,0 @@ -/** - aggrank.cpp: an abstract class for general thresholded ensembles - (c) 2006-2007 Hsuan-Tien Lin -**/ -#include -#include -#include -#include - -#include "aggrank.h" - -namespace lemga { - /** basics **/ - bool AggRank::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Aggregating, os, vl, 1); - assert(lm_wgt.size() == lm.size()); - for (UINT i = 0; i < lm_wgt.size(); ++i) - os << lm_wgt[i] << ' '; - if (!lm_wgt.empty()) os << '\n'; - os << n_rank << '\n'; - UINT t=0; - for (UINT i = 0; i <= lm.size(); i++){ - for (UINT k = 1; k < n_rank; k++) - os << thres[t++] << ' '; - os << '\n'; - } - - if (t > 0) os << '\n'; - return true; - } - - bool AggRank::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Aggregating, is, vl, 1, v); - - const UINT n = lm.size(); - lm_wgt.resize(n); - for (UINT i = 0; i < n; ++i) - if (!(is >> lm_wgt[i])) return false; - - if (!set_aggregation_size(n)) return false; - - UINT _n_rank; - if (!(is >> _n_rank)) - return false; - set_n_rank(_n_rank); - thres.resize((n+1) * (n_thres)); - UINT t=0; - for (UINT i = 0; i <= n; i++) - for (UINT k = 1; k < n_rank; k++) - if (!(is >> thres[t++])) return false; - - return true; - } - - Output AggRank::operator() (const Input& x, UINT iter) const { - //y[1] is the decision value, and y[0] is the prediction - assert(_n_out == 1); - Output y(2, 0); - - if (iter > n_in_agg) iter = n_in_agg; - - for (UINT i = 0; i < iter; ++i) { - Output out = (*lm[i])(x); - - y[1] += (out[0] * lm_wgt[i]); - } - - UINT rank = 1; - UINT idx = iter * n_thres; - for(UINT k = 1; k < n_rank; k++) - if (y[1] >= thres[idx++]) rank++; - y[0] = rank; - - return y; - } - - /** initialize routine **/ - void AggRank::reset() { - //clear the trained results, but retain other settings - Aggregating::reset(); - assert(n_rank >= 2); - lm_wgt.clear(); - thres.resize(n_thres); - for (UINT k = 0; k < n_thres; ++k){ - switch(init_mode){ - case INIT_NAIVE: - thres[k] = 1.0 + k - 0.5 * n_rank; - break; - case INIT_ZERO: - thres[k] = 0.0; - break; - case INIT_RAND: - if (k > 0) - thres[k] = thres[k-1] + randuc(); - else - thres[k] = randuc(); - break; - } - } - - dec_value.resize(n_samples); - for (UINT j = 0; j < n_samples; ++j) - dec_value[j] = 0; - } - - pLearnModel AggRank::learn_weak(const vREAL& rhopos, const vREAL& rhoneg, - const REAL sum_rhodiff) { - // rhopos: the sample weights supporting the prediction to be positive - // rhoneg: the sample weights supporting the prediction to be negative - DataSet* btd = new DataSet(); - DataWgt* btw = new DataWgt(); - - for(UINT j = 0; j < n_samples; ++j) { - if (rhopos[j] >= rhoneg[j]) - btd->append(ptd->x(j), - Output(1, +1)); - else - btd->append(ptd->x(j), - Output(1, -1)); - btw->push_back(fabs(rhopos[j] - rhoneg[j]) / sum_rhodiff); - } - - LearnModel *plm = lm_base->clone(); - - plm->set_train_data(btd, btw); - plm->train(); - return plm; - } - - void AggRank::recompute_dec() { - dec_value.resize(n_samples); - - for (UINT j = 0; j < n_samples; ++j){ - dec_value[j] = 0.0; - for (UINT i = 0; i < n_in_agg; ++i) { - Output out = (*lm[i])(ptd->x(j)); - dec_value[j] += (out[0] >= 0)? lm_wgt[i] : -lm_wgt[i]; - } - } - } - - void AggRank::compute_thres(vREAL::iterator it) { - vREAL thres_now(n_rank+1); - thres_now[0] = -std::numeric_limits::max(); - thres_now[n_rank] = std::numeric_limits::max(); - - switch(thres_mode){ - case THRES_EXPLOSS: - compute_thres_exploss(thres_now, false, false); - break; - case THRES_EXPLOSS_ORDERED: - compute_thres_exploss(thres_now, true, false); - break; - case THRES_ABSLOSS: - compute_thres_dploss(thres_now, true); - break; - case THRES_CLALOSS: - compute_thres_dploss(thres_now, false); - break; - default: - assert(thres_mode < THRES_CLALOSS); - } - - for(UINT k = 1; k < n_rank; ++k) - (*it++)=thres_now[k]; - } - - void AggRank::compute_thres_exploss(vREAL& th, bool ordered, bool full){ - assert(th[n_rank] >= std::numeric_limits::max()); - - vREAL wp(n_rank+1), wn(n_rank+1); - - for(UINT j = 0; j < n_samples; ++j) { - int o = (int)ptd->y(j)[0]; - if (full){ - for(int k=1; k <= o-1; ++k) - wp[k] += exp(-dec_value[j]); - for(int k=o; k <= (int)n_rank-1; ++k) - wn[k] += exp(dec_value[j]); - } - else{ - wp[o-1] += exp(-dec_value[j]); - wn[o] += exp(dec_value[j]); - } - } - - for(UINT k = n_rank-1; k > 0; --k){ - th[k] = 0.5 * log(wn[k] / wp[k]); - //clip th in the good region - if (isnan(th[k])) - th[k] = th[k+1]; - if (isinf(th[k])) - th[k] = isinf(th[k]) * std::numeric_limits::max(); - - if (ordered && th[k] > th[k+1]){ - UINT kk; - REAL wwn(wn[k]), wwp(wp[k]); - - for(kk = k+1; ; kk++){ - assert(kk <= n_rank-1); - // if th[n_rank] = INFTY, - // then the assertion should be true - wwn += wn[kk]; - wwp += wp[kk]; - th[kk] = 0.5 * log(wwn / wwp); - if (isnan(th[kk])) - th[kk] = th[kk+1]; - - if (th[kk] <= th[kk+1]) - break; - } - //update from k to kk - while(kk > k){ - kk--; - th[kk] = th[kk+1]; - } - } - } - } - - void AggRank::compute_thres_dploss(vREAL& th, bool do_abs){ - std::map > declbl; - std::vector cost; - std::map >::iterator it; - vREAL threserr(n_rank+1); - for(UINT j = 0; j < n_samples; ++j) - declbl[dec_value[j]].push_back((UINT)(ptd->y(j)[0])); - - it = declbl.begin(); - cost.resize(declbl.size()); - cost[0].resize(n_rank * 2 + 1); - cost[0][0] = it->first; - for(UINT k = 1; k <= n_rank; ++k){ - REAL loss = 0; - for(std::vector::iterator in = it->second.begin(); - in != it->second.end(); in++) - if (*in != k) loss += (do_abs ? fabs(REAL(*in) - REAL(k)) : 1); - - cost[0][k] = loss; - cost[0][k+n_rank] = -1; - } - - ++it; - for(UINT n = 1; it != declbl.end(); it++, n++){ - REAL bestloss = std::numeric_limits::infinity(); - REAL bestidx = -1; - cost[n].resize(n_rank * 2 + 1); - cost[n][0] = it->first; - for(UINT k = 1; k <= n_rank; ++k){ - if (cost[n-1][k] < bestloss){ - bestloss = cost[n-1][k]; - bestidx = k; - } - REAL loss = bestloss; - for(std::vector::iterator in = it->second.begin(); - in != it->second.end(); in++) - if (*in != k) - loss += (do_abs ? fabs(REAL(*in) - REAL(k)) : 1); - cost[n][k] = loss; - cost[n][k+n_rank] = bestidx; - } - } - - UINT bestidx = 1; - UINT nCost = cost.size(); - for(UINT k = 1; k <= n_rank; ++k){ - if (cost[nCost-1][k] < cost[nCost-1][bestidx]) - bestidx = k; - } - for(UINT j = bestidx; j < n_rank; ++j) - th[j] = std::numeric_limits::max(); - - for(UINT n = nCost-1; n > 0; n--){ - UINT nextidx = (UINT)(cost[n][bestidx + n_rank]); - if (bestidx != nextidx){ - for(UINT j = nextidx; j < bestidx; ++j) - th[j] = (cost[n][0] + cost[n-1][0]) * 0.5; - bestidx = nextidx; - } - } - for(UINT j = 1; j < bestidx; ++j) - th[j] = -std::numeric_limits::max(); - } -} //namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/aggrank.h b/src/Algorithms/orensemble/orensemble/aggrank.h deleted file mode 100644 index c762108..0000000 --- a/src/Algorithms/orensemble/orensemble/aggrank.h +++ /dev/null @@ -1,86 +0,0 @@ -/** - aggrank.h: an abstract class for general thresholded ensembles - (c) 2006-2007 Hsuan-Tien Lin -**/ -#ifndef __LEMGA_AGGREGATING_AGGRANK_H__ -#define __LEMGA_AGGREGATING_AGGRANK_H__ - -#include - - -namespace lemga { - - typedef std::vector vREAL; - - class AggRank : public Aggregating { - public: - enum {THRES_EXPLOSS, THRES_EXPLOSS_ORDERED, - THRES_ABSLOSS, THRES_CLALOSS}; - enum {INIT_ZERO, INIT_NAIVE, INIT_RAND}; - - protected: - UINT n_rank; - -#define n_thres (n_rank - 1) - - vREAL lm_wgt; - vREAL thres; //of size (n_iter+1) * n_thres - - vREAL dec_value; - - UINT thres_mode; - UINT init_mode; - - public: - explicit AggRank (UINT _n_rank = 2): - thres_mode(THRES_ABSLOSS), init_mode(INIT_ZERO) { - set_n_rank(_n_rank); - } - explicit AggRank (std::istream& is) { is >> *this; } - - virtual AggRank* create () const = 0; - virtual AggRank* clone () const = 0; - virtual bool support_weighted_data () const { return false; } - - REAL model_weight (UINT n) const { return lm_wgt[n]; } - REAL threshold (UINT iter, UINT k) const { - assert(k > 0 && k <= n_thres && iter <= n_in_agg); - return thres[iter * n_thres + (k-1)]; - } - UINT get_n_rank () const { return n_rank; } - - void set_n_rank (UINT _n_rank) { - assert(_n_rank >= 1); n_rank = _n_rank; - } - void set_init_mode (UINT _init_mode) { init_mode = _init_mode; } - void set_thres_mode (UINT _thres_mode) { thres_mode = _thres_mode; } - protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, - const id_t& = NIL_ID); - public: - virtual Output operator() (const Input&, UINT) const; - virtual Output operator() (const Input& _in) const { - return operator()(_in, n_in_agg); - } - - virtual void reset(); - virtual void train () = 0; - virtual void set_reg_param (REAL) = 0; - protected: - //should be in a level of boosting rather than aggregating - pLearnModel learn_weak(const vREAL& rhopos, - const vREAL& rhoneg, - const REAL sum_rhodiff); - void recompute_dec(); - void compute_thres(vREAL::iterator it); - void compute_thres_exploss(vREAL& th, bool ordered, bool full); - void compute_thres_dploss(vREAL& th, bool do_abs); - }; -} // namespace lemga - -#ifdef __AGGRANK_H__ -#warning "This header file may conflict with another `aggrank.h' file." -#endif -#define __AGGRANK_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/boostrank-predict.cpp b/src/Algorithms/orensemble/orensemble/boostrank-predict.cpp deleted file mode 100644 index 765d0b4..0000000 --- a/src/Algorithms/orensemble/orensemble/boostrank-predict.cpp +++ /dev/null @@ -1,150 +0,0 @@ -/** - boostrank-predict.cpp: main file for performing ordinal regression testing - with thresholded ensemble models - (c) 2006-2007 Hsuan-Tien Lin -**/ -#include -#include -#include - -#include "aggrank.h" -#include "orboost.h" -#include "rankboost.h" - -#include -#include -#include - -#include "softperc.h" - -int main (unsigned int argc, char* argv[]) { - if (argc < 7) { - std::cerr << "Usage: " << argv[0] << " datafile n_test" - << " #_input modelfile iter valuefile\n"; - exit(-1); - } - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": data file (" - << argv[1] << ") open error\n"; - exit(-2); - } - UINT n_test = atoi(argv[2]); - UINT n_in = atoi(argv[3]); - UINT n_out = 1; - std::ifstream fm(argv[4]); - if (!fm.is_open()) { - std::cerr << argv[0] << ": model file (" - << argv[4] << ") open error\n"; - exit(-2); - } - UINT iter = atoi(argv[5]); - std::ofstream fv(argv[6]); - - /* load test data */ - lemga::pDataSet ted = lemga::load_data(fd, n_test, n_in, n_out); - fd.close(); - - lemga::AggRank *pbag = (lemga::AggRank*) Object::create(fm); - fm.close(); - - UINT n_rank = pbag->get_n_rank(); - std::vector out(ted->size()); - std::vector< std::set > group(n_rank); - std::vector tbl(n_rank * n_rank); - - REAL ae(0.0), ce(0.0), rl(0.0), tl(0.0); - - REAL n_crit(0.0); - { - std::vector n_goal(n_rank); - - n_goal.clear(); - n_goal.insert(n_goal.begin(), n_rank, 0); - for (UINT i = 0; i < ted->size(); ++i){ - UINT goal = (UINT)(ted->y(i)[0] - 1); - n_goal[goal] ++; - } - - REAL larger = ted->size(); - for(UINT k = 0; k < n_rank; k++){ - larger -= n_goal[k]; - n_crit += n_goal[k] * larger; - } - } - - for(UINT k = 0; k < n_rank; k++) - group[k].clear(); - - tbl.clear(); - tbl.insert(tbl.begin(), n_rank * n_rank, 0); - - for (UINT i = 0; i < ted->size(); ++i){ - out[i] = (*pbag)(ted->x(i), iter); - UINT goal = (UINT)(ted->y(i)[0] - 1); - UINT pred = (UINT)(out[i][0] - 1); - - fv << out[i][0] << ' ' << out[i][1] << std::endl; - - group[pred].insert(i); - tbl[pred * n_rank + goal]++; - } - - for(UINT pred=0;pred pred2: absolutely wrong - tlt += tbl[pred1*n_rank + goal1] * tbl[pred2*n_rank + goal2]; - rlt += tbl[pred1*n_rank + goal1] * tbl[pred2*n_rank + goal2]; - } - } - } - } - - for(UINT pred=0;pred::iterator j = group[pred].begin(); - j != group[pred].end(); ++j){ - for(std::set::iterator k = group[pred].begin(); - k != group[pred].end(); ++k){ - UINT ij = (*j); - UINT ik = (*k); - if (ted->y(ij)[0] < ted->y(ik)[0]){ - tlt += 0.5; //thresholded rank loss - - REAL diff = out[ij][1] - out[ik][1]; - if (diff > 0) - rlt ++; - else if (diff == 0) - rlt += 0.5; - } - } - } - } - - ae = ae / n_test; - ce = ce / n_test; - tl = tlt / n_crit; - rl = rlt / n_crit; - - std::cout << "Absolute Error: " << ae << std::endl; - std::cout << "Classification Error: " << ce << std::endl; - std::cout << "Raw Ranking Loss: " << rl << std::endl; - std::cout << "Thresholded Ranking Loss: " << tl << std::endl; - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/boostrank-train-crossval.cpp b/src/Algorithms/orensemble/orensemble/boostrank-train-crossval.cpp deleted file mode 100644 index 9d9116a..0000000 --- a/src/Algorithms/orensemble/orensemble/boostrank-train-crossval.cpp +++ /dev/null @@ -1,211 +0,0 @@ -/** - boostrank-train.cpp: main file for performing ordinal regression training - with various thresholded ensemble algorithms - (c) 2006-2007 Hsuan-Tien Lin -**/ -#include -#include - -#include "aggrank.h" -#include "orboost.h" -#include "rankboost.h" -#include -#include -#include - -#include -#include - -#include "softperc.h" - -void exit_error(char** argv) { - std::cerr << "Usage: " << argv[0] << " trainfile n_train" - << " #_input bag base n_rank iter modelfile\n" - << "bag : 10 = RankBoost, cla_thres, reg_param=0.0\n" - << " 11 = RankBoost, cla_thres, reg_param=1e-32\n" - << " 20 = RankBoost, abs_thres, reg_param=0.0\n" - << " 21 = RankBoost, abs_thres, reg_param=1e-32\n" - << " 30 = ORBoost, FORM_LR, ordered, sub_iter=1, reg_param=0.0\n" - << " 31 = ORBoost, FORM_LR, ordered, sub_iter=1, reg_param=1e-32\n" - << " 40 = ORBoost, FORM_FULL, ordered, sub_iter=1, reg_param=0.0\n" - << " 41 = ORBoost, FORM_FULL, ordered, sub_iter=1, reg_param=1e-32\n" - << "base: 100 = stump (without constant)\n" - << " 200 = perc200 with special bias\n" - << " 201 = perc200 with special bias, scale=1\n" - << " 204 = perc200 with special bias, scale=4\n" - << " 3HH = neural net, number of hidden neurons=HH\n"; - exit(-1); -} - -int main (unsigned int argc, char* argv[]) { - if (argc < 9) - exit_error(argv); - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": training data file (" - << argv[1] << ") open error\n"; - exit(-2); - } - UINT n_train = atoi(argv[2]); - UINT n_in = atoi(argv[3]); - UINT n_out = 1; - UINT bag = atoi(argv[4]); - UINT base = atoi(argv[5]); - UINT n_rank = atoi(argv[6]); - UINT n_iter = atoi(argv[7]); - std::ofstream fm(argv[8]); - - /* load training data */ - lemga::DataSet *trd = lemga::load_data(fd, n_train, n_in, n_out); - fd.close(); - - lemga::LearnModel *pst = 0; - - switch(base){ - case 100: - { - lemga::Stump* p = new lemga::Stump(n_in); - pst = p; - break; - } - case 200: - { - lemga::Perceptron* p = new lemga::Perceptron(n_in); - p->set_parameter(0, 0, 200); - p->set_train_method(lemga::Perceptron::RAND_COOR_DESCENT_BIAS); - pst = p; - break; - } - case 201: - { - lemga::SoftPerc* p = new lemga::SoftPerc(n_in); - p->set_parameter(0, 0, 200); - p->set_train_method(lemga::Perceptron::RAND_COOR_DESCENT_BIAS); - p->set_scale(1); - pst = p; - break; - } - case 204: - { - lemga::SoftPerc* p = new lemga::SoftPerc(n_in); - p->set_parameter(0, 0, 200); - p->set_train_method(lemga::Perceptron::RAND_COOR_DESCENT_BIAS); - p->set_scale(4); - pst = p; - break; - } - default: - - if(base/300!=1) - { - std::cerr << base << " is not a correct base learner\n"; - exit(-1); - } - } - - - lemga::AggRank *pbag = 0; - - switch(bag/10){ - case 1: - { - lemga::RankBoost* p = new lemga::RankBoost(n_rank); - p->set_thres_mode(lemga::AggRank::THRES_CLALOSS); - pbag = p; - break; - } - case 2: - { - lemga::RankBoost* p = new lemga::RankBoost(n_rank); - p->set_thres_mode(lemga::AggRank::THRES_ABSLOSS); - pbag = p; - break; - } - case 3: - { - lemga::ORBoost* p = new lemga::ORBoost(n_rank); - p->set_sub_iter(1); - p->set_ordered(true); - p->set_form(lemga::ORBoost::FORM_LR); - pbag = p; - break; - } - case 4: - { - lemga::ORBoost* p = new lemga::ORBoost(n_rank); - p->set_sub_iter(1); - p->set_ordered(true); - p->set_form(lemga::ORBoost::FORM_FULL); - pbag = p; - break; - } - } - - - if(base/300!=1) - { - - if (bag % 10 == 1) - pbag->set_reg_param(1e-32); - else - pbag->set_reg_param(0.0); - - pbag->set_base_model(*pst); - pbag->set_max_models(n_iter); - pbag->reset(); - - pbag->set_train_data(trd); - pbag->train(); - } - else{ - - /* set up cross-validation */ - set_seed(0); - lemga::vFoldCrossVal cv; - cv.set_folds(5); // folds = 5 - - /* set up different versions of the ORBoost */ - int vv[6]={5,10,15,20,30,40}; - for (UINT i = 0; i < 6; i++) { - lemga::FeedForwardNN* p = new lemga::FeedForwardNN(); - lemga::NNLayer l(n_in, vv[i]); - l.set_weight_range(-1, 1); - p->add_top(l); - lemga::NNLayer lout(vv[i],1); - lout.set_weight_range(-1, 1); - p->add_top(lout); - p->initialize(); - p->set_train_method(lemga::FeedForwardNN::CONJUGATE_GRADIENT); - p->set_parameter(0.1, 1e-7, 200); - - lemga::AggRank *pbag_i = pbag->clone(); - if (bag % 10 == 1) - pbag_i->set_reg_param(1e-32); - else - pbag_i->set_reg_param(0.0); - pbag_i->set_base_model(*p); - pbag_i->set_max_models(n_iter); - pbag_i->reset(); - - cv.add_model(*pbag_i); - } - cv.set_train_data(trd); - cv.set_full_train(false); - cv.train(); - - pbag = (lemga::AggRank *) cv.best_model().clone(); - pbag->reset(); - - pbag->set_train_data(trd); - pbag->train(); - - } - - fm << (*pbag); - fm.close(); - - // let C++ free things - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/boostrank-train.cpp b/src/Algorithms/orensemble/orensemble/boostrank-train.cpp deleted file mode 100644 index 0ecd981..0000000 --- a/src/Algorithms/orensemble/orensemble/boostrank-train.cpp +++ /dev/null @@ -1,177 +0,0 @@ -/** - boostrank-train.cpp: main file for performing ordinal regression training - with various thresholded ensemble algorithms - (c) 2006-2007 Hsuan-Tien Lin -**/ -#include -#include - -#include "aggrank.h" -#include "orboost.h" -#include "rankboost.h" -#include -#include - -#include -#include - -#include "softperc.h" - -void exit_error(char** argv) { - std::cerr << "Usage: " << argv[0] << " trainfile n_train" - << " #_input bag base n_rank iter modelfile\n" - << "bag : 10 = RankBoost, cla_thres, reg_param=0.0\n" - << " 11 = RankBoost, cla_thres, reg_param=1e-32\n" - << " 20 = RankBoost, abs_thres, reg_param=0.0\n" - << " 21 = RankBoost, abs_thres, reg_param=1e-32\n" - << " 30 = ORBoost, FORM_LR, ordered, sub_iter=1, reg_param=0.0\n" - << " 31 = ORBoost, FORM_LR, ordered, sub_iter=1, reg_param=1e-32\n" - << " 40 = ORBoost, FORM_FULL, ordered, sub_iter=1, reg_param=0.0\n" - << " 41 = ORBoost, FORM_FULL, ordered, sub_iter=1, reg_param=1e-32\n" - << "base: 100 = stump (without constant)\n" - << " 200 = perc200 with special bias\n" - << " 201 = perc200 with special bias, scale=1\n" - << " 204 = perc200 with special bias, scale=4\n" - << " 3HH = neural net, number of hidden neurons=HH\n"; - exit(-1); -} - -int main (unsigned int argc, char* argv[]) { - if (argc < 9) - exit_error(argv); - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": training data file (" - << argv[1] << ") open error\n"; - exit(-2); - } - UINT n_train = atoi(argv[2]); - UINT n_in = atoi(argv[3]); - UINT n_out = 1; - UINT bag = atoi(argv[4]); - UINT base = atoi(argv[5]); - UINT n_rank = atoi(argv[6]); - UINT n_iter = atoi(argv[7]); - std::ofstream fm(argv[8]); - - /* load training data */ - lemga::DataSet *trd = lemga::load_data(fd, n_train, n_in, n_out); - fd.close(); - - - lemga::LearnModel *pst = 0; - - switch(base){ - case 100: - { - lemga::Stump* p = new lemga::Stump(n_in); - pst = p; - break; - } - case 200: - { - lemga::Perceptron* p = new lemga::Perceptron(n_in); - p->set_parameter(0, 0, 200); - p->set_train_method(lemga::Perceptron::RAND_COOR_DESCENT_BIAS); - pst = p; - break; - } - case 201: - { - lemga::SoftPerc* p = new lemga::SoftPerc(n_in); - p->set_parameter(0, 0, 200); - p->set_train_method(lemga::Perceptron::RAND_COOR_DESCENT_BIAS); - p->set_scale(1); - pst = p; - break; - } - case 204: - { - lemga::SoftPerc* p = new lemga::SoftPerc(n_in); - p->set_parameter(0, 0, 200); - p->set_train_method(lemga::Perceptron::RAND_COOR_DESCENT_BIAS); - p->set_scale(4); - pst = p; - break; - } - default: - if(base/300==1) - { - lemga::FeedForwardNN* p = new lemga::FeedForwardNN(); - lemga::NNLayer l(n_in, base%300); - l.set_weight_range(-1, 1); - p->add_top(l); - lemga::NNLayer lout(base%300,1); - lout.set_weight_range(-1, 1); - p->add_top(lout); - p->set_train_method(lemga::FeedForwardNN::CONJUGATE_GRADIENT); - p->set_parameter(0.1, 1e-7, 200); // 1e-10, 2000 - p->initialize(); - pst = p; - break; - } - else - { - std::cerr << base << " is not a correct base learner\n"; - exit(-1); - } - } - - - lemga::AggRank *pbag = 0; - - switch(bag/10){ - case 1: - { - lemga::RankBoost* p = new lemga::RankBoost(n_rank); - p->set_thres_mode(lemga::AggRank::THRES_CLALOSS); - pbag = p; - break; - } - case 2: - { - lemga::RankBoost* p = new lemga::RankBoost(n_rank); - p->set_thres_mode(lemga::AggRank::THRES_ABSLOSS); - pbag = p; - break; - } - case 3: - { - lemga::ORBoost* p = new lemga::ORBoost(n_rank); - p->set_sub_iter(1); - p->set_ordered(true); - p->set_form(lemga::ORBoost::FORM_LR); - pbag = p; - break; - } - case 4: - { - lemga::ORBoost* p = new lemga::ORBoost(n_rank); - p->set_sub_iter(1); - p->set_ordered(true); - p->set_form(lemga::ORBoost::FORM_FULL); - pbag = p; - break; - } - } - - if (bag % 10 == 1) - pbag->set_reg_param(1e-32); - else - pbag->set_reg_param(0.0); - - pbag->set_base_model(*pst); - pbag->set_max_models(n_iter); - pbag->reset(); - - pbag->set_train_data(trd); - pbag->train(); - - fm << (*pbag); - fm.close(); - - // let C++ free things - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/h b/src/Algorithms/orensemble/orensemble/h deleted file mode 100644 index 9876b9c..0000000 --- a/src/Algorithms/orensemble/orensemble/h +++ /dev/null @@ -1,801 +0,0 @@ ---*....... cost = 0.275925, step = 0.0246094 -epoch 1, cost = 0.275925 ----*........ cost = 0.261552, step = 0.00947266 ------*........ cost = 0.260707, step = 0.00322266 --*........ cost = 0.257274, step = 0.025 ----*......... cost = 0.25691, step = 0.00454102 -*....... cost = 0.235074, step = 0.0914063 ------*........ cost = 0.230901, step = 0.00211182 ------*........ cost = 0.223336, step = 0.00400391 ----*......... cost = 0.217723, step = 0.00292969 ------*........ cost = 0.215447, step = 0.00322266 ------*........ cost = 0.2143, step = 0.00229492 ------*........ cost = 0.213804, step = 0.00166016 -----*........ cost = 0.213152, step = 0.00432129 -*....... cost = 0.200341, step = 0.0984375 --------*....... cost = 0.196507, step = 0.000732422 --------*....... cost = 0.193375, step = 0.000622559 ------*........ cost = 0.191508, step = 0.00148926 -------*........ cost = 0.190512, step = 0.000915527 -----*........ cost = 0.188062, step = 0.00473633 -----*........ cost = 0.186522, step = 0.00310059 ------*........ cost = 0.183299, step = 0.00194092 -epoch 21, cost = 0.183299 -------*........ cost = 0.182195, step = 0.000939941 ------*....... cost = 0.181477, step = 0.00266113 ----*........ cost = 0.180682, step = 0.00927734 ------*........ cost = 0.179733, step = 0.00202637 ------*........ cost = 0.179396, step = 0.00164795 ------*....... cost = 0.178784, step = 0.00246582 -----*........ cost = 0.178254, step = 0.0046875 -------*....... cost = 0.178005, step = 0.00128174 -----*........ cost = 0.177721, step = 0.00341797 --*........ cost = 0.175074, step = 0.0273438 ------*........ cost = 0.173993, step = 0.00175781 -------*........ cost = 0.173661, step = 0.000970459 ------*....... cost = 0.17253, step = 0.00253906 -------*....... cost = 0.172198, step = 0.00136719 -----*........ cost = 0.170135, step = 0.00480957 -------*....... cost = 0.169045, step = 0.00126953 ------*....... cost = 0.168167, step = 0.00249023 -------*....... cost = 0.167687, step = 0.0013916 ----*........ cost = 0.166541, step = 0.00742187 --------*....... cost = 0.166226, step = 0.000732422 -epoch 41, cost = 0.166226 ------*........ cost = 0.165927, step = 0.00220947 ------*........ cost = 0.165718, step = 0.003125 ------*........ cost = 0.165225, step = 0.0013916 -------*........ cost = 0.164863, step = 0.00109863 -------*....... cost = 0.16444, step = 0.00144043 ---*....... cost = 0.163955, step = 0.0201172 -------*........ cost = 0.16373, step = 0.00119629 -----*........ cost = 0.163045, step = 0.00456543 ----*........ cost = 0.161671, step = 0.00844727 -----*........ cost = 0.160837, step = 0.00419922 -----*........ cost = 0.160354, step = 0.0046875 ----*........ cost = 0.159565, step = 0.00576172 -------*....... cost = 0.159347, step = 0.00137939 ------*....... cost = 0.158888, step = 0.00292969 -------*....... cost = 0.158746, step = 0.00140381 -----*........ cost = 0.158476, step = 0.00288086 ----*........ cost = 0.158265, step = 0.00664063 ------*....... cost = 0.157982, step = 0.00270996 ------*....... cost = 0.157779, step = 0.00307617 ----*........ cost = 0.157476, step = 0.00742187 -epoch 61, cost = 0.157476 -----*........ cost = 0.15707, step = 0.00395508 ----*....... cost = 0.155808, step = 0.0121094 ------*........ cost = 0.155044, step = 0.00231934 -----*....... cost = 0.153152, step = 0.00566406 -----*........ cost = 0.1526, step = 0.00427246 -----*........ cost = 0.152102, step = 0.00625 ---*........ cost = 0.151579, step = 0.0129883 -----*....... cost = 0.151428, step = 0.00610352 ----*........ cost = 0.151282, step = 0.00615234 ---*........ cost = 0.150793, step = 0.0180664 -----*........ cost = 0.150675, step = 0.00446777 -----*........ cost = 0.150484, step = 0.00625 ------*....... cost = 0.150213, step = 0.00268555 ----*........ cost = 0.149859, step = 0.00957031 -----*....... cost = 0.149656, step = 0.00527344 -----*........ cost = 0.149429, step = 0.00478516 ----*........ cost = 0.149252, step = 0.0132813 -----*........ cost = 0.149039, step = 0.00310059 ------*....... cost = 0.148941, step = 0.00283203 -----*........ cost = 0.148858, step = 0.0046875 -epoch 81, cost = 0.148858 -----*........ cost = 0.148813, step = 0.00625 ------*........ cost = 0.148774, step = 0.00186768 ------*........ cost = 0.148706, step = 0.00234375 -----*........ cost = 0.148668, step = 0.00429688 ---*........ cost = 0.148573, step = 0.0186523 ----*........ cost = 0.148432, step = 0.0074707 -----*....... cost = 0.148213, step = 0.00566406 ------*....... cost = 0.147961, step = 0.00292969 ----*........ cost = 0.147562, step = 0.00703125 -----*........ cost = 0.147479, step = 0.003125 ---*....... cost = 0.147206, step = 0.0234375 ----*....... cost = 0.147082, step = 0.0124023 -----*........ cost = 0.146861, step = 0.00693359 ----*........ cost = 0.146426, step = 0.00722656 ------*....... cost = 0.146303, step = 0.00273438 -----*........ cost = 0.146158, step = 0.00703125 -----*........ cost = 0.146107, step = 0.00463867 ----*........ cost = 0.146008, step = 0.0078125 -----*........ cost = 0.145952, step = 0.00471191 ------*........ cost = 0.145939, step = 0.00390625 -epoch 101, cost = 0.145939 -----*........ cost = 0.145887, step = 0.00429688 -----*........ cost = 0.14581, step = 0.00693359 ------*....... cost = 0.145777, step = 0.00300293 ----*........ cost = 0.145732, step = 0.00625 -----*........ cost = 0.14569, step = 0.00351563 -----*....... cost = 0.145651, step = 0.00507813 -----*....... cost = 0.145631, step = 0.00546875 ----*........ cost = 0.145541, step = 0.00947266 -----*........ cost = 0.145432, step = 0.00385742 ----*........ cost = 0.14533, step = 0.00820313 -----*........ cost = 0.145165, step = 0.00466309 -----*........ cost = 0.145041, step = 0.00410156 ------*........ cost = 0.144979, step = 0.00214844 -----*........ cost = 0.144941, step = 0.00356445 ----*....... cost = 0.144852, step = 0.0111328 -----*........ cost = 0.144789, step = 0.00703125 -----*........ cost = 0.144667, step = 0.00429688 -----*........ cost = 0.144553, step = 0.00351563 ----*........ cost = 0.144478, step = 0.00791016 ------*....... cost = 0.144439, step = 0.00310059 -epoch 121, cost = 0.144439 -----*........ cost = 0.144382, step = 0.00273438 -----*........ cost = 0.144347, step = 0.00373535 ------*........ cost = 0.144335, step = 0.00375977 -----*........ cost = 0.144312, step = 0.00393066 -----*........ cost = 0.144291, step = 0.00471191 -----*....... cost = 0.144269, step = 0.00541992 -----*....... cost = 0.144237, step = 0.00512695 -----*........ cost = 0.144203, step = 0.00463867 ------*....... cost = 0.14419, step = 0.00307617 -----*....... cost = 0.144156, step = 0.00576172 ------*....... cost = 0.144121, step = 0.00300293 -----------------*........ cost = 0.144121, step = 1.52588e-06 ------*........ cost = 0.144114, step = 0.0046875 -------------------*........ cost = 0.144114, step = 3.8147e-07 -------------------------*........ cost = 0.503771, step = 0.00148926 -epoch 1, cost = 0.503771 -----*....... cost = 0.438709, step = 0.0059082 -------*........ cost = 0.431514, step = 0.000952148 ------*........ cost = 0.422071, step = 0.0017334 ------*....... cost = 0.41197, step = 0.00280762 ------*........ cost = 0.407584, step = 0.00175781 ------*....... cost = 0.405051, step = 0.00253906 ------*........ cost = 0.401958, step = 0.00241699 -------*........ cost = 0.400925, step = 0.00115967 -----*........ cost = 0.399604, step = 0.00390625 ------*........ cost = 0.398214, step = 0.00174561 -----*........ cost = 0.396745, step = 0.00324707 -----*........ cost = 0.392765, step = 0.00371094 -------*........ cost = 0.391026, step = 0.00107422 -----*........ cost = 0.388875, step = 0.00456543 ----*........ cost = 0.387415, step = 0.00917969 -------*....... cost = 0.386644, step = 0.00150146 -----*....... cost = 0.383946, step = 0.00566406 ------*........ cost = 0.381522, step = 0.00161133 ---*........ cost = 0.377412, step = 0.0140625 ------*....... cost = 0.374441, step = 0.00275879 -epoch 21, cost = 0.374441 ------*........ cost = 0.372361, step = 0.0015625 -----*........ cost = 0.369524, step = 0.00410156 -----*........ cost = 0.368038, step = 0.00322266 ------*........ cost = 0.365794, step = 0.00228271 ------*........ cost = 0.364789, step = 0.00224609 -----*........ cost = 0.363761, step = 0.00336914 -----*........ cost = 0.361526, step = 0.00341797 ------*........ cost = 0.359445, step = 0.00222168 ------*....... cost = 0.357922, step = 0.00275879 ------*....... cost = 0.355569, step = 0.00266113 ------*........ cost = 0.354448, step = 0.00170898 -----*........ cost = 0.353608, step = 0.00332031 ------*....... cost = 0.353211, step = 0.00273438 -----*........ cost = 0.352642, step = 0.00429688 ----*........ cost = 0.351413, step = 0.00625 ------*........ cost = 0.350488, step = 0.00187988 -----*........ cost = 0.348698, step = 0.00427246 ------*....... cost = 0.345805, step = 0.00263672 ------*........ cost = 0.342799, step = 0.00200195 ------*....... cost = 0.340222, step = 0.0027832 -epoch 41, cost = 0.340222 ------*........ cost = 0.337439, step = 0.00200195 ------*........ cost = 0.336133, step = 0.00205078 ------*........ cost = 0.335181, step = 0.00231934 -----*........ cost = 0.331596, step = 0.00458984 ----*........ cost = 0.329102, step = 0.0078125 -------*........ cost = 0.327817, step = 0.0015625 ------*........ cost = 0.325928, step = 0.00244141 -----*........ cost = 0.324071, step = 0.0036377 -------*....... cost = 0.323286, step = 0.00134277 ------*........ cost = 0.322339, step = 0.00234375 ----*....... cost = 0.320923, step = 0.0101563 ------*....... cost = 0.320237, step = 0.00253906 ------*........ cost = 0.319437, step = 0.00181885 -----*....... cost = 0.317917, step = 0.00527344 ------*........ cost = 0.317073, step = 0.00146484 -----*........ cost = 0.314775, step = 0.00390625 -----*........ cost = 0.313495, step = 0.00437012 -------*....... cost = 0.313205, step = 0.00140381 -----*........ cost = 0.311986, step = 0.00439453 -----*........ cost = 0.310974, step = 0.00351563 -epoch 61, cost = 0.310974 -------*....... cost = 0.310518, step = 0.00146484 ------*....... cost = 0.309705, step = 0.00292969 --------*........ cost = 0.309427, step = 0.00078125 -----*........ cost = 0.308964, step = 0.00410156 -------*........ cost = 0.308703, step = 0.00117188 ------*........ cost = 0.308302, step = 0.00195312 -------*........ cost = 0.308162, step = 0.00112305 ------*........ cost = 0.308017, step = 0.00158691 ----*........ cost = 0.307687, step = 0.0078125 -------*....... cost = 0.307431, step = 0.00135498 -----*........ cost = 0.306808, step = 0.00405273 ------*....... cost = 0.306467, step = 0.00266113 -------*........ cost = 0.306297, step = 0.00107422 -----*........ cost = 0.305914, step = 0.00449219 -----*........ cost = 0.305247, step = 0.0046875 -------*........ cost = 0.305093, step = 0.000927734 -----*........ cost = 0.304633, step = 0.00332031 -----*........ cost = 0.304357, step = 0.00314941 -------*........ cost = 0.304183, step = 0.00175781 ------*........ cost = 0.303812, step = 0.00195312 -epoch 81, cost = 0.303812 -------*........ cost = 0.303684, step = 0.00078125 ------*....... cost = 0.303486, step = 0.00251465 -------*....... cost = 0.30338, step = 0.00131836 ------*....... cost = 0.303176, step = 0.00275879 -----*........ cost = 0.302949, step = 0.00432129 -------*........ cost = 0.302895, step = 0.000878906 -----*....... cost = 0.302788, step = 0.00546875 -----*........ cost = 0.302513, step = 0.00429688 ------*........ cost = 0.302265, step = 0.00195312 ------*........ cost = 0.302073, step = 0.00168457 ------*....... cost = 0.301685, step = 0.00253906 ------*........ cost = 0.301498, step = 0.00195312 ------*........ cost = 0.301295, step = 0.00317383 ------*........ cost = 0.300647, step = 0.00244141 ------*........ cost = 0.300088, step = 0.0015625 ------*....... cost = 0.299647, step = 0.00249023 ------*........ cost = 0.29887, step = 0.00214844 -------*........ cost = 0.298087, step = 0.000976562 --------*....... cost = 0.297767, step = 0.000683594 -------*........ cost = 0.297609, step = 0.00117188 -epoch 101, cost = 0.297609 --------*........ cost = 0.297566, step = 0.000476074 --------*........ cost = 0.297469, step = 0.000982666 --------*....... cost = 0.297441, step = 0.000683594 --------*....... cost = 0.29738, step = 0.000640869 -------*........ cost = 0.297301, step = 0.000823975 --------*....... cost = 0.297282, step = 0.000769043 ------*........ cost = 0.297245, step = 0.00186768 --------*....... cost = 0.297127, step = 0.000695801 ------*........ cost = 0.296999, step = 0.00142822 --------*........ cost = 0.296975, step = 0.0012207 ------*........ cost = 0.296904, step = 0.00155029 ------*....... cost = 0.296739, step = 0.00273438 -------*........ cost = 0.296531, step = 0.000732422 --------*........ cost = 0.296411, step = 0.000982666 --------*........ cost = 0.296383, step = 0.000817871 --------*........ cost = 0.296313, step = 0.000952148 -------*........ cost = 0.296168, step = 0.000946045 -------*........ cost = 0.296112, step = 0.000732422 --------*........ cost = 0.296068, step = 0.000927734 ------------------------*......... cost = 0.569073, step = 0.00239258 -epoch 1, cost = 0.569073 ------*........ cost = 0.48709, step = 0.00327148 -----*........ cost = 0.471992, step = 0.00356445 -------*....... cost = 0.459422, step = 0.00129395 -------*....... cost = 0.458015, step = 0.00129395 -----*........ cost = 0.456186, step = 0.00332031 ------*........ cost = 0.453738, step = 0.00214844 -------*....... cost = 0.45255, step = 0.00141602 ------*....... cost = 0.451461, step = 0.0029541 -------*........ cost = 0.450754, step = 0.00106812 -----*........ cost = 0.450163, step = 0.00385742 -----*........ cost = 0.446514, step = 0.00722656 --------*....... cost = 0.445533, step = 0.000683594 ------*........ cost = 0.444857, step = 0.0022583 ------*........ cost = 0.443391, step = 0.00219727 -------*....... cost = 0.443057, step = 0.00141602 ----*....... cost = 0.441087, step = 0.0120117 ------*........ cost = 0.440261, step = 0.00214844 -------*........ cost = 0.438545, step = 0.0015625 --------*........ cost = 0.437168, step = 0.00078125 -------*....... cost = 0.436368, step = 0.00136719 -epoch 21, cost = 0.436368 ------*........ cost = 0.435544, step = 0.00223389 -------*....... cost = 0.434875, step = 0.00136719 ------*........ cost = 0.433698, step = 0.00180664 ------*....... cost = 0.432915, step = 0.00273438 -------*........ cost = 0.432443, step = 0.000866699 ------*........ cost = 0.43173, step = 0.00195312 -----*........ cost = 0.430186, step = 0.00322266 -------*....... cost = 0.427083, step = 0.00125732 --------*....... cost = 0.426024, step = 0.000653076 ------*........ cost = 0.425032, step = 0.00200195 ------*........ cost = 0.424116, step = 0.00151367 ------*........ cost = 0.423635, step = 0.0017334 ------*........ cost = 0.422536, step = 0.00166016 --------*........ cost = 0.421974, step = 0.00057373 ------*........ cost = 0.421286, step = 0.00192871 -----*........ cost = 0.420715, step = 0.00297852 -------*........ cost = 0.420179, step = 0.00102539 -----*....... cost = 0.418906, step = 0.00620117 -------*....... cost = 0.417597, step = 0.00151367 -------*....... cost = 0.415795, step = 0.00131836 -epoch 41, cost = 0.415795 -------*........ cost = 0.414568, step = 0.00119629 -------*........ cost = 0.413886, step = 0.00090332 ------*........ cost = 0.413364, step = 0.00178223 -------*....... cost = 0.412708, step = 0.00142822 -------*........ cost = 0.411844, step = 0.00107422 ------*........ cost = 0.410859, step = 0.00219727 -------*........ cost = 0.410425, step = 0.00114746 --------*....... cost = 0.410066, step = 0.000634766 -------*........ cost = 0.40961, step = 0.00106812 --------*........ cost = 0.409268, step = 0.00078125 -------*........ cost = 0.409029, step = 0.000756836 -------*........ cost = 0.408874, step = 0.000793457 -------*........ cost = 0.408675, step = 0.00116577 ----*........ cost = 0.408058, step = 0.00942383 --------*....... cost = 0.407656, step = 0.000732422 -------*........ cost = 0.407269, step = 0.000878906 -------*....... cost = 0.407001, step = 0.00131836 -------*........ cost = 0.406618, step = 0.00112305 ------*........ cost = 0.405945, step = 0.00175781 -------*......... cost = 0.405641, step = 0.000582886 -epoch 61, cost = 0.405641 ------*........ cost = 0.405346, step = 0.00230713 -------*....... cost = 0.405174, step = 0.00151367 --------*....... cost = 0.40497, step = 0.000671387 ------*........ cost = 0.40482, step = 0.00209961 -------*........ cost = 0.404687, step = 0.000830078 -------*........ cost = 0.404409, step = 0.00115967 ------*....... cost = 0.403985, step = 0.00251465 -----*........ cost = 0.403263, step = 0.00371094 -------*........ cost = 0.402062, step = 0.00107422 --------*....... cost = 0.401671, step = 0.000769043 ------*....... cost = 0.40004, step = 0.00275879 --------*........ cost = 0.398741, step = 0.000585938 -------*........ cost = 0.397029, step = 0.000964355 -------*........ cost = 0.396454, step = 0.00100098 ------*....... cost = 0.395049, step = 0.00302734 -------*........ cost = 0.394542, step = 0.00078125 -------*....... cost = 0.393681, step = 0.00136719 --------*....... cost = 0.392739, step = 0.000750732 -------*....... cost = 0.391331, step = 0.00148926 --------*........ cost = 0.390518, step = 0.000390625 -epoch 81, cost = 0.390518 -------*........ cost = 0.389519, step = 0.0012207 -------*........ cost = 0.388427, step = 0.00117188 ---------*........ cost = 0.387915, step = 0.000411987 ------*........ cost = 0.386373, step = 0.00200195 ---------*........ cost = 0.386133, step = 0.000396729 -------*........ cost = 0.385532, step = 0.000982666 -------*........ cost = 0.384957, step = 0.00111694 --------*......... cost = 0.384661, step = 0.000292969 -------*........ cost = 0.38431, step = 0.000708008 ------*........ cost = 0.384007, step = 0.00135498 --------*........ cost = 0.383885, step = 0.000463867 ------*....... cost = 0.383058, step = 0.00251465 ---------*........ cost = 0.382798, step = 0.000421143 --------*....... cost = 0.382424, step = 0.000732422 ---------*........ cost = 0.382195, step = 0.000230408 -------*......... cost = 0.382017, step = 0.000585938 ---------*........ cost = 0.381962, step = 0.000228882 -------*....... cost = 0.381863, step = 0.00129395 --------*....... cost = 0.381731, step = 0.000720215 --------*....... cost = 0.381531, step = 0.00067749 -epoch 101, cost = 0.381531 --------*........ cost = 0.381452, step = 0.000564575 ---------*....... cost = 0.381292, step = 0.000387573 ---------*........ cost = 0.381206, step = 0.000216675 -------*....... cost = 0.381106, step = 0.00152588 --------------------*........ cost = 0.381099, step = 9.53674e-08 ---------*........ cost = 0.381018, step = 0.00057373 -------*....... cost = 0.380988, step = 0.00155029 --------*........ cost = 0.380839, step = 0.000585938 --------*........ cost = 0.380757, step = 0.000592041 ------*........ cost = 0.380369, step = 0.00183105 ------*........ cost = 0.379704, step = 0.00235596 -------*........ cost = 0.378686, step = 0.00107422 ---------*........ cost = 0.378199, step = 0.000234985 ---------*........ cost = 0.377742, step = 0.000294495 --------*........ cost = 0.377402, step = 0.000387573 -------*........ cost = 0.377246, step = 0.000878906 -------*....... cost = 0.376964, step = 0.00155029 ---------*........ cost = 0.37683, step = 0.000196838 --------*....... cost = 0.37674, step = 0.000634766 ---------*........ cost = 0.376672, step = 0.000271606 -epoch 121, cost = 0.376672 -------*........ cost = 0.376607, step = 0.00230713 ----------*....... cost = 0.376591, step = 0.000170898 ---------*........ cost = 0.376518, step = 0.000195313 --------*........ cost = 0.376452, step = 0.000927734 ------------------------*........ cost = 0.594205, step = 0.00322266 -epoch 1, cost = 0.594205 ------*........ cost = 0.516324, step = 0.00200195 ------*........ cost = 0.502678, step = 0.00201416 ------*........ cost = 0.482847, step = 0.00179443 -------*........ cost = 0.480004, step = 0.00112305 ------*........ cost = 0.478652, step = 0.00217285 ------*........ cost = 0.477357, step = 0.00205078 -------*........ cost = 0.476464, step = 0.00118408 ------*........ cost = 0.476055, step = 0.00175781 ------*....... cost = 0.475563, step = 0.00253906 -------*........ cost = 0.475427, step = 0.000872803 -------*........ cost = 0.475318, step = 0.00117188 ------*........ cost = 0.47515, step = 0.00311279 ------*......... cost = 0.475099, step = 0.000775146 -----*........ cost = 0.475, step = 0.00314941 -------*........ cost = 0.47496, step = 0.00117188 ---*........ cost = 0.474542, step = 0.0188477 ------*........ cost = 0.473076, step = 0.00244141 --------*....... cost = 0.472508, step = 0.000701904 ----*........ cost = 0.47213, step = 0.00761719 ------*........ cost = 0.47195, step = 0.00311279 -epoch 21, cost = 0.47195 ------*....... cost = 0.471506, step = 0.00273438 ----*........ cost = 0.471241, step = 0.00615234 -------*....... cost = 0.471157, step = 0.00126953 ----*........ cost = 0.470986, step = 0.00761719 ------*........ cost = 0.470817, step = 0.00205078 -----*....... cost = 0.469992, step = 0.00566406 -------*....... cost = 0.46955, step = 0.00131836 -----*........ cost = 0.468444, step = 0.00419922 ------*....... cost = 0.468269, step = 0.00292969 -----*........ cost = 0.467755, step = 0.00432129 -----*....... cost = 0.46654, step = 0.00595703 -------*....... cost = 0.465636, step = 0.00136719 ------*....... cost = 0.465044, step = 0.00288086 ------*........ cost = 0.464505, step = 0.00146484 -----*........ cost = 0.463463, step = 0.0046875 -----*........ cost = 0.46181, step = 0.00319824 ------*........ cost = 0.460363, step = 0.00202637 ------*........ cost = 0.459377, step = 0.00195312 ------*........ cost = 0.458689, step = 0.00239258 ------*........ cost = 0.457687, step = 0.00166016 -epoch 41, cost = 0.457687 -----*....... cost = 0.456425, step = 0.00576172 -------*....... cost = 0.455881, step = 0.00152588 -----*........ cost = 0.454493, step = 0.00361328 -------*........ cost = 0.453654, step = 0.00102539 -----*....... cost = 0.452496, step = 0.00498047 -------*........ cost = 0.451918, step = 0.00111084 -----*........ cost = 0.448935, step = 0.00454102 -----*........ cost = 0.447432, step = 0.00327148 --------*....... cost = 0.446877, step = 0.000683594 -------*....... cost = 0.44668, step = 0.00146484 -----*........ cost = 0.445928, step = 0.00371094 -------*........ cost = 0.445627, step = 0.000793457 ----*........ cost = 0.44456, step = 0.00615234 --------*........ cost = 0.444321, step = 0.00078125 ------*....... cost = 0.444041, step = 0.00292969 -------*........ cost = 0.443791, step = 0.00078125 ------*........ cost = 0.443526, step = 0.00216064 -------*....... cost = 0.443145, step = 0.00131836 -------*........ cost = 0.443069, step = 0.000616455 ------*........ cost = 0.443008, step = 0.0015625 -epoch 61, cost = 0.443008 -------*........ cost = 0.442935, step = 0.00106201 -----*........ cost = 0.442807, step = 0.00307617 ----*....... cost = 0.442427, step = 0.0103516 -------*........ cost = 0.442269, step = 0.000872803 ------*....... cost = 0.441658, step = 0.0027832 -------*........ cost = 0.441389, step = 0.000976562 ------*........ cost = 0.441252, step = 0.00175781 ------*....... cost = 0.4409, step = 0.0026123 ------*........ cost = 0.440726, step = 0.0015625 -----*....... cost = 0.440084, step = 0.00498047 --------*....... cost = 0.439931, step = 0.000646973 --------*........ cost = 0.43987, step = 0.000946045 ------*........ cost = 0.439757, step = 0.00238037 -------*........ cost = 0.439611, step = 0.000805664 -------*........ cost = 0.439552, step = 0.000787354 ------*........ cost = 0.43952, step = 0.00195312 -------*........ cost = 0.439465, step = 0.00114136 ------*........ cost = 0.439407, step = 0.00205078 ------*....... cost = 0.439247, step = 0.00249023 -------*........ cost = 0.439152, step = 0.000982666 -epoch 81, cost = 0.439152 ------*........ cost = 0.438957, step = 0.003125 -------*........ cost = 0.438765, step = 0.00107422 --------*....... cost = 0.438607, step = 0.000726318 -------*....... cost = 0.43854, step = 0.00128174 -------*........ cost = 0.43846, step = 0.00112305 -------*........ cost = 0.438395, step = 0.000872803 -----*........ cost = 0.438274, step = 0.00429688 -------*....... cost = 0.438215, step = 0.0013916 --------*....... cost = 0.438173, step = 0.000708008 --------*........ cost = 0.43809, step = 0.000482178 ------*....... cost = 0.438019, step = 0.00253906 -------------*........ cost = 0.437982, step = 1.67847e-05 --------------------**........ cost = 0.437981, step = 5.72205e-07 ------------------------*........ cost = 0.586245, step = 0.00693359 -epoch 1, cost = 0.586245 --------*....... cost = 0.560159, step = 0.00065918 ------*........ cost = 0.524981, step = 0.00195312 -----*........ cost = 0.49675, step = 0.00305176 -------*........ cost = 0.492404, step = 0.00110474 ------*....... cost = 0.489646, step = 0.00268555 -------*........ cost = 0.488364, step = 0.00100098 -----*........ cost = 0.487722, step = 0.00351563 ------*........ cost = 0.486669, step = 0.00214844 ------*....... cost = 0.486234, step = 0.0026123 ------*........ cost = 0.485447, step = 0.00311279 ------*....... cost = 0.48533, step = 0.00283203 ------*........ cost = 0.485127, step = 0.00217285 -------*....... cost = 0.485044, step = 0.00134277 -----*........ cost = 0.484912, step = 0.00319824 ------*........ cost = 0.484835, step = 0.00238037 ------*........ cost = 0.484759, step = 0.00234375 -----*........ cost = 0.484643, step = 0.003125 ------*........ cost = 0.484517, step = 0.003125 -------*........ cost = 0.484437, step = 0.00180664 -------*........ cost = 0.484267, step = 0.00117188 -epoch 21, cost = 0.484267 ------*........ cost = 0.484101, step = 0.00213623 -------*........ cost = 0.484046, step = 0.00117188 -----*....... cost = 0.483972, step = 0.00507813 -----*........ cost = 0.483886, step = 0.0048584 ------*........ cost = 0.483709, step = 0.00234375 ------*....... cost = 0.483622, step = 0.00263672 -----*........ cost = 0.483016, step = 0.00703125 -------*........ cost = 0.48294, step = 0.0015625 -------*....... cost = 0.482892, step = 0.0013916 --------*........ cost = 0.482835, step = 0.00078125 ----*........ cost = 0.48275, step = 0.00751953 -------*........ cost = 0.482644, step = 0.00119629 ------*....... cost = 0.482577, step = 0.00302734 ----------*....... cost = 0.482568, step = 0.000189209 --------*........ cost = 0.482539, step = 0.000830078 --------*........ cost = 0.482527, step = 0.000390625 ------------------------*........ cost = 0.57724, step = 0.00732422 -epoch 1, cost = 0.57724 --------*....... cost = 0.559659, step = 0.000665283 ------*........ cost = 0.524427, step = 0.00216064 ------*........ cost = 0.501492, step = 0.00214844 -------*....... cost = 0.496864, step = 0.00131836 ------*....... cost = 0.493963, step = 0.00302734 -------*........ cost = 0.492526, step = 0.00120239 ------*........ cost = 0.492143, step = 0.00175781 -----*........ cost = 0.491271, step = 0.0036377 -------*........ cost = 0.490994, step = 0.00100098 -----*........ cost = 0.490606, step = 0.0046875 ------*........ cost = 0.490178, step = 0.00185547 -------*....... cost = 0.490067, step = 0.00128174 -----*........ cost = 0.489865, step = 0.00476074 --------*....... cost = 0.489826, step = 0.000775146 ------*........ cost = 0.489795, step = 0.003125 ------*........ cost = 0.489724, step = 0.00195312 -------*....... cost = 0.489701, step = 0.00136719 ---------*........ cost = 0.489683, step = 0.000402832 -------------------*........ cost = 0.489681, step = 1.90735e-07 ------------------------*........ cost = 0.572815, step = 0.00737305 -epoch 1, cost = 0.572815 --------*....... cost = 0.557752, step = 0.000732422 ------*........ cost = 0.521657, step = 0.00229492 -------*........ cost = 0.503955, step = 0.0015564 ------*........ cost = 0.499147, step = 0.00175781 ------*........ cost = 0.496613, step = 0.00166016 ------*........ cost = 0.495023, step = 0.0015625 -------*....... cost = 0.494496, step = 0.00137939 -------*........ cost = 0.494232, step = 0.0015625 -------*....... cost = 0.493853, step = 0.00136719 ------*........ cost = 0.493616, step = 0.00195312 ------*........ cost = 0.493312, step = 0.00235596 -------*....... cost = 0.493096, step = 0.00126953 -------*....... cost = 0.492964, step = 0.00146484 -----*........ cost = 0.492622, step = 0.00351563 ----*........ cost = 0.492363, step = 0.0078125 -------*....... cost = 0.492272, step = 0.00136719 -----*........ cost = 0.492228, step = 0.00410156 -------*....... cost = 0.492198, step = 0.0013916 -----*........ cost = 0.492076, step = 0.00664063 -------*........ cost = 0.491996, step = 0.000958252 -epoch 21, cost = 0.491996 ----------***....... cost = 0.491954, step = 0.00113525 -----*....... cost = 0.491928, step = 0.00546875 ------*........ cost = 0.491888, step = 0.00239258 ----*........ cost = 0.491754, step = 0.00664063 -------*....... cost = 0.49161, step = 0.00123291 -----*........ cost = 0.491521, step = 0.00488281 -------*....... cost = 0.49141, step = 0.00126953 -------*........ cost = 0.491371, step = 0.000793457 -----*........ cost = 0.491317, step = 0.00449219 --------*........ cost = 0.491298, step = 0.00078125 -----*........ cost = 0.491261, step = 0.00622559 --------------*........ cost = 0.491248, step = 1.44005e-05 --------*....... cost = 0.491233, step = 0.000762939 -----*........ cost = 0.49119, step = 0.00622559 ---------*....... cost = 0.491167, step = 0.000378418 -------*....... cost = 0.491142, step = 0.00148926 ------*....... cost = 0.491116, step = 0.00251465 -------*........ cost = 0.491084, step = 0.00090332 ------*........ cost = 0.491037, step = 0.00390625 -------*......... cost = 0.491015, step = 0.000466919 -epoch 41, cost = 0.491015 --------*........ cost = 0.491006, step = 0.000793457 ----*....... cost = 0.490934, step = 0.0109375 ------*........ cost = 0.490815, step = 0.0019165 -----*....... cost = 0.490273, step = 0.00566406 -------*........ cost = 0.489902, step = 0.00117188 -------*........ cost = 0.48972, step = 0.000732422 -------*........ cost = 0.489667, step = 0.00115356 -----*....... cost = 0.489613, step = 0.00620117 ------------------------*........ cost = 0.570925, step = 0.00732422 -epoch 1, cost = 0.570925 -------*........ cost = 0.556958, step = 0.000787354 ------*....... cost = 0.517134, step = 0.00270996 -------*........ cost = 0.504406, step = 0.00120239 ------*........ cost = 0.500454, step = 0.00229492 ------*........ cost = 0.497269, step = 0.0017334 -------*........ cost = 0.496457, step = 0.00108643 -------*........ cost = 0.496204, step = 0.00195312 ------*........ cost = 0.495909, step = 0.00147705 -------*........ cost = 0.495642, step = 0.00109863 ------*........ cost = 0.495511, step = 0.00231934 ------*........ cost = 0.495304, step = 0.00197754 ------*........ cost = 0.495176, step = 0.00177002 -----*........ cost = 0.494958, step = 0.00314941 -------*....... cost = 0.494725, step = 0.00145264 -------*........ cost = 0.49467, step = 0.0015625 -----*........ cost = 0.494493, step = 0.0046875 -------*........ cost = 0.494408, step = 0.000891113 ------*........ cost = 0.494352, step = 0.0015625 ------*....... cost = 0.494325, step = 0.00251465 ------*........ cost = 0.494304, step = 0.00178223 -epoch 21, cost = 0.494304 -------*........ cost = 0.494284, step = 0.00198975 ------*........ cost = 0.494238, step = 0.00211182 --------------*........ cost = 0.494235, step = 6.10352e-06 ----------------*........ cost = 0.494235, step = 3.05176e-06 --------------------*........ cost = 0.494235, step = 1.90735e-07 -----------------------*........ cost = 0.570526, step = 0.00722656 -epoch 1, cost = 0.570526 -------*........ cost = 0.556213, step = 0.000811768 -----*........ cost = 0.513147, step = 0.00324707 -------*........ cost = 0.504449, step = 0.00100098 ------*....... cost = 0.5007, step = 0.00273438 -------*....... cost = 0.498045, step = 0.00137939 -------*........ cost = 0.497642, step = 0.00107422 ----*........ cost = 0.496773, step = 0.00546875 -------*....... cost = 0.496691, step = 0.00136719 ------*....... cost = 0.496511, step = 0.00302734 -------*........ cost = 0.496364, step = 0.000976562 ----*....... cost = 0.496061, step = 0.00996094 -------*........ cost = 0.495898, step = 0.00106201 -----*........ cost = 0.495737, step = 0.0034668 ------*........ cost = 0.495532, step = 0.00206299 -------*....... cost = 0.49549, step = 0.0013916 ------*....... cost = 0.495416, step = 0.00302734 -------*....... cost = 0.495381, step = 0.00146484 --------*........ cost = 0.49536, step = 0.00113525 ----*........ cost = 0.495297, step = 0.0078125 -------*........ cost = 0.495249, step = 0.000976562 -epoch 21, cost = 0.495249 ---*........ cost = 0.495151, step = 0.015625 -------------*........ cost = 0.495144, step = 2.59399e-05 ------------------*........ cost = 0.495144, step = 7.62939e-07 -----------------------*........ cost = 0.570811, step = 0.00722656 -epoch 1, cost = 0.570811 -------*........ cost = 0.556857, step = 0.000830078 -----*........ cost = 0.51045, step = 0.00358887 -------*........ cost = 0.505055, step = 0.000976562 -----*........ cost = 0.500606, step = 0.00332031 -------*........ cost = 0.49885, step = 0.00117188 -------*....... cost = 0.498424, step = 0.00150146 ------*........ cost = 0.497651, step = 0.003125 ----*........ cost = 0.4973, step = 0.00629883 -------*....... cost = 0.49719, step = 0.00140381 -------*........ cost = 0.497163, step = 0.0015625 -------*........ cost = 0.497096, step = 0.00117188 ----*........ cost = 0.496953, step = 0.00703125 -------*....... cost = 0.496784, step = 0.00136719 ------*........ cost = 0.496706, step = 0.0015625 ------*....... cost = 0.496668, step = 0.00310059 -----------------------*........ cost = 0.571183, step = 0.00722656 -epoch 1, cost = 0.571183 -------*........ cost = 0.557309, step = 0.000811768 -----*........ cost = 0.509545, step = 0.00375977 -------*....... cost = 0.505218, step = 0.00131836 ------*....... cost = 0.500269, step = 0.00302734 -------*........ cost = 0.499035, step = 0.00107422 -------*........ cost = 0.498715, step = 0.0015625 ------*....... cost = 0.498077, step = 0.00292969 ------*........ cost = 0.497937, step = 0.00151367 -----*........ cost = 0.49778, step = 0.00422363 -------*........ cost = 0.497676, step = 0.000976562 -------*........ cost = 0.497605, step = 0.000787354 -----*....... cost = 0.497531, step = 0.00507813 -------*........ cost = 0.497506, step = 0.000775146 -----*....... cost = 0.49744, step = 0.0050293 --------*........ cost = 0.497409, step = 0.00115967 -----*....... cost = 0.497285, step = 0.00615234 -------*....... cost = 0.497208, step = 0.00150146 ----*........ cost = 0.497069, step = 0.00712891 ------*....... cost = 0.496777, step = 0.00253906 -------*........ cost = 0.496693, step = 0.00116577 -epoch 21, cost = 0.496693 -------------*......... cost = 0.49668, step = 7.62939e-06 -----------------------*........ cost = 0.57199, step = 0.00727539 -epoch 1, cost = 0.57199 -------*........ cost = 0.558481, step = 0.000805664 -----*........ cost = 0.509747, step = 0.00375977 ------*........ cost = 0.5042, step = 0.00244141 ------*........ cost = 0.499818, step = 0.00202637 -------*........ cost = 0.49932, step = 0.00103149 -----*........ cost = 0.498903, step = 0.0032959 ------*........ cost = 0.498338, step = 0.00222168 -------*........ cost = 0.498229, step = 0.00108032 ------*....... cost = 0.498155, step = 0.00270996 ------*....... cost = 0.498087, step = 0.00273438 ------*........ cost = 0.497985, step = 0.00214844 ------*....... cost = 0.497879, step = 0.00273438 ------*........ cost = 0.497731, step = 0.00212402 -------*........ cost = 0.497656, step = 0.00119019 ----*........ cost = 0.497605, step = 0.00830078 -------*....... cost = 0.497488, step = 0.00128174 ------*....... cost = 0.497406, step = 0.00288086 ------*....... cost = 0.497254, step = 0.00273438 ------*........ cost = 0.497176, step = 0.00184326 ------*....... cost = 0.497063, step = 0.00253906 -epoch 21, cost = 0.497063 --------------------*....... cost = 0.497062, step = 3.33786e-07 -----------------------*........ cost = 0.572885, step = 0.00722656 -epoch 1, cost = 0.572885 -------*........ cost = 0.558984, step = 0.00078125 -----*........ cost = 0.510685, step = 0.00380859 ------*....... cost = 0.504581, step = 0.00258789 ------*........ cost = 0.499908, step = 0.00209961 -------*........ cost = 0.499573, step = 0.00102539 -----*........ cost = 0.499084, step = 0.00458984 ------*........ cost = 0.498623, step = 0.0015625 -------*........ cost = 0.498555, step = 0.00106201 ----*........ cost = 0.498451, step = 0.00546875 ------*........ cost = 0.498366, step = 0.00236816 ---------*........ cost = 0.498341, step = 0.000442505 -------*........ cost = 0.49824, step = 0.000939941 ------*....... cost = 0.498218, step = 0.00310059 ------*........ cost = 0.498174, step = 0.00212402 ------*........ cost = 0.498135, step = 0.00410156 -------*........ cost = 0.498111, step = 0.00157471 --------*....... cost = 0.498054, step = 0.000683594 -----*........ cost = 0.497956, step = 0.00466309 -------------------*........ cost = 0.497956, step = 3.8147e-07 ------------*........ cost = 0.497954, step = 4.88281e-05 -epoch 21, cost = 0.497954 -------------------*........ cost = 0.497954, step = 3.8147e-07 --------------------*........ cost = 0.497954, step = 1.90735e-07 -----------------------*........ cost = 0.573381, step = 0.00727539 -epoch 1, cost = 0.573381 -------*........ cost = 0.559825, step = 0.000775146 -----*........ cost = 0.512021, step = 0.00366211 ------*........ cost = 0.505971, step = 0.00203857 ------*........ cost = 0.500868, step = 0.00229492 -------*........ cost = 0.499897, step = 0.0010437 ------*........ cost = 0.499592, step = 0.00219727 ------*....... cost = 0.499107, step = 0.00268555 -------*....... cost = 0.498827, step = 0.00136719 -------*....... cost = 0.498758, step = 0.00134277 -----*....... cost = 0.498627, step = 0.00507813 ------*........ cost = 0.498464, step = 0.00175781 -----*........ cost = 0.498408, step = 0.003125 -----------*....... cost = 0.498394, step = 8.7738e-05 --------------------*........ cost = 0.498394, step = 1.90735e-07 -----------------------*........ cost = 0.573966, step = 0.00727539 -epoch 1, cost = 0.573966 -------*........ cost = 0.560314, step = 0.000811768 -----*........ cost = 0.512162, step = 0.00356445 ------*........ cost = 0.50717, step = 0.00158691 ------*....... cost = 0.501135, step = 0.00292969 -------*........ cost = 0.500251, step = 0.000927734 ------*........ cost = 0.499803, step = 0.003125 -------*....... cost = 0.499376, step = 0.00151367 -----*....... cost = 0.499068, step = 0.00507813 -------*....... cost = 0.498889, step = 0.00124512 ------*........ cost = 0.498832, step = 0.00195312 ------*........ cost = 0.498723, step = 0.00195312 ----*....... cost = 0.498662, step = 0.00986328 -------*........ cost = 0.498558, step = 0.00116577 --------*....... cost = 0.498355, step = 0.000683594 ------*......... cost = 0.498315, step = 0.00078125 -----*....... cost = 0.498223, step = 0.00507813 ------*........ cost = 0.498178, step = 0.00155029 ----*........ cost = 0.498082, step = 0.00820313 --------*........ cost = 0.497998, step = 0.000982666 --------*....... cost = 0.49795, step = 0.00136719 -epoch 21, cost = 0.49795 ------*........ cost = 0.497872, step = 0.00189209 --------*........ cost = 0.497797, step = 0.000585938 ---------------*........ cost = 0.497792, step = 6.86646e-06 -------------------*........ cost = 0.497792, step = 3.8147e-07 ----------------*........ cost = 0.497791, step = 3.05176e-06 --------------------*........ cost = 0.497791, step = 1.90735e-07 -------------------- \ No newline at end of file diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/LEMGA.dox b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/LEMGA.dox deleted file mode 100644 index b53e298..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/LEMGA.dox +++ /dev/null @@ -1,39 +0,0 @@ -/** @mainpage - * @section intro Introduction - * LEMGA stands for ``Learning Models and Generic Algorithms.'' - * - * It was rewritten from an old implementation and is still under - * development and test. - * - * [Go back] - */ - - -/** @namespace lemga - * @brief Learning models and generic algorithms - * - * The idea is to separate the learning model and optimization techniques. - * - * Using vectorop.h for default vector operation - * @todo documentation: basic idea, ... - */ - -/** @namespace lemga::cost - * @brief Cost functions and their derivatives. - * - * Boosting can be applied to optimize different cost functions, as long - * as the derivative exists, and line-search is deployed. - */ - -/** @namespace lemga::kernel - * @brief Kernels (inner product in some space) - * - * Currently kernels also pass parameters to LIBSVM, which is used as - * the underlying implementation of SVM. - */ - -/** @namespace lemga::op - * @brief Operators used in optimization - * - * @todo documentation - */ diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/Makefile b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/Makefile deleted file mode 100644 index c0e6d21..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/Makefile +++ /dev/null @@ -1,258 +0,0 @@ -CC = g++ -CC_FINAL = -D NDEBUG -O3 -CC_DEBUG = -O1 -CC_LIB_FLAGS = -lm - -ifeq ($(CC), g++) # for GCC >= 3.2.x -CC_FLAGS = -Wall -Wshadow -Wcast-qual\ - -Wpointer-arith -Wconversion -Wredundant-decls\ - -Wwrite-strings -Woverloaded-virtual -CC_FINAL += -funroll-loops -#CC_FINAL += -march=pentium3 -mfpmath=sse -msse -mmmx -#CC_FINAL += -march=pentium4 -mfpmath=sse -msse -msse2 -mmmx -endif - -ifeq ($(CC), icc) # for ICC 8.0 -CC_FLAGS = -Wall -w1 -pch -CC_FINAL += -i_dynamic -ipo -unroll -CC_FINAL += -tpp6 -xK -#CC_FINAL += -tpp7 -xW -endif - -lemga_do_not_exec: - echo Do not run "make" under this directory. - -shared_ptr_h = shared_ptr.h -vectorop_h = vectorop.h -optimize_h = $(vectorop_h) optimize.h - -object_h = object.h -object_o = object.o -object_src = object.cpp $(object_h) -robject.o: $(object_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dobject.o: $(object_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -random_h = $(object_h) random.h -random_o = $(object_o) random.o -random_src = random.c $(random_h) -rrandom.o: $(random_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -drandom.o: $(random_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -quickfun_h = $(object_h) quickfun.h -quickfun_o = $(object_o) quickfun.o -quickfun_src = quickfun.c $(quickfun_h) -rquickfun.o: $(quickfun_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dquickfun.o: $(quickfun_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -utility_h = $(object_h) utility.h -utility_o = utility.o -utility_src = utility.cpp $(utility_h) -rutility.o: $(utility_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dutility.o: $(utility_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -dataset_h = $(random_h) dataset.h -cost_h = $(object_h) cost.h - -kernel_h = $(learnmodel_h) kernel.h -// have to include svm.o since Kernel::pass_params is implemented there -kernel_tmp_o = $(object_o) kernel.o -kernel_o = $(svm_o) $(kernel_tmp_o) -kernel_src = kernel.cpp $(kernel_h) -rkernel.o: $(kernel_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dkernel.o: $(kernel_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -learnmodel_h = $(object_h) $(dataset_h) $(shared_ptr_h) learnmodel.h -learnmodel_o = $(object_o) learnmodel.o -learnmodel_src = learnmodel.cpp $(learnmodel_h) -rlearnmodel.o: $(learnmodel_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dlearnmodel.o: $(learnmodel_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -datafeeder_h = $(learnmodel_h) datafeeder.h -datafeeder_o = $(learnmodel_o) $(random_o) datafeeder.o -datafeeder_src = datafeeder.cpp $(random_h) $(datafeeder_h) -rdatafeeder.o: $(datafeeder_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -ddatafeeder.o: $(datafeeder_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -crossval_h = $(shared_ptr_h) $(learnmodel_h) crossval.h -crossval_o = $(random_o) $(learnmodel_o) crossval.o -crossval_src = crossval.cpp $(vectorop_h) $(random_h) $(crossval_h) -rcrossval.o: $(crossval_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dcrossval.o: $(crossval_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -nnlayer_h = $(learnmodel_h) nnlayer.h -nnlayer_o = $(random_o) $(quickfun_o) $(learnmodel_o) nnlayer.o -nnlayer_src = nnlayer.cpp $(random_h) $(quickfun_h) $(nnlayer_h) -rnnlayer.o: $(nnlayer_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dnnlayer.o: $(nnlayer_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -feedforwardnn_h = $(learnmodel_h) $(nnlayer_h) feedforwardnn.h -feedforwardnn_o = $(learnmodel_o) $(nnlayer_o) feedforwardnn.o -feedforwardnn_src = feedforwardnn.cpp $(feedforwardnn_h) $(optimize_h) -rfeedforwardnn.o: $(feedforwardnn_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dfeedforwardnn.o: $(feedforwardnn_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -aggregating_h = $(learnmodel_h) $(shared_ptr_h) aggregating.h -aggregating_o = $(learnmodel_o) aggregating.o -aggregating_src = aggregating.cpp $(aggregating_h) -raggregating.o: $(aggregating_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -daggregating.o: $(aggregating_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -bagging_h = $(aggregating_h) bagging.h -bagging_o = $(aggregating_o) bagging.o -bagging_src = bagging.cpp $(bagging_h) -rbagging.o: $(bagging_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dbagging.o: $(bagging_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -boosting_h = $(aggregating_h) $(cost_h) boosting.h -boosting_o = $(aggregating_o) boosting.o -boosting_src = boosting.cpp $(vectorop_h) $(optimize_h) $(boosting_h) -rboosting.o: $(boosting_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dboosting.o: $(boosting_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -adaboost_h = $(boosting_h) adaboost.h -adaboost_o = $(boosting_o) adaboost.o -adaboost_src = adaboost.cpp $(adaboost_h) -radaboost.o: $(adaboost_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dadaboost.o: $(adaboost_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -stump_h = $(learnmodel_h) stump.h -stump_o = $(learnmodel_o) stump.o -stump_src = stump.cpp $(stump_h) -rstump.o: $(stump_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dstump.o: $(stump_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -cascade_h = $(aggregating_h) cascade.h -cascade_o = $(aggregating_o) cascade.o -cascade_src = cascade.cpp $(cascade_h) -rcascade.o: $(cascade_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dcascade.o: $(cascade_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -pulse_h = $(learnmodel_h) pulse.h -pulse_o = $(learnmodel_o) pulse.o -pulse_src = pulse.cpp $(pulse_h) -rpulse.o: $(pulse_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dpulse.o: $(pulse_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -cgboost_h = $(boosting_h) cgboost.h -cgboost_o = $(boosting_o) cgboost.o -cgboost_src = cgboost.cpp $(cgboost_h) $(vectorop_h) $(optimize_h) -rcgboost.o: $(cgboost_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dcgboost.o: $(cgboost_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -mgnboost_h = $(boosting_h) mgnboost.h -mgnboost_o = $(boosting_o) mgnboost.o -mgnboost_src = mgnboost.cpp $(optimize_h) $(mgnboost_h) -rmgnboost.o: $(mgnboost_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dmgnboost.o: $(mgnboost_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -## external module: LIBSVM ## -LIBSVM = ../../libsvm-2.81 -libsvm_h = ${LIBSVM}/svm.h -libsvm_src = ${LIBSVM}/svm.cpp $(libsvm_h) -rlibsvm.o: $(libsvm_src) - $(CC) -I${LIBSVM} $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dlibsvm.o: $(libsvm_src) - $(CC) -I${LIBSVM} $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -svm_h = $(learnmodel_h) $(kernel_h) svm.h -svm_o = $(learnmodel_o) ${kernel_tmp_o} libsvm.o svm.o -svm_src = svm.cpp $(libsvm_h) $(svm_h) -rsvm.o: $(svm_src) - $(CC) -I${LIBSVM} $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dsvm.o: $(svm_src) - $(CC) -I${LIBSVM} $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -## external module: GLPK ## -## You'll have to configure/make/install GLPK before use. -GLPK = ../../../glpk-4.8 -glpk_h = $(GLPK)/include/glpk.h -glpk_a = $(GLPK)/src/libglpk.a - -lpboost_h = $(boosting_h) lpboost.h -lpboost_o = $(boosting_o) lpboost.o -lpboost_lib = $(glpk_a) -lpboost_src = lpboost.cpp $(lpboost_h) $(glpk_h) -rlpboost.o: $(lpboost_src) - $(CC) -I${GLPK}/include $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dlpboost.o: $(lpboost_src) - $(CC) -I${GLPK}/include $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -perceptron_h = $(learnmodel_h) $(svm_h) perceptron.h -perceptron_o = $(random_o) $(utility_o) $(svm_o) $(stump_o) perceptron.o -perceptron_src = perceptron.cpp $(random_h) $(utility_h) $(stump_h) \ - $(perceptron_h) -rperceptron.o: $(perceptron_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dperceptron.o: $(perceptron_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -multiclass_ecoc_h = $(aggregating_h) multiclass_ecoc.h -multiclass_ecoc_o = $(aggregating_o) multiclass_ecoc.o -multiclass_ecoc_src = multiclass_ecoc.cpp $(multiclass_ecoc_h) -rmulticlass_ecoc.o: $(multiclass_ecoc_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dmulticlass_ecoc.o: $(multiclass_ecoc_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -adaboost_ecoc_h = $(multiclass_ecoc_h) $(shared_ptr_h) adaboost_ecoc.h -adaboost_ecoc_o = $(multiclass_ecoc_o) $(random_o) adaboost_ecoc.o -adaboost_ecoc_src = adaboost_ecoc.cpp $(random_h) $(utility_h) \ - $(vectorop_h) $(adaboost_ecoc_h) -radaboost_ecoc.o: $(adaboost_ecoc_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dadaboost_ecoc.o: $(adaboost_ecoc_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -adaboost_erp_h = $(adaboost_ecoc_h) adaboost_erp.h -adaboost_erp_o = $(adaboost_ecoc_o) adaboost_erp.o -adaboost_erp_src = adaboost_erp.cpp $(adaboost_erp_h) -radaboost_erp.o: $(adaboost_erp_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dadaboost_erp.o: $(adaboost_erp_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -ordinal_ble_h = $(learnmodel_h) $(multiclass_ecoc_h) ordinal_ble.h -ordinal_ble_o = $(learnmodel_o) ordinal_ble.o -ordinal_ble_src = ordinal_ble.cpp $(ordinal_ble_h) -rordinal_ble.o: $(ordinal_ble_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dordinal_ble.o: $(ordinal_ble_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.cpp deleted file mode 100644 index b0c0f01..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/** @file - * $Id: adaboost.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include -#include -#include "adaboost.h" - -REGISTER_CREATOR(lemga::AdaBoost); - -namespace lemga { - -void AdaBoost::train () { - cur_err.resize(n_samples); - Boosting::train(); - cur_err.clear(); -} - -REAL AdaBoost::linear_weight (const DataWgt& sw, const LearnModel& l) { - assert(exact_dimensions(l)); - assert(l.train_data() == ptd); - - REAL err = 0; - for (UINT i = 0; i < n_samples; ++i) { - if ((cur_err[i] = l.c_error(l.get_output(i), ptd->y(i))) > 0.1) - err += sw[i]; - } -#if VERBOSE_OUTPUT - std::cout << "Weighted classification error: " << err*100 << "%\n"; -#endif - - if (err >= 0.5) return -1; - - REAL beta; - if (err <= 0) - beta = 1000; - else - beta = 1 / err - 1; - return std::log(beta) / 2; -} - -REAL AdaBoost::convex_weight (const DataWgt&, const LearnModel&) { - std::cerr << "Please use the gradient descent methods for" - " convex combinations\n"; - OBJ_FUNC_UNDEFINED("convex_weight"); -} - -/* We assume classification problem here. The density update rule is - * d <- d * e^(-w y f) - * if y and f are binary, it is equivalent to say - * d <- d * beta for y != f, where beta = e^2w */ -void AdaBoost::linear_smpwgt (DataWgt& sw) { - const REAL beta = std::exp(2 * lm_wgt[n_in_agg-1]); - REAL bw_sum = 0; - for (UINT i = 0; i < n_samples; ++i) { - if (cur_err[i] > 0.1) - sw[i] *= beta; - bw_sum += sw[i]; - } - - assert(bw_sum != 0); - for (UINT i = 0; i < n_samples; ++i) - sw[i] /= bw_sum; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.h deleted file mode 100644 index 5576af1..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost.h +++ /dev/null @@ -1,49 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_ADABOOST_H__ -#define __LEMGA_AGGREGATING_ADABOOST_H__ - -/** @file - * @brief Declare @link lemga::AdaBoost AdaBoost@endlink class. - * - * $Id: adaboost.h 2696 2006-04-05 20:10:13Z ling $ - */ - -#include "boosting.h" - -namespace lemga { - -/** @brief %AdaBoost (adaptive boosting). - * - * %AdaBoost can be seen as gradient descent in the function space - * where the pointwise cost functional is defined as - * @f[ c(F(x_i),y_i) = e^{-y_i F(x_i)}\,. @f] - */ -class AdaBoost : public Boosting { -public: - explicit AdaBoost (bool cvx = false, const cost::Cost& c = cost::_cost) - : Boosting(cvx, c) {} - AdaBoost (const Boosting& s) : Boosting(s) {} - explicit AdaBoost (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual AdaBoost* create () const { return new AdaBoost(); } - virtual AdaBoost* clone () const { return new AdaBoost(*this); } - - virtual void train (); - -protected: - /// data only valid within training (remove?) - std::vector cur_err; - - virtual REAL convex_weight (const DataWgt&, const LearnModel&); - virtual REAL linear_weight (const DataWgt&, const LearnModel&); - virtual void linear_smpwgt (DataWgt&); -}; - -} // namespace lemga - -#ifdef __ADABOOST_H__ -#warning "This header file may conflict with another `adaboost.h' file." -#endif -#define __ADABOOST_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.cpp deleted file mode 100644 index 44147a6..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.cpp +++ /dev/null @@ -1,257 +0,0 @@ -/** @file - * $Id: adaboost_ecoc.cpp 2695 2006-04-05 05:21:47Z ling $ - */ - -#include -#include -#include -#include -#include "random.h" -#include "utility.h" -#include "vectorop.h" -#include "adaboost_ecoc.h" - -REGISTER_CREATOR(lemga::AdaBoost_ECOC); - -namespace lemga { - -void AdaBoost_ECOC::setup_aux () { - joint_wgt = JointWgt(n_class(), *ptw); - // to facilitate the normalization in update_aux(), - // we explicitly set joint_wgt[y_i][i] to zero - for (UINT i = 0; i < n_samples; ++i) - joint_wgt[ex_class[i]][i] = 0; - cur_err.resize(n_samples); - if (n_in_agg == 0) return; - - REAL wsum = 0; - for (UINT i = 0; i < n_samples; ++i) { - const std::vector& d = distances(i); - UINT y = ex_class[i]; - for (UINT c = 0; c < nclass; ++c) { - joint_wgt[c][i] *= std::exp(d[y] - d[c]); - wsum += joint_wgt[c][i]; - } - } - REAL r = nclass / wsum; - if (r > 1e-5 && r < 1e5) return; - using namespace op; - joint_wgt *= r; // normalize -} - -typedef std::vector > WMAT; -static REAL binary_cut (const WMAT& w, const ECOC_VECTOR& p) { - const UINT n = p.size(); - assert(w.size() == n); - REAL c = 0; - for (UINT i = 0; i < n; ++i) { - assert(w[i].size() == n); - for (UINT j = 0; j < i; ++j) { - assert(w[i][j] == w[j][i]); - if (p[i] != p[j]) - c += w[i][j]; - } - } - return c; -} - -// compute the edge weight -WMAT AdaBoost_ECOC::confusion_matrix () const { - WMAT w(nclass, std::vector(nclass, 0)); - for (UINT j = 0; j < n_samples; ++j) { - int y = ex_class[j]; - for (UINT c = 0; c < nclass; ++c) - w[y][c] += joint_wgt[c][j]; - } - for (UINT c = 0; c < nclass; ++c) - for (UINT y = 0; y < c; ++y) - w[y][c] = w[c][y] = w[y][c] + w[c][y]; - return w; -} - -/// test all combinations to find out the max-cut -ECOC_VECTOR AdaBoost_ECOC::max_cut (UINT nr) const { - assert(nr == nclass); - if (nclass > 13) - std::cerr << "Warning: Max-cut is very slow for too many classes\n"; - WMAT ewgt = confusion_matrix(); - for (UINT c = 0; c < nclass; ++c) - assert(ewgt[c][c] == 0); // needed for the max-cut below - - ECOC_VECTOR maxp, p(nclass, 0); - REAL maxc = 0, cut = 0; - size_t flip = 0; - while (gray_next(p, flip), flip != 0) { // p[0] will be fixed at 0 - for (UINT c = 0; c < nclass; ++c) - if (p[c] == p[flip]) - cut -= ewgt[c][flip]; - else - cut += ewgt[c][flip]; - if (maxc < cut) { - maxp = p; maxc = cut; - } - } - for (UINT c = 0; c < nclass; ++c) - if (maxp[c] == 0) maxp[c] = -1; - assert(std::fabs(maxc - binary_cut(ewgt, maxp)) < EPSILON); - return maxp; -} - -/// nr (or nr+1) is the maximal # of classes that can be "colored" -ECOC_VECTOR AdaBoost_ECOC::max_cut_greedy (UINT nr) const { - assert(nr <= nclass); - WMAT ewgt = confusion_matrix(); - - // we could use O(n^2 log(K)) to find the first K edges with largest - // weights. Here we just use O(n^2 log(n^2)). - typedef std::multimap MMAP; - MMAP edge; - for (UINT j = 1; j < nclass; ++j) - for (UINT c = 0; c+j < nclass; ++c) { - UINT y = c + j; - edge.insert(std::pair(-ewgt[y][c], c*nclass+y)); - } - - ECOC_VECTOR p(nclass, 0); - UINT pn = 0; - for (MMAP::iterator pe = edge.begin(); pn < nr && pe != edge.end(); ++pe) { - UINT k1 = pe->second / nclass, k2 = pe->second % nclass; - if (randu() < 0.5) std::swap(k1, k2); // not really useful - if (p[k1] == 0 && p[k2] == 0) { - p[k1] = 1; p[k2] = -1; - pn += 2; - } else if (p[k1] == 0) { - p[k1] = -p[k2]; - ++pn; - } else if (p[k2] == 0) { - p[k2] = -p[k1]; - ++pn; - } - } - - return p; -} - -ECOC_VECTOR AdaBoost_ECOC::random_half (UINT nr) const { - assert(nr <= nclass); - ECOC_VECTOR p(nclass, 0); - - // If nr == n, setting p to be all 1's and then randomly - // flipping half of p should be the fastest approach. - std::vector idx(nclass); - for (UINT i = 0; i < nclass; ++i) - idx[i] = i; - std::random_shuffle(idx.begin(), idx.end()); - while (nr--) - p[idx[nr]] = (nr % 2)? 1 : -1; - - return p; -} - -bool AdaBoost_ECOC::ECOC_partition (UINT i, ECOC_VECTOR& p) const { - if (MultiClass_ECOC::ECOC_partition(i, p)) return true; - const UINT n = n_class(); - - switch (par_method) { - case RANDOM_HALF: - p = random_half(n); - break; - - case MAX_CUT: - p = max_cut(n); - break; - - case MAX_CUT_GREEDY: - p = max_cut_greedy(n); - break; - - default: - assert(false); - } - - return true; -} - -pDataWgt AdaBoost_ECOC::smpwgt_with_partition (const ECOC_VECTOR& p) const { - assert(is_full_partition(p)); - DataWgt* btw = new DataWgt(n_samples); - REAL wsum = 0; - for (UINT i = 0; i < n_samples; ++i) { - int y = p[ex_class[i]]; - REAL w = 0; - for (UINT c = 0; c < n_class(); ++c) - if (p[c] + y == 0) - w += joint_wgt[c][i]; - wsum += w; (*btw)[i] = w; - } - REAL r = 1 / wsum; - for (UINT i = 0; i < n_samples; ++i) - (*btw)[i] *= r; - return btw; -} - -pLearnModel -AdaBoost_ECOC::train_with_full_partition (const ECOC_VECTOR& p) const { - LearnModel *plm = lm_base->clone(); - assert(plm != 0); - - DataSet* btd = new DataSet(); - for (UINT i = 0; i < n_samples; ++i) - btd->append(ptd->x(i), Output(1, p[ex_class[i]])); - cur_smpwgt = smpwgt_with_partition(p); // saved for future use - - plm->set_train_data(btd, cur_smpwgt); - plm->train(); - return plm; -} - -pLearnModel AdaBoost_ECOC::train_with_partition (ECOC_VECTOR& p) const { - assert(is_full_partition(p)); - pLearnModel plm = train_with_full_partition(p); - // Put back the full training set and an arbitrary weight - plm->set_train_data(ptd, ptw); - for (UINT i = 0; i < n_samples; ++i) - cur_err[i] = (plm->get_output(i)[0] * p[ex_class[i]] <= 0); - return plm; -} - -REAL -AdaBoost_ECOC::assign_weight (const ECOC_VECTOR&, const LearnModel&) const { - const DataWgt& sw = *cur_smpwgt; - REAL err = 0; - for (UINT i = 0; i < n_samples; ++i) { - if (cur_err[i]) - err += sw[i]; - } - if (err >= 0.5) return -1; - - REAL beta; - if (err <= 0) - beta = 1000; - else - beta = 1/err - 1; - return std::log(beta) / 2; -} - -void AdaBoost_ECOC::update_aux (const ECOC_VECTOR& p) { - assert(is_full_partition(p)); - const REAL beta = std::exp(lm_wgt[n_in_agg-1]); - const REAL ibeta = 1 / beta; - - REAL wsum = 0; - for (UINT i = 0; i < n_samples; ++i) { - int y = p[ex_class[i]]; - REAL r = (cur_err[i]? beta : ibeta); - for (UINT c = 0; c < nclass; ++c) { - if (p[c] != y) - joint_wgt[c][i] *= r; - wsum += joint_wgt[c][i]; - } - } - REAL r = nclass / wsum; - if (r > 1e-5 && r < 1e5) return; - using namespace op; - joint_wgt *= r; // normalize -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.h deleted file mode 100644 index 7913bb3..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_ecoc.h +++ /dev/null @@ -1,76 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_MULTICLASS_ADABOOST_ECOC_H__ -#define __LEMGA_MULTICLASS_ADABOOST_ECOC_H__ - -/** @file - * @brief Declare @link lemga::AdaBoost_ECOC AdaBoost_ECOC@endlink class. - * - * $Id: adaboost_ecoc.h 2696 2006-04-05 20:10:13Z ling $ - */ - -#include "multiclass_ecoc.h" -#include "shared_ptr.h" - -namespace lemga { - -typedef std::vector JointWgt; -typedef const_shared_ptr pJointWgt; - -/** @brief AdaBoost.ECC with exponential cost and Hamming distance. - */ -class AdaBoost_ECOC : public MultiClass_ECOC { -public: - enum PARTITION_METHOD { - RANDOM_HALF, - MAX_CUT, MAX_CUT_GREEDY, - RANDOM_2, - MAX_2 - }; - -protected: - PARTITION_METHOD par_method; - -public: - AdaBoost_ECOC () : MultiClass_ECOC(), par_method(MAX_CUT_GREEDY) {} - AdaBoost_ECOC (const MultiClass_ECOC& s) - : MultiClass_ECOC(s), par_method(MAX_CUT_GREEDY) {} - explicit AdaBoost_ECOC (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual AdaBoost_ECOC* create () const { return new AdaBoost_ECOC(); } - virtual AdaBoost_ECOC* clone () const { - return new AdaBoost_ECOC(*this); } - - void set_partition_method (PARTITION_METHOD m) { par_method = m; } - -protected: - /// set up by setup_aux(); updated by update_aux(); - /// used by a lot of functions here. - JointWgt joint_wgt; - /// set up by train_with_partition(); - /// used by assign_weight() and update_aux(). - mutable std::vector cur_err; - /// set up by train_with_partition(); used by assign_weight(). - mutable pDataWgt cur_smpwgt; - pDataWgt smpwgt_with_partition (const ECOC_VECTOR&) const; - pLearnModel train_with_full_partition (const ECOC_VECTOR&) const; - - virtual void setup_aux (); - virtual bool ECOC_partition (UINT, ECOC_VECTOR&) const; - virtual pLearnModel train_with_partition (ECOC_VECTOR&) const; - virtual REAL assign_weight (const ECOC_VECTOR&, const LearnModel&) const; - virtual void update_aux (const ECOC_VECTOR&); - - std::vector > confusion_matrix () const; - ECOC_VECTOR max_cut (UINT) const; - ECOC_VECTOR max_cut_greedy (UINT) const; - ECOC_VECTOR random_half (UINT) const; -}; - -} // namespace lemga - -#ifdef __ADABOOST_ECOC_H__ -#warning "This header file may conflict with another `adaboost_ecoc.h' file." -#endif -#define __ADABOOST_ECOC_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.cpp deleted file mode 100644 index 4cf109f..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/** @file - * $Id: adaboost_erp.cpp 2637 2006-02-13 01:59:50Z ling $ - */ - -#include -#include "adaboost_erp.h" - -REGISTER_CREATOR(lemga::AdaBoost_ERP); - -namespace lemga { - -bool AdaBoost_ERP::ECOC_partition (UINT i, ECOC_VECTOR& p) const { - if (MultiClass_ECOC::ECOC_partition(i, p)) return true; - const UINT n = 2; // (n_class() + 3) / 4; - - switch (par_method) { - case RANDOM_2: - p = random_half(n); - break; - - case MAX_2: - p = max_cut_greedy(n); - break; - - default: - return AdaBoost_ECOC::ECOC_partition(i, p); - } - - return true; -} - -pLearnModel -AdaBoost_ERP::train_with_partial_partition (const ECOC_VECTOR& p) const { - LearnModel *plm = lm_base->clone(); - assert(plm != 0); - - // We will ignore examples with "color" p[] = 0. - // The best way is to first group examples by their classes, - // then each time collect classes with nonzero p. - // We don't do this for easier modification for future changes, - // e.g., if we want to allow continuous values of p[] - DataSet* btd = new DataSet(); - DataWgt* btw = new DataWgt(); - REAL wsum = 0; - for (UINT i = 0; i < n_samples; ++i) { - int y = p[ex_class[i]]; - if (y == 0) continue; - - btd->append(ptd->x(i), Output(1, y)); - REAL w = 0; - for (UINT c = 0; c < nclass; ++c) - if (p[c] + y == 0) - w += joint_wgt[c][i]; - wsum += w; btw->push_back(w); - } - REAL r = 1 / wsum; - for (UINT i = 0; i < btw->size(); ++i) - (*btw)[i] *= r; - - plm->set_train_data(btd, btw); - plm->train(); - return plm; -} - -#define OUTPUT_PARTITION(p,o) \ - for (UINT c = 0; c < p.size(); ++c) \ - o << (p[c]>0? '+':(p[c]<0?'-':'0')); \ - o << std::flush - -pLearnModel AdaBoost_ERP::train_with_partition (ECOC_VECTOR& p) const { -#if VERBOSE_OUTPUT - std::cout << " "; - OUTPUT_PARTITION(p, std::cout); -#endif - - pLearnModel plm = 0; - bool calc_smpwgt = true; - UINT s = lrs; - while (s--) { - // learning - assert(calc_smpwgt); - if (is_full_partition(p)) { - plm = train_with_full_partition(p); // cur_smpwgt is set - calc_smpwgt = false; - } else - plm = train_with_partial_partition(p); - assert(plm != 0); -#if VERBOSE_OUTPUT - std::cout << " ... trained" << std::flush; -#endif - - // Put back the full training set and an arbitrary weight - // to collect the outputs (in cur_err). - // Note output 0 will be put as -1. - plm->set_train_data(ptd, ptw); - for (UINT i = 0; i < n_samples; ++i) - cur_err[i] = (plm->get_output(i)[0] > 0); // tmp use - - if (!(s--)) break; - - // re-partitioning - std::vector mkt(nclass, 0); - for (UINT i = 0; i < n_samples; ++i) { - UINT y = ex_class[i]; - int out = (cur_err[i]? 1 : -1); - for (UINT c = 0; c < nclass; ++c) { - const REAL jwo = joint_wgt[c][i] * out; - mkt[c] += jwo; mkt[y] -= jwo; - } - } - bool changed = false; - for (UINT c = 0; c < nclass; ++c) { - int np = (mkt[c]>0? -1 : 1); - changed |= (np != p[c]); - p[c] = np; - } -#if VERBOSE_OUTPUT - std::cout << "\n => "; - if (changed) { - OUTPUT_PARTITION(p, std::cout); - } else - std::cout << "NO CHANGE"; -#endif - - if (!changed) break; - calc_smpwgt = true; - } -#if VERBOSE_OUTPUT - std::cout << '\n'; -#endif - assert(is_full_partition(p) /* && cur_err == output of plm */); - - // Update the current error & sample weights - for (UINT i = 0; i < n_samples; ++i) - cur_err[i] = cur_err[i] ^ (p[ex_class[i]] > 0); - if (calc_smpwgt) - cur_smpwgt = smpwgt_with_partition(p); - - return plm; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.h deleted file mode 100644 index c79936b..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/adaboost_erp.h +++ /dev/null @@ -1,49 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_MULTICLASS_ADABOOST_ERP_H__ -#define __LEMGA_MULTICLASS_ADABOOST_ERP_H__ - -/** @file - * @brief Declare @link lemga::AdaBoost_ERP AdaBoost_ERP@endlink class. - * - * $Id: adaboost_erp.h 2746 2006-04-21 22:45:38Z ling $ - */ - -#include "adaboost_ecoc.h" - -namespace lemga { - -/** @brief AdaBoost.ERP (AdaBoost.ECC with Re-Partitioning). - * - * Alter the ECC table after learning. - */ -class AdaBoost_ERP : public AdaBoost_ECOC { -protected: - UINT lrs; ///< # of steps in the learning / re-partitioning cycle - //? shall we save rp_step when serializing? - -public: - AdaBoost_ERP () : AdaBoost_ECOC(), lrs(2) {} - AdaBoost_ERP (const AdaBoost_ECOC& s) : AdaBoost_ECOC(s), lrs(2) {} - explicit AdaBoost_ERP (std::istream& is) : lrs(2) { is >> *this; } - - virtual const id_t& id () const; - virtual AdaBoost_ERP* create () const { return new AdaBoost_ERP(); } - virtual AdaBoost_ERP* clone () const { - return new AdaBoost_ERP(*this); } - - void set_lr_step (UINT s) { assert(s > 1); lrs = s; } - -protected: - pLearnModel train_with_partial_partition (const ECOC_VECTOR&) const; - - virtual bool ECOC_partition (UINT, ECOC_VECTOR&) const; - virtual pLearnModel train_with_partition (ECOC_VECTOR&) const; -}; - -} // namespace lemga - -#ifdef __ADABOOST_ERP_H__ -#warning "This header file may conflict with another `adaboost_erp.h' file." -#endif -#define __ADABOOST_ERP_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.cpp deleted file mode 100644 index a42c82b..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/** @file - * $Id: aggregating.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "aggregating.h" - -namespace lemga { - -/** Delete learning models stored in @a lm. This is only used in - * operator= and load(). - * @note @a lm_base is not deleted since load() will need it - * @todo make it public under the name clear()? Or remove it - */ -void Aggregating::reset () { - LearnModel::reset(); - lm.clear(); n_in_agg = 0; - assert(lm_base == 0 || valid_dimensions(*lm_base)); -} - -/** @note Brand new models are used in the new born object. Thus - * any future change to the learning models @a a will not affect - * this model. - */ -Aggregating::Aggregating (const Aggregating& a) - : LearnModel(a), lm_base(a.lm_base), - n_in_agg(a.n_in_agg), max_n_model(a.max_n_model) -{ - const UINT lms = a.lm.size(); - assert(n_in_agg <= lms); - for (UINT i = 0; i < lms; ++i) - lm.push_back(a.lm[i]->clone()); -} - -/** @copydoc Aggregating(const Aggregating&) */ -const Aggregating& Aggregating::operator= (const Aggregating& a) { - if (&a == this) return *this; - - LearnModel::operator=(a); - lm_base = a.lm_base; - n_in_agg = a.n_in_agg; - max_n_model = a.max_n_model; - - const UINT lms = a.lm.size(); - assert(n_in_agg <= lms); - lm.clear(); - for (UINT i = 0; i < lms; ++i) - lm.push_back(a.lm[i]->clone()); - - return *this; -} - -bool Aggregating::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - - if (!(os << lm.size() << ' ' << (lm_base != 0) << '\n')) - return false; - if (lm_base != 0) - if (!(os << *lm_base)) return false; - for (UINT i = 0; i < lm.size(); ++i) - if (!(os << *lm[i])) return false; - - return true; -} - -bool Aggregating::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - - if (v == 0) /* Take care of _n_in and _n_out */ - if (!(is >> _n_in >> _n_out)) return false; - - UINT t3, t4; - if (!(is >> t3 >> t4) || t4 > 1) return false; - - if (!t4) lm_base = 0; - else { - if (v == 0) { /* ignore a one-line comment */ - char c; is >> c; - assert(c == '#'); - is.ignore(100, '\n'); - } - LearnModel* p = (LearnModel*) Object::create(is); - lm_base = p; - if (p == 0 || !valid_dimensions(*p)) return false; - } - - lm.clear(); - for (UINT i = 0; i < t3; ++i) { - LearnModel* p = (LearnModel*) Object::create(is); - lm.push_back(p); - if (p == 0 || !exact_dimensions(*p)) return false; - } - n_in_agg = t3; - - return true; -} - -/** @brief Set the base learning model. - * @todo Allowed to call when !empty()? - */ -void Aggregating::set_base_model (const LearnModel& blm) { - assert(valid_dimensions(blm)); - lm_base = blm.clone(); -} - -/** @brief Specify the number of hypotheses used in aggregating. - * @return @c false if @a n is larger than size(). - * - * Usually all the hypotheses are used in aggregating. However, a - * smaller number @a n can also be specified so that only the first - * @a n hypotheses are used. - */ -bool Aggregating::set_aggregation_size (UINT n) { - if (n <= size()) { - n_in_agg = n; - return true; - } - return false; -} - -void Aggregating::set_train_data (const pDataSet& pd, const pDataWgt& pw) { - LearnModel::set_train_data(pd, pw); - // Note: leave the compatibility check of the base learner to training. - for (UINT i = 0; i < lm.size(); ++i) - if (lm[i] != 0) - lm[i]->set_train_data(ptd, ptw); -} - -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.h deleted file mode 100644 index dabb780..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/aggregating.h +++ /dev/null @@ -1,86 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_H__ -#define __LEMGA_AGGREGATING_H__ - -/** @file - * @brief Declare @link lemga::Aggregating Aggregating@endlink class. - * - * $Id: aggregating.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "learnmodel.h" - -namespace lemga { - -/** @brief An abstract class for aggregating. - * - * %Aggregating in learning stands for a series of techniques which - * generate several hypotheses and combine them into a large and - * usually better one. Bagging and AdaBoost are two famous examples - * of such techniques. This class provides member functions to - * store and retrieve hypotheses used in aggregating. - * - * The class has a vector of hypotheses, and a base learning model, - * which is the ``parent'' of all those hypotheses. For users of this - * class, a possible calling order for training is - * -# Aggregating *ag = new Some_Aggregating_Method (6, 5);\n - * Create an instance of aggregating with 6 as input dimension - * and 5 output dimension. - * -# ag->@link - * set_base_model() set_base_model@endlink(a_neural_net);\n - * Specify the base learning model (in this example, a neural network). - * -# Follow the - * @ref learnmodel_training_order "normal calling order for training" - * to complete the training. - * - * We do not provide...? - * @todo Documentation - */ -class Aggregating : public LearnModel { -protected: - pcLearnModel lm_base; ///< The base learning model - std::vector lm; ///< Pointers to learning models - UINT n_in_agg; ///< \# of models in aggregating - UINT max_n_model; ///< Maximal # of models allowed - -public: - Aggregating () : LearnModel(), n_in_agg(0), max_n_model(0) {} - Aggregating (const Aggregating&); - const Aggregating& operator= (const Aggregating&); - - virtual Aggregating* create () const = 0; - virtual Aggregating* clone () const = 0; - - //@{ @name Set/get the base model (weak learner) - void set_base_model (const LearnModel&); - const LearnModel& base_model () const { return *lm_base; } - //@} - - void set_max_models (UINT max) { max_n_model = max; } - - //@{ @name Hypotheses operation - /// Total number of hypotheses - UINT size () const { return lm.size(); } - bool empty () const { return lm.empty(); } - const LearnModel& model (UINT n) const { return *lm[n]; } - const LearnModel& operator[] (UINT n) const { return *lm[n]; } - //@} - - virtual bool set_aggregation_size (UINT); - UINT aggregation_size () const { return n_in_agg; } - virtual void set_train_data (const pDataSet&, const pDataWgt& = 0); - virtual void reset (); - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __AGGREGATING_H__ -#warning "This header file may conflict with another `aggregating.h' file." -#endif -#define __AGGREGATING_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.cpp deleted file mode 100644 index a6c6e00..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/** @file - * $Id: bagging.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include -#include "bagging.h" - -REGISTER_CREATOR(lemga::Bagging); - -namespace lemga { - -Output Bagging::operator() (const Input& x) const { - assert(n_in_agg <= size() && _n_out > 0); - - Output y(_n_out, 0); - for (UINT i = 0; i < n_in_agg; ++i) { - assert(lm[i] != 0 && exact_dimensions(*lm[i])); - Output yi = (*lm[i])(x); - for (UINT j = 0; j < _n_out; ++j) - y[j] += (yi[j] > 0)? 1 : -1; - } - - return y; -} - -void Bagging::train () { - assert(ptd != 0 && ptw != 0); - set_dimensions(*ptd); - - for (n_in_agg = size(); n_in_agg < max_n_model; ++n_in_agg) { -#if VERBOSE_OUTPUT - std::cout << "=== " << id() << " #" << n_in_agg+1 << " / " - << max_n_model << " ===\n"; -#endif - assert(lm_base != 0); - LearnModel *p = lm_base->clone(); - p->set_train_data(ptd->random_sample(*ptw, n_samples)); - p->train(); - p->set_train_data(ptd, ptw); // get rid of the random sample - set_dimensions(*p); - lm.push_back(p); - } -} - -REAL Bagging::margin_norm () const { - return n_in_agg; -} - -REAL Bagging::margin_of (const Input& x, const Output& y) const { - assert(std::fabs(y[0]*y[0]-1) < INFINITESIMAL); - return (*this)(x)[0] * y[0]; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.h deleted file mode 100644 index 9a942ea..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/bagging.h +++ /dev/null @@ -1,43 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_BAGGING_H__ -#define __LEMGA_AGGREGATING_BAGGING_H__ - -/** @file - * @brief Declare @link lemga::Bagging Bagging@endlink class. - * - * $Id: bagging.h 2696 2006-04-05 20:10:13Z ling $ - */ - -#include "aggregating.h" - -namespace lemga { - -/** @brief %Bagging (boostrap aggregating). - * - * %Bagging averages over all hypotheses. - * @todo Documentation - */ -class Bagging : public Aggregating { -public: - Bagging () : Aggregating() {} - Bagging (const Aggregating& s) : Aggregating(s) {} - explicit Bagging (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Bagging* create () const { return new Bagging(); } - virtual Bagging* clone () const { return new Bagging(*this); } - - virtual bool support_weighted_data () const { return true; } - virtual void train (); - virtual Output operator() (const Input&) const; - virtual REAL margin_norm () const; - virtual REAL margin_of (const Input&, const Output&) const; -}; - -} // namespace lemga - -#ifdef __BAGGING_H__ -#warning "This header file may conflict with another `bagging.h' file." -#endif -#define __BAGGING_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.cpp deleted file mode 100644 index fb6897f..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.cpp +++ /dev/null @@ -1,277 +0,0 @@ -/** @file - * $Id: boosting.cpp 2696 2006-04-05 20:10:13Z ling $ - */ - -#include -#include "vectorop.h" -#include "optimize.h" -#include "boosting.h" - -REGISTER_CREATOR(lemga::Boosting); - -#define _cost(F,y) cost_functor.cost(F[0],y[0]) -#define _cost_deriv(F,y) cost_functor.deriv1(F[0],y[0]) - -namespace lemga { - -/** @copydoc LearnModel(UINT,UINT) - * @param cvx \c true if convex combination is used; \c false if - * linear combination is used */ -Boosting::Boosting (bool cvx, const cost::Cost& c) - : Aggregating(), convex(cvx), grad_desc_view(false), - min_cst(0), min_err(-1), cost_functor(c) -{ /* empty */ } - -Boosting::Boosting (const Aggregating& s) - : Aggregating(s), lm_wgt(lm.size(), 1), convex(false), - grad_desc_view(false), min_cst(0), min_err(-1), - cost_functor(cost::_cost) -{} - -bool Boosting::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Aggregating, os, vl, 1); - assert(lm_wgt.size() == lm.size()); - for (UINT i = 0; i < lm_wgt.size(); ++i) - os << lm_wgt[i] << ' '; - if (!lm_wgt.empty()) os << '\n'; - return (os << convex << '\n'); -} - -bool Boosting::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Aggregating, is, vl, 1, v); - - const UINT n = lm.size(); - lm_wgt.resize(n); - for (UINT i = 0; i < n; ++i) - if (!(is >> lm_wgt[i])) return false; - - UINT c; - if (!(is >> c)) { - if (v != 0) return false; - convex = false; // some old version: no convex - } - else if (c > 1) return false; - convex = c; - return true; -} - -void Boosting::reset () { - Aggregating::reset(); - lm_wgt.clear(); -#if BOOSTING_OUTPUT_CACHE - clear_cache(); -#endif -} - -REAL Boosting::margin_norm () const { - return convex? 1 : model_weight_sum(); -} - -REAL Boosting::margin_of (const Input& x, const Output& y) const { - assert(std::fabs(y[0]*y[0]-1) < INFINITESIMAL); - return (*this)(x)[0] * y[0]; -} - -REAL Boosting::margin (UINT i) const { - REAL y = ptd->y(i)[0]; - assert(std::fabs(y*y-1) < INFINITESIMAL); - return get_output(i)[0] * y; -} - -Output Boosting::operator() (const Input& x) const { - assert(n_in_agg <= lm.size() && lm.size() == lm_wgt.size() && _n_out > 0); -#ifndef NDEBUG - for (UINT i = 0; i < n_in_agg; ++i) - assert(lm_wgt[i] >= 0); -#endif - - Output y(_n_out, 0); - for (UINT i = 0; i < n_in_agg; ++i) { - assert(lm[i] != 0 && exact_dimensions(*lm[i])); - Output out = (*lm[i])(x); - for (UINT j = 0; j < _n_out; ++j) - y[j] += (out[j] > 0)? lm_wgt[i] : -lm_wgt[i]; - } - - if (convex && n_in_agg > 0) { - using namespace op; - y *= 1 / model_weight_sum(); - } - return y; -} - -Output Boosting::get_output (UINT idx) const { - assert(n_in_agg <= lm.size() && lm.size() == lm_wgt.size() && _n_out > 0); - assert(ptw != 0); // no data sampling - -#if BOOSTING_OUTPUT_CACHE - if (cache_n[idx] > n_in_agg) - clear_cache(idx); - Output& y = cache_y[idx]; - UINT start = cache_n[idx]; - cache_n[idx] = n_in_agg; - if (start == 0) { // y is either empty, or already filled with 0 - assert(y.empty() || y[0] == 0); // only check y[0] - y.resize(_n_out, 0); - } -#else - Output y(_n_out, 0); - UINT start = 0; -#endif - assert(y.size() == _n_out); - for (UINT i = start; i < n_in_agg; ++i) { - assert(lm[i] != 0 && exact_dimensions(*lm[i])); - assert(lm[i]->train_data() == ptd); - Output out = lm[i]->get_output(idx); - for (UINT j = 0; j < _n_out; ++j) - y[j] += (out[j] > 0)? lm_wgt[i] : -lm_wgt[i]; - } - - if (convex && n_in_agg > 0) { - using namespace op; -#if BOOSTING_OUTPUT_CACHE - Output y2 = y; - return (y2 *= 1 / model_weight_sum()); -#else - y *= 1 / model_weight_sum(); -#endif - } - return y; -} - -#if BOOSTING_OUTPUT_CACHE -void Boosting::set_train_data (const pDataSet& pd, const pDataWgt& pw) { - pDataSet old_ptd = ptd; - Aggregating::set_train_data(pd, pw); - if (old_ptd != ptd) clear_cache(); -} -#endif - -void Boosting::train () { - assert(ptd != 0 && ptw != 0); - assert(lm_base != 0); // we need lm_base to create new hypotheses - set_dimensions(*ptd); - - if (grad_desc_view) { - train_gd(); - return; - } - - n_in_agg = size(); - pDataWgt sample_wgt = sample_weight(); - - while (n_in_agg < max_n_model) { - const pLearnModel p = train_with_smpwgt(sample_wgt); - - // update sample_wgt, set up hypothesis wgt (lm_wgt) - const REAL w = assign_weight(*sample_wgt, *p); - if (w <= 0) break; - - set_dimensions(*p); - lm.push_back(p); lm_wgt.push_back(w); - n_in_agg++; - if (min_cst > 0 && cost() < min_cst) break; - if (min_err >= 0 && train_c_error() <= min_err) break; - sample_wgt = update_smpwgt(*sample_wgt, *p); - } -} - -void Boosting::train_gd () { - _boost_gd bgd(this); - iterative_optimize(_line_search<_boost_gd,BoostWgt,REAL,REAL> - (&bgd, convex? 1.0 : 0.5)); -} - -pLearnModel Boosting::train_with_smpwgt (const pDataWgt& sw) const { -#if VERBOSE_OUTPUT - std::cout << "=== " << id() - << " [" << (convex? "convex" : "linear") << "] #" - << n_in_agg+1 << " / " << max_n_model << " ===\n"; -#endif - LearnModel *plm = lm_base->clone(); - assert(plm != 0); - - plm->set_train_data(ptd, sw); - plm->train(); - // put back ptd for future get_output() call, and put back ptw to - // save memory -- however, plm has to support sample weight - assert(plm->support_weighted_data()); - plm->set_train_data(ptd, ptw); - return plm; -} - -REAL Boosting::convex_weight (const DataWgt&, const LearnModel&) { - OBJ_FUNC_UNDEFINED("convex_weight"); -} -REAL Boosting::linear_weight (const DataWgt&, const LearnModel&) { - OBJ_FUNC_UNDEFINED("linear_weight"); -} - -void Boosting::convex_smpwgt (DataWgt&) { - OBJ_FUNC_UNDEFINED("convex_smpwgt"); -} -void Boosting::linear_smpwgt (DataWgt&) { - OBJ_FUNC_UNDEFINED("linear_smpwgt"); -} - -REAL Boosting::cost () const { - assert(ptd != 0 && ptw != 0); - REAL cst = 0; - for (UINT i = 0; i < n_samples; ++i) { - REAL c = _cost(get_output(i), ptd->y(i)); - cst += c * (*ptw)[i]; - } - return cst; -} - -/** Compute weight (probability) vector according to - * @f[ D_i \propto -\frac{w_i}{y_i} c'_F (F(x_i), y_i) @f] - * @sa #cost_deriv_functor - */ -pDataWgt Boosting::sample_weight () const { - assert(ptd != 0 && ptw != 0); - if (n_in_agg == 0) return ptw; - - DataWgt* pdw = new DataWgt(n_samples); - REAL sum = 0; - for (UINT i = 0; i < n_samples; ++i) { - REAL yi = ptd->y(i)[0]; - REAL p = - (*ptw)[i] / yi * _cost_deriv(get_output(i), ptd->y(i)); - assert(p >= 0); - (*pdw)[i] = p; sum += p; - } - assert(sum > 0); - const REAL k = 1 / sum; - for (UINT i = 0; i < n_samples; ++i) - (*pdw)[i] *= k; - - return pdw; -} - -Boosting::BoostWgt& Boosting::BoostWgt::operator+= (const BoostWgt& bw) { - const UINT ts = size(); - assert(ts+1 == bw.size()); - - for (UINT i = 0; i < ts; ++i) { - assert(lm[i] == bw.lm[i]); - lm_wgt[i] += bw.lm_wgt[i]; - } - lm.push_back(bw.lm[ts]); - lm_wgt.push_back(bw.lm_wgt[ts]); - - return *this; -} - -Boosting::BoostWgt Boosting::BoostWgt::operator- () const { - using namespace op; - return BoostWgt(lm, -lm_wgt); -} - -Boosting::BoostWgt& Boosting::BoostWgt::operator*= (REAL r) { - using namespace op; - lm_wgt *= r; - return *this; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.h deleted file mode 100644 index 05d4d33..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/boosting.h +++ /dev/null @@ -1,295 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_BOOSTING_H__ -#define __LEMGA_AGGREGATING_BOOSTING_H__ - -/** @file - * @brief Declare @link lemga::Boosting Boosting@endlink (AnyBoost) class. - * - * $Id: boosting.h 2696 2006-04-05 20:10:13Z ling $ - */ - -#include -#include -#include "aggregating.h" -#include "cost.h" - -#define BOOSTING_OUTPUT_CACHE 1 - -namespace lemga { - -/// Interface for optimization of Boosting -struct _boost_gd; - -/** @brief %Boosting generates a linear combination of hypotheses. - * - * As one specific aggregating technique, boosting generates a linear - * (may be restricted to convex) combination of hypotheses by - * sequentially calling a weak learner (the base model) with varying - * sample weights. - * - * For many problems, convex combination may result in a same super - * learning model as linear combination does. However, the training - * algorithms used in different combinations may differ. Whether to - * use convex or linear combinations is specified with the constructor - * (Boosting()). The default is linear combination. - * - * Traditional boosting techniques (e.g., %AdaBoost) carry out training - * in the following form to generate @a n hypotheses: - * -# Initialize sample weights; - * -# For @a i from 1 to @a n, do - * -# Call the weak learner with the current sample weights to - * get a new hypothesis; - * -# Calculate the weight for this hypothesis; - * -# Update the sample weights. - * -# Return the weighted sum of hypotheses (with a @c sign operation). - * - * (See the code of Boosting::train() for more details.) - * The function assign_weight() is used to calculate the weight for - * the hypothesis; update_smpwgt() is used to update the sample weights. - * Modifying these two functions properly is usually enough for - * designing a new boosting algorithm. (To be precise, functions to - * be modified are convex_weight(), convex_smpwgt(), and/or - * linear_weight(), linear_smpwgt().) - * - * @attention We do not assume update_smpwgt() will ever be called. - * However, if it is called, we do assume assign_weight() has been - * called immediately before it. - * - * %Boosting algorithms (at least some of them) can be viewed as - * gradient descent in a function space. A cost functional - * @f[ C(F) = \sum_i w_i \cdot c(F(x_i), y_i) @f] - * is the sample average of some pointwise cost (#cost_functor). - * A @em corresponding boosting algorithm decreases this cost - * functional by adding proper weak hypotheses to @a F. - * - * The gradient, which is a function, is also defined on training samples: - * @f[ \nabla C(F)(x_i) = w_i \cdot c'_F (F(x_i), y_i)\,. @f] - * The next hypothesis should maximize the inner-product between it - * and @f$\nabla C(F)@f$. Thus the sample weights used to training - * this hypothesis is - * @f[ D_i \propto -\frac{w_i}{y_i} c'_F (F(x_i), y_i) @f] - * where @f$c'_F(\cdot,\cdot)@f$ is the partial derivative to the - * first argument (see #cost_functor.deriv1()). - * - * Denote the newly generated hypothesis by @a g. The weight for @a g - * can be determined by doing a line search along @f$F+\alpha g@f$, - * where @f$\alpha@f$ is a positive scalar. The weight also makes - * the updated vector of sample weights perpendicular to the vector - * of errors of @a g. - * - * We use the @link _line_search line search template@endlink to - * implement the training of gradient descent view. Employing different - * #cost_functor should be enough to get a different boosting algorithm. - * See also _boost_gd. - * - * @note For some cost functionals, the weight can be calculated directly - * without a line search. Replacing the line search with the direct - * calculation would improve the performace. - */ -class Boosting : public Aggregating { -protected: - std::vector lm_wgt; ///< hypothesis weight - bool convex; ///< convex or linear combination - - // runtime parameters - bool grad_desc_view; ///< Traditional way or gradient descent - REAL min_cst, min_err; - -public: - /// Calculate @f$c(F(x),y)@f$ and its derivative - const cost::Cost& cost_functor; - - explicit Boosting (bool cvx = false, const cost::Cost& = cost::_cost); - Boosting (const Aggregating&); - explicit Boosting (std::istream& is): cost_functor(cost::_cost) - { is >> *this; } - - virtual const id_t& id () const; - virtual Boosting* create () const { return new Boosting(); } - virtual Boosting* clone () const { return new Boosting(*this); } - - bool is_convex () const { return convex; } - REAL model_weight (UINT n) const { return lm_wgt[n]; } - void use_gradient_descent (bool gd = true) { grad_desc_view = gd; } - void set_min_cost (REAL mincst) { min_cst = mincst; } - void set_min_error (REAL minerr) { min_err = minerr; } - virtual REAL margin_norm () const; - virtual REAL margin_of (const Input&, const Output&) const; - virtual REAL margin (UINT) const; - - virtual bool support_weighted_data () const { return true; } - virtual void train (); - virtual void reset (); - virtual Output operator() (const Input&) const; - virtual Output get_output (UINT) const; - -#if BOOSTING_OUTPUT_CACHE - virtual void set_train_data (const pDataSet&, const pDataWgt& = 0); - -private: - mutable std::vector cache_n; - mutable std::vector cache_y; -protected: - inline void clear_cache (UINT idx) const { - assert(idx < n_samples && cache_n.size() == n_samples); - cache_n[idx] = 0; - cache_y[idx].clear(); - } - inline void clear_cache () const { - cache_n.clear(); cache_y.clear(); - cache_n.resize(n_samples, 0); - cache_y.resize(n_samples); - } -#endif - -protected: - REAL model_weight_sum () const { - return std::accumulate - (lm_wgt.begin(), lm_wgt.begin()+n_in_agg, REAL(0)); - } - - //@{ @name Common routines - pLearnModel train_with_smpwgt (const pDataWgt&) const; - //@} - - //@{ @name Traditional way - /// Assign weight to a newly generated hypothesis - /** We assume @a l is not but will be added. - * @param sw the sample weight used in training @a l. - * @param l the newly generated hypothesis. - * @return The weight of @a l. A nonpositive weight means the - * hypothesis @a l should not be added into the aggregation. - */ - REAL assign_weight (const DataWgt& sw, const LearnModel& l) { - assert(n_samples == sw.size()); - return convex? convex_weight(sw, l) : linear_weight(sw, l); - } - /// Update sample weights after adding the new hypothesis - /** We assume @a l has just been added to the aggregation. - * @param sw the sample weight before adding @a l. - * @param l the newly added hypothesis. - * @return The updated sample weights. - */ - pDataWgt update_smpwgt (const DataWgt& sw, const LearnModel& l) { - assert(n_in_agg > 0 && lm[n_in_agg-1] == &l); - assert(n_samples == sw.size()); - DataWgt* pdw = new DataWgt(sw); // create a separate copy - convex? convex_smpwgt(*pdw) : linear_smpwgt(*pdw); - return pdw; - } - - /** @copydoc assign_weight */ - virtual REAL convex_weight (const DataWgt&, const LearnModel&); - virtual REAL linear_weight (const DataWgt&, const LearnModel&); - virtual void convex_smpwgt (DataWgt&); - virtual void linear_smpwgt (DataWgt&); - //@} - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); - - //@{ @name Gradient-descent view -public: - friend struct _boost_gd; - REAL cost () const; - -protected: - virtual void train_gd (); ///< Training using gradient-descent - pDataWgt sample_weight () const;///< Sample weights for new hypothesis - -public: - /// Weight in gradient descent - class BoostWgt { - std::vector lm; - std::vector lm_wgt; - - public: - BoostWgt () {} - BoostWgt (const std::vector& _lm, - const std::vector& _wgt) - : lm(_lm), lm_wgt(_wgt) { assert(lm.size() == lm_wgt.size()); } - - UINT size () const { return lm.size(); } - const std::vector& models () const { return lm; } - const std::vector& weights () const { return lm_wgt; } - void clear () { lm.clear(); lm_wgt.clear(); } - - BoostWgt& operator+= (const BoostWgt&); - BoostWgt& operator*= (REAL); - BoostWgt operator- () const; -#ifndef NDEBUG - bool operator== (const BoostWgt& w) const { - return (lm == w.lm && lm_wgt == w.lm_wgt); - } -#endif - }; - //@} -}; - -struct _boost_gd { - Boosting* b; - UINT max_step; - explicit _boost_gd (Boosting* pb) : b(pb) - { max_step = b->max_n_model - b->size(); } - - REAL cost () const { return b->cost(); } - - Boosting::BoostWgt weight () const { - return Boosting::BoostWgt(b->lm, b->lm_wgt); - } - - void set_weight (const Boosting::BoostWgt& bw) const { -#if BOOSTING_OUTPUT_CACHE - b->clear_cache(); -#endif - b->lm = bw.models(); b->lm_wgt = bw.weights(); - b->n_in_agg = b->lm.size(); - assert(b->lm.size() == b->lm_wgt.size()); - for (UINT i = 0; i < b->lm.size(); ++i) - b->set_dimensions(*(b->lm[i])); - } - - Boosting::BoostWgt gradient () const { - std::vector lm = b->lm; - std::vector wgt(lm.size(), 0); - - lm.push_back(b->train_with_smpwgt(b->sample_weight())); - wgt.push_back(-1); - return Boosting::BoostWgt(lm, wgt); - } - - bool stop_opt (UINT step, REAL cst) const { - return (step >= max_step || cst < b->min_cst); - } -}; - -namespace op { - -template -R inner_product (const Boosting::BoostWgt& w1, const Boosting::BoostWgt& w2) { -#ifndef NDEBUG - std::vector w1t(w1.size()); w1t.back() = -1; - assert(w1.weights() == w1t); - std::vector w2t(w2.size()); w2t.back() = -1; - assert(w2.weights() == w2t); -#endif - LearnModel& g1 = *w1.models().back(); - LearnModel& g2 = *w2.models().back(); - - UINT n = g1.train_data()->size(); - assert(g1.train_data() == g2.train_data()); - R sum = 0; - for (UINT i = 0; i < n; ++i) - sum += g1.get_output(i)[0] * g2.get_output(i)[0]; - return sum / n; -} - -} // namespace lemga::op -} // namespace lemga - -#ifdef __BOOSTING_H__ -#warning "This header file may conflict with another `boosting.h' file." -#endif -#define __BOOSTING_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.cpp deleted file mode 100644 index db72872..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/** @file - * $Id: cascade.cpp 2550 2006-01-17 04:00:54Z ling $ - */ - -#include -#include "cascade.h" - -namespace lemga { - -bool Cascade::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Aggregating, os, vl, 1); - assert(lm.size() == upper_margin.size() && - lm.size() == lower_margin.size()); - for (UINT i = 0; i < lm.size(); ++i) - if (!(os << '(' << lower_margin[i] << ',' - << upper_margin[i] << ')' << ' ')) return false; - return true; -} - -bool Cascade::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Aggregating, is, vl, 1, v); - - const UINT n = lm.size(); - upper_margin.resize(n); - lower_margin.resize(n); - - for (std::vector::iterator - pu = upper_margin.begin(), pl = lower_margin.begin(); - pu != upper_margin.end(); ++pu, ++pl) - { - char c; - if (!(is >> c >> *pl >> c >> *pu >> c)) return false; - if (*pl > *pu) return false; - } - - return true; -} - -Output Cascade::operator() (const Input& x) const { - assert(n_in_agg > 0 && n_in_agg <= lm.size()); - assert(lm.size() == upper_margin.size() && - lm.size() == lower_margin.size()); - - for (UINT i = 0; i < n_in_agg-1; ++i) { - assert(lm[i] != NULL); - Output out = (*lm[i])(x); - REAL b = belief(*lm[i], x, out); - //assert(lower_margin[i] < upper_margin[i]); - if (b > upper_margin[i] || b < lower_margin[i]) - return out; - } - - return (*lm[n_in_agg-1])(x); -} - -/** We use the ``sign'' part of margin in AdaBoost */ -REAL Cascade::belief (const LearnModel& l, const Input& x, - const Output& y) const { - assert(l(x) == y); - assert(l.n_output() == 1); - return y[0]; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.h deleted file mode 100644 index 6f20ace..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cascade.h +++ /dev/null @@ -1,76 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_CASCADE_H__ -#define __LEMGA_AGGREGATING_CASCADE_H__ - -/** @file - * @brief Declare @link lemga::Cascade Cascade@endlink class - * - * $Id: cascade.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "aggregating.h" - -namespace lemga { - -/** @brief Aggregate hypotheses in a cascade (sequential) way. - * - * For classification problems, aggregating of hypotheses can be - * done in a cascade way. That is, a list of classifiers are trained - * from the training data; an unknown input is first classified - * using the first hypothesis in the list. If the first hypothesis - * cannot decide the classification with high reliability, the input - * is fed into the next one and so on. (See Cascade::operator() for - * details.) We can have as many hypotheses as the problem demands. - * - * The ``reliability'' of a decision is usually determined by a - * concept named @em margin. There exist different definitions in - * literature, such as @f$yf(x)@f$ in %AdaBoost, - * @f$y(w\cdot x-t)/|w|@f$ in SVM. Despite of the differences in - * definitions, higher margins usually implicit more robustness to - * input disturbance and thus better generalization. - * - * When margin is used in cascade to decide whether to go on to the - * next hypothesis, the real output @a y is unknown. A natual - * alternative is to use the ``sign'' part of the margin, i.e., - * @f$f(x)@f$ in %AdaBoost and @f$(w\cdot x-t)/|w|@f$ in SVM, and - * take the magnitude as the margin. Similar concepts, such as - * belief in belief propagation and log-likelihood in coding, can - * also be used. - * - * We use the name ``belief'' in this class for binary-class problems. - * Very positive and very negative beliefs indicate strong confidence - * in the predicting and thus high reliability. - * - * @todo General definition of margin; More explanation of Cascade - */ -class Cascade : public Aggregating { -protected: - std::vector upper_margin; ///< - std::vector lower_margin; ///< - -public: - virtual Cascade* create () const = 0; - virtual Cascade* clone () const = 0; - - /** @todo Unclear about the support of weghted data */ - virtual bool support_weighted_data () const { return true; } - virtual void train () = 0; - virtual Output operator() (const Input&) const; - - /// Belief at a specific pair of input and output - virtual REAL belief (const LearnModel&, - const Input&, const Output&) const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __CASCADE_H__ -#warning "This header file may conflict with another `cascade.h' file." -#endif -#define __CASCADE_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.cpp deleted file mode 100644 index c76ea3f..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.cpp +++ /dev/null @@ -1,189 +0,0 @@ -/** @file - * $Id: cgboost.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include -#include "cgboost.h" -#include "vectorop.h" -#include "optimize.h" - -/** The trained hypothesis f (with norm 1) is the real used search - * direction. One option is to replace cgd (d) by f everytime, while - * trying to keep the norm of d. - */ -#define USE_F_FOR_D false - -REGISTER_CREATOR(lemga::CGBoost); - -namespace lemga { - -void CGBoost::reset () { - Boosting::reset(); - all_wgts.clear(); -} - -bool CGBoost::set_aggregation_size (UINT n) { - if (grad_desc_view) { - assert(size() == all_wgts.size() || size()+1 == all_wgts.size()); - if (n > all_wgts.size()) return false; - if (n > 0) lm_wgt = all_wgts[n-1]; -#if BOOSTING_OUTPUT_CACHE - clear_cache(); -#endif - } - return Boosting::set_aggregation_size(n); -} - -bool CGBoost::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Boosting, os, vl, 1); - if (grad_desc_view) { - const UINT n = size(); - assert(n == all_wgts.size() || n+1 == all_wgts.size()); - if (!(os << n << '\n')) return false; - for (UINT i = 0; i+1 < n; ++i) { // Boosting saved the last one - assert(all_wgts[i].size() == i+1); - for (UINT j = 0; j <= i; ++j) - if (!(os << all_wgts[i][j] << ' ')) return false; - if (!(os << '\n')) return false; - } - return true; - } - else - return (os << 0 << '\n'); -} - -bool CGBoost::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Boosting, is, vl, 1, v); - assert(v > 0); - - UINT n; - if (!(is >> n)) return false; - if (n > 0 && n != size()) return false; - - if (n == 0 && size() > 0) - use_gradient_descent(false); - - if (n > 0) { - use_gradient_descent(true); - all_wgts.clear(); - for (UINT i = 1; i < n; ++i) { - std::vector wgt(i); - for (UINT j = 0; j < i; ++j) - if (!(is >> wgt[j])) return false; - all_wgts.push_back(wgt); - } - all_wgts.push_back(lm_wgt); - } - - return true; -} - -void CGBoost::train () { - if (!grad_desc_view) { - using namespace op; - ncd = *ptw; ncd *= n_samples; // optional, make numbers not too small - cgd = ncd; - cur_err.resize(n_samples); - } - - Boosting::train(); - - cur_err.clear(); - ncd.clear(); cgd.clear(); -} - -void CGBoost::train_gd () { - _boost_cg bcg(this); - iterative_optimize(_conjugate_gradient<_boost_cg,BoostWgt,REAL,REAL> - (&bcg, convex? 1 : 0.5)); -} - -/** @note The sample weight (probability) used in training a new - * hypothesis is no longer same as the one for calculating the - * step length, which is exactly the gradient. We have to recover - * the gradient from CGBoost::ncd. - */ -REAL CGBoost::linear_weight (const DataWgt&, const LearnModel& l) { - assert(exact_dimensions(l)); - assert(l.train_data() == ptd); - - REAL cor = 0, err = 0; - for (UINT i = 0; i < n_samples; ++i) { - assert(ncd[i] >= 0); - cur_err[i] = l.c_error(l.get_output(i), ptd->y(i)); - if (cur_err[i] > 0.1) - err += ncd[i]; - else cor += ncd[i]; - } - assert(err+cor > 0); -#if VERBOSE_OUTPUT - std::cout << "?Weighted classification error: " << - err/(err+cor)*100 << "%%\n"; -#endif - - if (err >= cor) return -1; - - REAL beta; - if (err <= 0) - beta = 1000; - else - beta = cor / err; - return std::log(beta) / 2; -} - -/** ncd is actually - * @f[ -g/y_i = -\frac{w_i}{y_i} c'_F(F(x_i),y_i) = w_i e^{-y_iF(x_i)} @f] - * It can be iteratively computed as - * @f[ ncd_i \leftarrow ncd_i \cdot e^{-\alpha y_i f(x_i)} @f] - */ -void CGBoost::linear_smpwgt (DataWgt& sw) { - // update ratio (\beta) for error and correct samples - const REAL be = std::exp(lm_wgt[n_in_agg-1]), bc = 1 / be; - REAL s1 = 0, s2 = 0; - for (UINT i = 0; i < n_samples; ++i) { - const REAL tmp = ncd[i] * (cur_err[i]? be : bc); - s1 += tmp * (tmp - ncd[i]); - s2 += ncd[i] * ncd[i]; - ncd[i] = tmp; - assert(fabs(ncd[i] - n_samples * (*ptw)[i] * - std::exp(- ptd->y(i)[0]*get_output(i)[0])) < EPSILON); - } - assert(s2 != 0); - REAL beta = s1 / s2; - if (beta < 0) beta = 0; - -#if USE_F_FOR_D - /* Compute the norm ratio between d and f, which will mutiply - * f so as to keep the correct norm. */ - REAL d2_sum = 0; - for (UINT i = 0; i < n_samples; i++) - d2_sum += cgd[i] * cgd[i]; - const REAL cf_ratio = std::sqrt(d2_sum / n_samples); -#if VERBOSE_OUTPUT - std::cout << "cf_ratio = " << cf_ratio << ", "; -#endif -#endif - - REAL bw_sum = 0; - for (UINT i = 0; i < n_samples; ++i) { -#if USE_F_FOR_D - /* if we want to use f insted of cgd */ - cgd[i] = (cur_err[i] > 0.1)? -cf_ratio : cf_ratio; -#endif - cgd[i] = ncd[i] + beta * cgd[i]; - bw_sum += cgd[i]; - } -#if VERBOSE_OUTPUT - std::cout << "beta = " << beta << '\n'; -#endif - - assert(bw_sum != 0); - for (UINT i = 0; i < n_samples; ++i) { - sw[i] = cgd[i] / bw_sum; - assert(sw[i] >= 0); - } -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.h deleted file mode 100644 index 9dccb6e..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cgboost.h +++ /dev/null @@ -1,101 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_CGBOOST_H__ -#define __LEMGA_AGGREGATING_CGBOOST_H__ - -/** @file - * @brief Declare @link lemga::CGBoost CGBoost@endlink class. - * - * $Id: cgboost.h 2696 2006-04-05 20:10:13Z ling $ - */ - -#include "boosting.h" - -namespace lemga { - -struct _boost_cg; - -/** @brief %CGBoost (Conjugate Gradient Boosting). - * - * This class provides two ways to implement the conjugate gradient - * idea in the Boosting frame. - * - * The first way is to manipulate the sample weight. - * - * The other way is to adjust the projected search direction @a f. - * The adjusted direction is also a linear combination of weak learners. - * We prefer this way (by @c use_gradient_descent(true)). - * - * Differences between AdaBoost and CGBoost (gradient descent view): - * - The weights of all hypotheses (CGBoost), instead of only the - * weight of the newly added hypothesis (AdaBoost), will be updated - * during one iteration. Thus in order to ``set_aggregation_size()'' - * correctly, we have to save weights in every iteration. - * - * @todo Documentation - */ -class CGBoost : public Boosting { - std::vector > all_wgts; - friend struct _boost_cg; - -protected: - /* only valid within training */ - std::vector ncd, //!< @f$-g/y_i@f$ where @f$g=\nabla C(F)@f$ - cgd; //!< @f$d/y_i@f$ where @f$d=-g+\beta d_{\mathrm{prev}}@f$ - -public: - explicit CGBoost (bool cvx = false, const cost::Cost& c = cost::_cost) - : Boosting(cvx, c) {} - CGBoost (const Boosting& s) : Boosting(s) { - const std::vector::const_iterator b = lm_wgt.begin(); - for (UINT i = 1; i <= lm_wgt.size(); ++i) - all_wgts.push_back(std::vector(b, b+i)); - } - explicit CGBoost (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual CGBoost* create () const { return new CGBoost(); } - virtual CGBoost* clone () const { return new CGBoost(*this); } - - virtual bool set_aggregation_size (UINT); - virtual void train (); - virtual void reset (); - -protected: - /// data only valid within training (remove?) - std::vector cur_err; - - virtual void train_gd (); - virtual REAL linear_weight (const DataWgt&, const LearnModel&); - virtual void linear_smpwgt (DataWgt&); - - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -struct _boost_cg : public _boost_gd { - CGBoost* cg; - _boost_cg (CGBoost* pc) : _boost_gd(pc), cg(pc) {} - - void set_weight (const Boosting::BoostWgt& bw) const { - _boost_gd::set_weight(bw); - assert(cg->n_in_agg == bw.size() && cg->n_in_agg == cg->lm_wgt.size()); - - // save weights to all_wgts - if (cg->n_in_agg == 0) return; - const UINT n = cg->n_in_agg - 1; - if (n < cg->all_wgts.size()) - cg->all_wgts[n] = cg->lm_wgt; - else { - assert(n == cg->all_wgts.size()); // allow size inc <= 1 - cg->all_wgts.push_back(cg->lm_wgt); - } - } -}; - -} // namespace lemga - -#ifdef __CGBOOST_H__ -#warning "This header file may conflict with another `cgboost.h' file." -#endif -#define __CGBOOST_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cost.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cost.h deleted file mode 100644 index 6650c4f..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/cost.h +++ /dev/null @@ -1,95 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_COST_H__ -#define __LEMGA_COST_H__ - -/** @file - * @brief Cost functions (functors) used in learning - * - * $Id: cost.h 2537 2006-01-08 08:40:36Z ling $ - */ - -#include -#include "object.h" - -namespace lemga { -namespace cost { - -template -struct AdaCost { - virtual ~AdaCost () { /* get rid of GCC 4.0 warnings */ } - virtual _Num cost (const _Num& F, const _Num& y) const { - assert(std::fabs(y*y-1) < INFINITESIMAL); - return std::exp(-(F * y)); - } - inline _Num operator() (const _Num& F, const _Num& y) const - { return cost(F, y); } - - virtual _Num deriv1 (const _Num& F, const _Num& y) const { - assert(std::fabs(y*y-1) < INFINITESIMAL); - return -std::exp(- (F * y)) * y; - } -}; - -/* A temporary workaround: Using real class instead of functor */ -typedef AdaCost Cost; - -const Cost _cost = Cost(); - -struct exponential : public Cost { - REAL lambda; - exponential () : lambda(1) {}; - virtual REAL cost (const REAL& F, const REAL& y) const { - return std::exp(-lambda*F*y); - } - virtual REAL deriv1 (const REAL& F, const REAL& y) const { - return -lambda*y * std::exp(-lambda*F*y); - } -}; - -struct logistic : public Cost { - REAL lambda; - logistic () : lambda(1) {}; - virtual REAL cost (const REAL& F, const REAL& y) const { - return std::log(1 + std::exp(-lambda*F*y)); - } - virtual REAL deriv1 (const REAL& F, const REAL& y) const { - const REAL t = std::exp(-lambda*F*y); - return -lambda*y * t / (1+t); - } -}; - -struct sigmoid : public Cost { - REAL lambda; - sigmoid () : lambda(1) {}; - virtual REAL cost (const REAL& F, const REAL& y) const { - return 1 - std::tanh(lambda*F*y); - } - virtual REAL deriv1 (const REAL& F, const REAL& y) const { - const REAL t = std::tanh(lambda*F*y); - return lambda*y * (t*t - 1); - } -}; - -struct bisigmoid : public Cost { - REAL lambda; - REAL ratio; ///< ratio = lambda_neg / lambda <= 1 - bisigmoid () : lambda(1), ratio(0.8) {}; - virtual REAL cost (const REAL& F, const REAL& y) const { - const REAL m = lambda*F*y; - if (m > 0) return 1 - std::tanh(m)/lambda; - return 1 - std::tanh(ratio*m)/(ratio*lambda); - } - virtual REAL deriv1 (const REAL& F, const REAL& y) const { - const REAL m = lambda*F*y; - const REAL t = std::tanh(m>0? m:(ratio*m)); - return y * (t*t - 1); - } -}; - -}} // namespace lemga::cost - -#ifdef __COST_H__ // check duplicate filenames -#warning "This header file may conflict with another `cost.h' file." -#endif -#define __COST_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.cpp deleted file mode 100644 index 42b3065..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/** @file - * $Id: crossval.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "vectorop.h" -#include "random.h" -#include "crossval.h" - -REGISTER_CREATOR2(lemga::vFoldCrossVal, vfold); -REGISTER_CREATOR2(lemga::HoldoutCrossVal, holdout); - -namespace lemga { - -CrossVal::CrossVal (const CrossVal& cv) - : LearnModel(cv), fullset(cv.fullset), lm(cv.lm), err(cv.err), - n_rounds(cv.n_rounds), best(cv.best) -{ - best_lm = 0; - if (cv.best_lm != 0) { - best_lm = cv.best_lm->clone(); - assert(best >= 0 && best_lm->id() == lm[best]->id()); - } -} - -const CrossVal& CrossVal::operator= (const CrossVal& cv) { - if (&cv == this) return *this; - - LearnModel::operator=(cv); - fullset = cv.fullset; - lm = cv.lm; - err = cv.err; - n_rounds = cv.n_rounds; - best = cv.best; - - best_lm = 0; - if (cv.best_lm != 0) { - best_lm = cv.best_lm->clone(); - assert(best >= 0 && best_lm->id() == lm[best]->id()); - } - - return *this; -} - -bool CrossVal::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - - UINT n = size(); - if (!(os << n << ' ' << best << ' ' << (best_lm != 0) << '\n')) - return false; - for (UINT i = 0; i < n; ++i) - if (!(os << *lm[i])) return false; - for (UINT i = 0; i < n; ++i) - if (!(os << err[i] << ' ')) return false; - os << '\n'; - if (best_lm != 0) { - assert(best >= 0 && best_lm->id() == lm[best]->id()); - if (!(os << *best_lm)) return false; - } - return (os << n_rounds << ' ' << fullset << '\n'); -} - -bool CrossVal::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - assert(d == NIL_ID); - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - assert(v > 0); - - UINT n; - bool trained; - if (!(is >> n >> best >> trained)) return false; - if (best < -1 || best >= (int) n) return false; - - lm.clear(); err.resize(n); - for (UINT i = 0; i < n; ++i) { - LearnModel* p = (LearnModel*) Object::create(is); - lm.push_back(p); - if (p == 0 || !valid_dimensions(*p)) return false; - } - for (UINT i = 0; i < n; ++i) - if (!(is >> err[i])) return false; - - best_lm = 0; - if (trained) { - LearnModel* p = (LearnModel*) Object::create(is); - best_lm = p; - if (p == 0 || !exact_dimensions(*p)) return false; - if (best < 0 || p->id() != lm[best]->id()) return false; - } - - return (is >> n_rounds >> fullset) && (n_rounds > 0); -} - -void CrossVal::add_model (const LearnModel& l) { - set_dimensions(l); - lm.push_back(l.clone()); - err.push_back(-1); -} - -void CrossVal::set_train_data (const pDataSet& pd, const pDataWgt& pw) { - assert(pw == 0); // cannot deal with sample weights - LearnModel::set_train_data(pd, 0); - if (best_lm != 0) { - assert(best >= 0 && best_lm->id() == lm[best]->id()); - best_lm->set_train_data(pd, 0); - } -} - -void CrossVal::train () { - assert(n_rounds > 0 && ptd != 0 && ptw == 0); - best_lm = 0; - - std::fill(err.begin(), err.end(), 0); - using namespace op; - for (UINT r = 0; r < n_rounds; ++r) - err += cv_round(); - err *= 1 / (REAL) n_rounds; - - best = std::min_element(err.begin(), err.end()) - err.begin(); - if (fullset) { - best_lm = lm[best]->clone(); - best_lm->initialize(); - best_lm->set_train_data(ptd); - best_lm->train(); - set_dimensions(*best_lm); - } -} - -void CrossVal::reset () { - LearnModel::reset(); - std::fill(err.begin(), err.end(), -1); - best_lm = 0; best = -1; -} - -bool vFoldCrossVal::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(CrossVal, os, vl, 1); - return (os << n_folds << '\n'); -} - -bool -vFoldCrossVal::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(CrossVal, is, vl, 1, v); - assert(v > 0); - return (is >> n_folds) && (n_folds > 1); -} - -std::vector vFoldCrossVal::cv_round () const { - assert(ptd != 0); - UINT n = size(), ds = ptd->size(); - std::vector cve(n, 0); - - // get a random index - std::vector perm(ds); - for (UINT i = 0; i < ds; ++i) perm[i] = i; - std::random_shuffle(perm.begin(), perm.end()); - - UINT b, e = 0; - for (UINT f = 1; f <= n_folds; ++f) { - // [b,e) is the index range for the testing set - b = e; e = f * ds / n_folds; - assert(e-b == ds/n_folds || e-b == (ds+n_folds-1)/n_folds); - - // generate the training and testing sets - DataSet *p_tr = new DataSet(); - DataSet *p_te = new DataSet(); - //! using perm[i] is bad for system cache - for (UINT i = 0; i < b; ++i) - p_tr->append(ptd->x(perm[i]), ptd->y(perm[i])); - for (UINT i = b; i < e; ++i) - p_te->append(ptd->x(perm[i]), ptd->y(perm[i])); - for (UINT i = e; i < ds; ++i) - p_tr->append(ptd->x(perm[i]), ptd->y(perm[i])); - pDataSet ptr = p_tr, pte = p_te; - - // go over all candidates and collect the errors - for (UINT i = 0; i < n; ++i) { - pLearnModel p = lm[i]->clone(); - p->set_train_data(ptr); - p->train(); - // which error to collect? let's assume classification error - cve[i] += p->test_c_error(pte) * pte->size(); - } - } - using namespace op; - cve *= 1 / (REAL) ds; - - return cve; -} - -bool HoldoutCrossVal::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(CrossVal, os, vl, 1); - return (os << p_test << '\n'); -} - -bool -HoldoutCrossVal::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(CrossVal, is, vl, 1, v); - assert(v > 0); - return (is >> p_test) && (p_test > 0 && p_test < 0.9); -} - -std::vector HoldoutCrossVal::cv_round () const { - assert(ptd != 0); - const UINT n = ptd->size(); - UINT k = UINT(n * p_test + 0.5); if (k < 1) k = 1; - DataSet *p_tr = new DataSet(); - DataSet *p_te = new DataSet(); - - // (n,k): choosing k examples from n ones. - // To generate (n,k), we pick the 1st example with probability k/n, - // and do (n-1,k-1) if the example is picked, or (n-1,k) if it is not. - // Note: we may break out when k reaches 0 to save some randu() calls. - for (UINT i = 0; i < n; ++i) { - UINT toss = UINT(randu() * (n-i)); - assert(0 <= toss && toss < n-i); - if (toss < k) { - p_te->append(ptd->x(i), ptd->y(i)); - --k; - } else - p_tr->append(ptd->x(i), ptd->y(i)); - } - assert(k == 0); - - pDataSet ptr = p_tr, pte = p_te; - const UINT lms = size(); - std::vector cve(lms); - // go over all candidates and collect the errors - for (UINT i = 0; i < lms; ++i) { - pLearnModel p = lm[i]->clone(); - p->set_train_data(ptr); - p->train(); - // which error to collect? let's assume classification error - cve[i] = p->test_c_error(pte); - } - - return cve; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.h deleted file mode 100644 index 8d031f4..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/crossval.h +++ /dev/null @@ -1,149 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_CROSSVAL_H__ -#define __LEMGA_CROSSVAL_H__ - -/** @file - * @brief Declare @link lemga::CrossVal Cross-Validation@endlink classes. - * - * $Id: crossval.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "shared_ptr.h" -#include "learnmodel.h" - -namespace lemga { - -/** @brief A combination of cross-validation and model selection. - * - * @note The interface is experimental. Say, it might be under LearnModel. - */ -class CrossVal : public LearnModel { -protected: - bool fullset; ///< train the best model on the full set? - std::vector lm;///< all candidate models - std::vector err; ///< cross-validation errors - UINT n_rounds; ///< # of CV rounds, to beat the variance - pLearnModel best_lm; ///< the best model (trained on the full set) - int best; ///< @a best_lm was actually @a lm[best] - ///< @note Before cross-validation, best is -1. After, lm[best] is the - ///< best model. If full-training is required, best_lm is then assigned. - -public: - CrossVal () : fullset(true), n_rounds(1), best(-1) {} - CrossVal (const CrossVal&); - const CrossVal& operator= (const CrossVal&); - - virtual CrossVal* create () const = 0; - virtual CrossVal* clone () const = 0; - - /// add a candidate model to be cross-validated - void add_model (const LearnModel&); - /// the number of candidate models under cross-validation - UINT size () const { assert(lm.size() == err.size()); return lm.size(); } - /// the n-th candidate model - const LearnModel& model (UINT n) const { - assert(n < size() && lm[n] != 0); return *lm[n]; } - - /// how many rounds of cross-validation? - UINT rounds () const { return n_rounds; } - /// specifiy the number of rounds of cross-validation - void set_rounds (UINT r) { assert(r > 0); n_rounds = r; } - /// train the best model on the full set? - bool full_train () const { return fullset; } - void set_full_train (bool f = true) { fullset = f; } - - virtual void set_train_data (const pDataSet&, const pDataWgt& = 0); - virtual void train (); - virtual void reset (); - virtual Output operator() (const Input& x) const { - assert(best >= 0 && best_lm != 0); - return (*best_lm)(x); } - virtual Output get_output (UINT i) const { - assert(best >= 0 && best_lm != 0 && ptd == best_lm->train_data()); - return best_lm->get_output(i); } - virtual REAL margin_norm () const { - assert(best >= 0 && best_lm != 0); - return best_lm->margin_norm(); } - virtual REAL margin_of (const Input& x, const Output& y) const { - assert(best >= 0 && best_lm != 0); - return best_lm->margin_of(x, y); } - virtual REAL margin (UINT i) const { - assert(best >= 0 && best_lm != 0 && ptd == best_lm->train_data()); - return best_lm->margin(i); } - - /// the cross-validation error of the n-th candidate model - REAL error (UINT n) const { - assert(n < size() && err[n] >= 0); return err[n]; } - /// the best model (trained if full_train() == true) - const LearnModel& best_model () const { - assert(best >= 0); - return best_lm? *best_lm : *lm[best]; } - -protected: - /// one round of the cross-validation operation - virtual std::vector cv_round () const = 0; - - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -/// v-fold cross validation. -class vFoldCrossVal : public CrossVal { -public: - vFoldCrossVal (UINT v = 10, UINT r = 0) { set_folds(v, r); } - explicit vFoldCrossVal (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual vFoldCrossVal* create () const { return new vFoldCrossVal(); } - virtual vFoldCrossVal* clone () const { return new vFoldCrossVal(*this); } - - UINT folds () const { return n_folds; } - /// set the folds and optionally also set the number of rounds - void set_folds (UINT v, UINT r = 0) { - assert(v > 1); n_folds = v; - if (r > 0) set_rounds(r); - } - -protected: - UINT n_folds; - virtual std::vector cv_round () const; - - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; -typedef vFoldCrossVal kFoldCrossVal; - -/// Randomized holdout cross-validation. -class HoldoutCrossVal : public CrossVal { -public: - HoldoutCrossVal (REAL p = 1.0/6, UINT r = 0) { set_holdout(p, r); } - explicit HoldoutCrossVal (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual HoldoutCrossVal* create () const { return new HoldoutCrossVal(); } - virtual HoldoutCrossVal* clone () const { - return new HoldoutCrossVal(*this); } - - REAL holdout () const { return p_test; } - /// set the holdout portion and optionally set the number of rounds - void set_holdout (REAL p, UINT r = 0) { - assert(p > 0 && p < 0.9); p_test = p; - if (r > 0) set_rounds(r); - } - -protected: - REAL p_test; - virtual std::vector cv_round () const; - - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __CROSSVAL_H__ -#warning "This header file may conflict with another `crossval.h' file." -#endif -#define __CROSSVAL_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.cpp deleted file mode 100644 index 7d18296..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.cpp +++ /dev/null @@ -1,164 +0,0 @@ -/** @file - * $Id: datafeeder.cpp 2563 2006-01-20 05:04:22Z ling $ - */ - -#include -#include -#include -#include "random.h" -#include "datafeeder.h" - -namespace lemga { - -DataFeeder::DataFeeder (const pDataSet& pd) - : dat(pd), perms(0), _do_normalize(MIN_MAX), tr_size(0), tr_flip(0) { - fsize = dat->size(); -} - -DataFeeder::DataFeeder (std::istream& ds) - : perms(0), _do_normalize(MIN_MAX), tr_size(0), tr_flip(0) { - /* load all the data */ - dat = load_data(ds, (1L<<30)-1); - fsize = dat->size(); -} - -void DataFeeder::set_train_size (UINT trn) { - assert(trn < fsize); - tr_size = trn; -} - -void DataFeeder::set_train_noise (REAL p) { - assert(p >= 0 && p <= 1); - tr_flip = p; -} - -bool DataFeeder::next_train_test (pDataSet& ptr, pDataSet& pte) const { - DataSet *p_tr = new DataSet(); - DataSet *p_te = new DataSet(); - - std::vector perm; - if (!next_permutation(perm)) return false; - - for (UINT i = 0; i < tr_size; ++i) { - bool flip = (tr_flip > 0) && (randu() < tr_flip); - if (!flip) - p_tr->append(dat->x(perm[i]), dat->y(perm[i])); - else { - const Output& y = dat->y(perm[i]); - assert(y.size() == 1 && - std::fabs(std::fabs(y[0]) - 1) < INFINITESIMAL); - p_tr->append(dat->x(perm[i]), Output(1, -y[0])); - } - } - for (UINT i = tr_size; i < fsize; ++i) - p_te->append(dat->x(perm[i]), dat->y(perm[i])); - - if (_do_normalize != NONE) { - LINEAR_SCALE_PARAMS lsp; - switch (_do_normalize) { - case MIN_MAX: lsp = min_max(*p_tr); break; - case MEAN_VAR: lsp = mean_var(*p_tr); break; - default: assert(false); - } - linear_scale(*p_tr, lsp); - linear_scale(*p_te, lsp); - } - - ptr = p_tr; pte = p_te; - return true; -} - -bool DataFeeder::next_permutation (std::vector& perm) const { - perm.resize(fsize); - - if (perms == 0) { - for (UINT i = 0; i < fsize; ++i) - perm[i] = i; - std::random_shuffle(perm.begin(), perm.end()); - return true; - } - - std::vector visited(fsize, false); - for (UINT i = 0; i < fsize; ++i) { - UINT idx; // starting from 0 - if (!((*perms) >> idx)) { - if (i) std::cerr << "DataFeeder: " - "Permutation stream ends prematurely\n"; - return false; - } - if (idx >= fsize || visited[idx]) { - std::cerr << "DataFeeder: " - "Permutation stream has errors\n"; - return false; - } - visited[idx] = true; - perm[i] = idx; - } - return true; -} - -DataFeeder::LINEAR_SCALE_PARAMS DataFeeder::min_max (DataSet& d) { - assert(d.size() > 0); - - const Input& x0 = d.x(0); - const UINT ls = x0.size(); - std::vector dmin(x0), dmax(x0); - for (UINT i = 1; i < d.size(); ++i) { - const Input& x = d.x(i); - for (UINT j = 0; j < ls; ++j) { - if (dmin[j] > x[j]) - dmin[j] = x[j]; - else if (dmax[j] < x[j]) - dmax[j] = x[j]; - } - } - - LINEAR_SCALE_PARAMS l(ls); - for (UINT j = 0; j < ls; ++j) { - l[j].center = (dmin[j] + dmax[j]) / 2; - if (dmin[j] != dmax[j]) - l[j].scale = 2 / (dmax[j] - dmin[j]); - else - l[j].scale = 0; - } - return l; -} - -DataFeeder::LINEAR_SCALE_PARAMS DataFeeder::mean_var (DataSet& d) { - const UINT n = d.size(); - assert(n > 0); - const UINT ls = d.x(0).size(); - - std::vector sum1(ls, 0), sum2(ls, 0); - for (UINT i = 0; i < n; ++i) { - const Input& x = d.x(i); - for (UINT j = 0; j < ls; ++j) { - sum1[j] += x[j]; - sum2[j] += x[j] * x[j]; - } - } - - LINEAR_SCALE_PARAMS l(ls); - for (UINT j = 0; j < ls; ++j) { - l[j].center = sum1[j] / n; - REAL n_1_var = sum2[j] - sum1[j] * l[j].center; - if (n_1_var > INFINITESIMAL) - l[j].scale = std::sqrt((n-1) / n_1_var); - else - l[j].scale = 0; - } - return l; -} - -void DataFeeder::linear_scale (DataSet& d, const LINEAR_SCALE_PARAMS& l) { - const UINT ls = l.size(); - for (UINT i = 0; i < d.size(); ++i) { - Input x = d.x(i); - assert(x.size() == ls); - for (UINT j = 0; j < ls; ++j) - x[j] = (x[j] - l[j].center) * l[j].scale; - d.replace(i, x, d.y(i)); - } -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.h deleted file mode 100644 index bfc7bd4..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/datafeeder.h +++ /dev/null @@ -1,71 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_DATAFEEDER_H__ -#define __LEMGA_DATAFEEDER_H__ - -/** @file - * @brief Declare @link lemga::DataFeeder DataFeeder@endlink class. - * - * $Id: datafeeder.h 2563 2006-01-20 05:04:22Z ling $ - */ - -#include -#include "learnmodel.h" - -namespace lemga { - -/** @brief Feed (random splitted) training and testing data. - */ -class DataFeeder { -public: - enum NORMETHOD { // normalization method - MIN_MAX, // make min -1, max +1 - MEAN_VAR, // make mean 0, var 1 - NONE - }; - -private: - pDataSet dat; - std::istream* perms; - NORMETHOD _do_normalize; - UINT fsize, tr_size; - REAL tr_flip; - -public: - DataFeeder (const pDataSet&); - DataFeeder (std::istream&); - - void set_permutation (std::istream& i) { perms = &i; } - void do_normalize (NORMETHOD dn = MIN_MAX) { _do_normalize = dn; } - UINT size () const { return fsize; } - /// the size of the training set - UINT train_size () const { return tr_size; } - /// set the size of the training set - void set_train_size (UINT); - /// the artificial flipping noise level for the training set - REAL train_noise () const { return tr_flip; } - /// set the artificial flipping noise level for the training set - void set_train_noise (REAL); - - pDataSet data () const { return dat; } - bool next_train_test (pDataSet&, pDataSet&) const; - -protected: - bool next_permutation (std::vector&) const; - - struct LINEAR_SCALE_PARAM { - REAL center, scale; - }; - typedef std::vector LINEAR_SCALE_PARAMS; - - static LINEAR_SCALE_PARAMS min_max (DataSet&); - static LINEAR_SCALE_PARAMS mean_var (DataSet&); - static void linear_scale (DataSet&, const LINEAR_SCALE_PARAMS&); -}; - -} // namespace lemga - -#ifdef __DATAFEEDER_H__ -#warning "This header file may conflict with another `datafeeder.h' file." -#endif -#define __DATAFEEDER_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/dataset.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/dataset.h deleted file mode 100644 index be19674..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/dataset.h +++ /dev/null @@ -1,176 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_DATASET_H__ -#define __LEMGA_DATASET_H__ - -/** @file - * @brief Class template @link lemga::dataset dataset@endlink for - * wrapping a vector of input-output pairs. - * - * $Id: dataset.h 2537 2006-01-08 08:40:36Z ling $ - */ - -#include -#include -#include "random.h" - -namespace lemga { - -/** @brief Class template for storing, retrieving, and manipulating a - * vector of input-output style data. - * @param Tx Type of input @a x. - * @param Ty Type of output @a y. - * - * What we want are: - * 1. a place to store data - * 2. be able to deal with missing feature? - * - * Member functions: - * - basic info.: size, dimension, ... - * - load from / save to a file - * - copy from / to another format (REAL **) - * - ways to produce new data set (by sampling...) - * - * @todo documentation - */ -template -class dataset { - /** @note An alternative way to store the input-output pairs is - * to put them into separate variables, i.e., one vector for @a x - * and one for @a y. However, usually the input @a x and output - * @a y in a same pair will be used together. Thus it is better - * for systems with cache to put together corresponding @a x and - * @a y. - */ - std::vector< std::pair > d; - -public: - typedef Tx x_type; - typedef Ty y_type; - - dataset () : d() {} - - //@{ @name Basic - UINT size () const { return d.size(); } - bool empty () const { return d.empty(); } - void clear () { d.clear(); } - const Tx& x (UINT i) const { return d[i].first; } - const Ty& y (UINT i) const { return d[i].second; } - //@} - - //@{ @name Convert from other data types - template - dataset (IIX xb, IIX xe, IIY yb, IIY ye) { import(xb, xe, yb, ye); } - - /** @brief Import data from other types. - * @param IIX stands for Input Iterator for @a x. - * @param IIY stands for Input Iterator for @a y. - * - * import() copies input @a x from range [@a xb, @a xe) and - * output @a y from range [@a yb, @a ye). Old data stored in - * the set will be erased. - * - * To import data from two vectors @a vx and @a vy, use - * @code import(vx.begin(), vx.end(), vy.begin(), vy.end()); @endcode - * To import @a n samples from two pointers px and py, use - * @code import(px, px+n, py, py+n); @endcode - */ - template - void import (IIX xb, IIX xe, IIY yb, IIY ye) { - d.clear(); - d.reserve(xe - xb); - for (; xb != xe && yb != ye; ++xb, ++yb) - append(*xb, *yb); - } - //@} - - //@{ @name Data manipulation - /** @brief Generate a randomly sampled copy of the data set. - * @param n Number of random samples requested. - * @return A pointer to the new born data set. - * - * Samples are chosen with uniform probability. - */ - dataset* random_sample (UINT n) const { - const UINT dn = d.size(); - assert(n == 0 || dn > 0); - - dataset* rd = new dataset(); - rd->d.reserve(n); - while (n--) { - const UINT sel = UINT(randu() * dn); - assert(sel < dn); - rd->d.push_back(d[sel]); - } - return rd; - } - - /** @copydoc random_sample() - * @param W Sample weight type, which usually is @c vector. - * @a W should support @c operator[]. - * @param wgt Sample weight. - */ - template - dataset* random_sample (const W& wgt, UINT n) const { - const UINT dn = d.size(); - assert(n == 0 || dn > 0); - - std::vector cdf(dn+1); - cdf[0] = 0; - for (UINT i = 0; i < dn; ++i) - cdf[i+1] = cdf[i] + wgt[i]; - assert(cdf[dn]-1 > -EPSILON && cdf[dn]-1 < EPSILON); - - dataset* rd = new dataset(); - rd->d.reserve(n); - while (n--) { - const PROBAB r = randu(); - - UINT b = 0, e = dn, m; - while (b+1 < e) { - m = (b + e) / 2; - if (r < cdf[m]) e = m; - else b = m; - } - - rd->d.push_back(d[b]); - } - return rd; - } - //@} - - /** @brief Combine two data sets. - * @note Code - * @code copy(ds.d.begin(), ds.d.end(), back_inserter(d)); @endcode - * does almost the same thing, but will fail when @a ds is just - * *this. That is, @code ds += ds; @endcode doesn't work. - * @todo We need more functions to add/remove samples. - */ - dataset& operator+= (const dataset& ds) { - const UINT n = ds.d.size(); - d.reserve(d.size() + n); - for (UINT i = 0; i < n; ++i) - d.push_back(ds.d[i]); - return *this; - } - - void append (const Tx& _x, const Ty& _y) { - d.resize(d.size()+1); // 1 constructor + 1 copy - d.back().first = _x; // 1 copy - d.back().second = _y; - //d.push_back(make_pair(_x, _y)); // 2 copy - } - - void replace (UINT i, const Tx& _x, const Ty& _y) { - assert (i < d.size()); - d[i].first = _x; - d[i].second = _y; - } -}; - -} // namespace lemga - -#ifdef __DATASET_H__ -#warning "This header file may conflict with another `dataset.h' file." -#endif -#define __DATASET_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.cpp deleted file mode 100644 index b98b5a4..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.cpp +++ /dev/null @@ -1,286 +0,0 @@ -/** @file - * $Id: feedforwardnn.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "feedforwardnn.h" -#include "optimize.h" - -REGISTER_CREATOR(lemga::FeedForwardNN); - -namespace lemga { - -void FeedForwardNN::free_space () { - for (UINT i = 1; i <= n_layer; ++i) { - assert(layer[i] != NULL); - delete layer[i]; - } - layer.resize(1); - n_layer = 0; -} - -FeedForwardNN::FeedForwardNN () - : LearnModel(0,0), n_layer(0), - online_learn(false), train_method(GRADIENT_DESCENT), - learn_rate(0.01), min_cst(0), max_run(500) -{ - layer.push_back(NULL); - /** @todo Online learning is not implemented */ -} - -FeedForwardNN::FeedForwardNN (const FeedForwardNN& nn) - : LearnModel(nn), n_layer(nn.n_layer), _y(nn._y), _dy(nn._dy), - online_learn(nn.online_learn), train_method(nn.train_method), - learn_rate(nn.learn_rate), min_cst(nn.min_cst), max_run(nn.max_run) -{ - assert(n_layer+1 == nn.layer.size()); - layer.push_back(NULL); - for (UINT i = 1; i <= n_layer; ++i) - layer.push_back(nn.layer[i]->clone()); -} - -FeedForwardNN::~FeedForwardNN () { - free_space(); -} - -const FeedForwardNN& FeedForwardNN::operator= (const FeedForwardNN& nn) { - if (&nn == this) return *this; - LearnModel::operator=(nn); - - free_space(); - n_layer = nn.n_layer; - _y = nn._y; - _dy = nn._dy; - online_learn = nn.online_learn; - train_method = nn.train_method; - learn_rate = nn.learn_rate; - min_cst = nn.min_cst; - max_run = nn.max_run; - - assert(n_layer+1 == nn.layer.size()); - for (UINT i = 1; i <= n_layer; ++i) - layer.push_back(nn.layer[i]->clone()); - - return *this; -} - -bool FeedForwardNN::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - - if (!(os << n_layer << '\n')) return false; - if (!(os << online_learn << ' ' << learn_rate << ' ' - << min_cst << ' ' << max_run << '\n')) return false; - - for (UINT i = 1; i <= n_layer; ++i) - if (!(os << *layer[i])) return false; - return true; -} - -bool -FeedForwardNN::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - - UINT tmp_layer; - if (!(is >> tmp_layer) || tmp_layer == 0) return false; - - std::vector lsize; - if (v == 0) { - lsize.resize(tmp_layer+1); - for (UINT i = 0; i <= tmp_layer; ++i) - if (!(is >> lsize[i]) || lsize[i] == 0) return false; - } - - int online; - if (!(is >> online >> learn_rate >> min_cst >> max_run)) - return false; - if (online > 1 || learn_rate <= 0 || min_cst < 0 || max_run < 1) - return false; - online_learn = (online != 0); - - // free the old model !!! - const UINT n_in_got = _n_in, n_out_got = _n_out; - free_space(); - _y.clear(); - _dy.clear(); - - for (UINT i = 0; i < tmp_layer; ++i) { - NNLayer* pl = (NNLayer*) Object::create(is); - if (pl == 0) return false; - - if (v == 0) { - if (pl->n_input() != lsize[i] || pl->n_output() != lsize[i+1]) - return false; - } - else { - static UINT last_output; - if (i > 0 && pl->n_input() != last_output) return false; - last_output = pl->n_output(); - } - - add_top(*pl); delete pl; - } - if (v > 0) - if (n_in_got != _n_in || n_out_got != _n_out) return false; - - return true; -} - -void FeedForwardNN::add_top (const NNLayer& nl) { - assert(n_layer+1 == layer.size()); - assert(n_output() == nl.n_input() || n_layer == 0); - if (n_layer == 0) { - assert(_y.empty() && _dy.empty()); - _n_in = nl.n_input(); - _y.push_back(Output(_n_in)); - _dy.push_back(Output(_n_in)); - } - - n_layer++; - _n_out = nl.n_output(); - layer.push_back(nl.clone()); - _y.push_back(Output(nl.n_output())); - _dy.push_back(Output(nl.n_output())); -} - -void FeedForwardNN::initialize () { - for (UINT i = 1; i <= n_layer; ++i) - layer[i]->initialize(); -} - -void FeedForwardNN::train () { - assert(n_layer > 0); - assert(ptd != NULL && ptw != NULL); - - switch (train_method) { - case GRADIENT_DESCENT: - iterative_optimize(_gradient_descent - (this, learn_rate)); - break; - case LINE_SEARCH: - iterative_optimize(_line_search - (this, learn_rate)); - break; - case CONJUGATE_GRADIENT: - iterative_optimize - (_conjugate_gradient - (this, learn_rate)); - break; - case WEIGHT_DECAY: - iterative_optimize(_gd_weightdecay - (this, learn_rate, 0.01)); - break; - case ADAPTIVE_LEARNING_RATE: - iterative_optimize(_gd_adaptive - (this, learn_rate, 1.15, 0.5)); - break; - default: - assert(0); - } -} - -void FeedForwardNN::log_cost (UINT epoch, REAL cst) { - if (logf != NULL) - fprintf(logf, "%lu %g %g\n", epoch, learn_rate, cst); - - if (epoch % 20 == 1) - printf("epoch %lu, cost = %g\n", epoch, cst); -} - -Output FeedForwardNN::operator() (const Input& x) const { - assert(n_layer > 0); - assert(x.size() == n_input()); - - forward(x); - return _y[n_layer]; -} - -FeedForwardNN::WEIGHT FeedForwardNN::weight () const { - WEIGHT wgt; - for (UINT i = 1; i <= n_layer; ++i) - wgt.push_back(layer[i]->weight()); - return wgt; -} - -void FeedForwardNN::set_weight (const WEIGHT& wgt) { - assert(wgt.size() == n_layer); - for (UINT i = 1; i <= n_layer; ++i) - layer[i]->set_weight(wgt[i-1]); -} - -Output FeedForwardNN::_cost_deriv (const Output& F, const Output& y) const { - assert(F.size() == n_output() && y.size() == n_output()); - - Output d(_n_out); - for (UINT i = 0; i < _n_out; ++i) - d[i] = F[i] - y[i]; - return d; -} - -REAL FeedForwardNN::cost (UINT idx) const { - return _cost(get_output(idx), ptd->y(idx)); -} - -REAL FeedForwardNN::cost () const { - assert(ptd != NULL && ptw != NULL); - const UINT n = ptd->size(); - REAL cst = 0; - for (UINT i = 0; i < n; ++i) - cst += cost(i) * (*ptw)[i]; - return cst; -} - -FeedForwardNN::WEIGHT FeedForwardNN::gradient (UINT idx) const { - assert(ptd != NULL); - assert(n_layer > 0); - - clear_gradient(); - - forward(_y[0] = ptd->x(idx)); - _dy[n_layer] = _cost_deriv(_y[n_layer], ptd->y(idx)); - for (UINT i = n_layer; i; --i) - layer[i]->back_propagate(_y[i-1], _dy[i], _dy[i-1]); - - WEIGHT grad; - for (UINT i = 1; i <= n_layer; ++i) - grad.push_back(layer[i]->gradient()); - return grad; -} - -FeedForwardNN::WEIGHT FeedForwardNN::gradient () const { - assert(ptd != NULL && ptw != NULL); - assert(n_layer > 0); - - clear_gradient(); - - const UINT n = ptd->size(); - for (UINT idx = 0; idx < n; idx++) { - forward(_y[0] = ptd->x(idx)); - - _dy[n_layer] = _cost_deriv(_y[n_layer], ptd->y(idx)); - assert(_dy[n_layer].size() == _n_out); - const REAL w = (*ptw)[idx] * n; - for (UINT j = 0; j < _n_out; ++j) - _dy[n_layer][j] *= w; - - for (UINT i = n_layer; i; --i) - layer[i]->back_propagate(_y[i-1], _dy[i], _dy[i-1]); - } - - WEIGHT grad; - for (UINT i = 1; i <= n_layer; ++i) - grad.push_back(layer[i]->gradient()); - return grad; -} - -void FeedForwardNN::clear_gradient () const { - for (UINT i = 1; i <= n_layer; ++i) - layer[i]->clear_gradient(); -} - -bool FeedForwardNN::stop_opt (UINT step, REAL cst) { - log_cost(step, cst); - return (step >= max_run || cst < min_cst); -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.h deleted file mode 100644 index dfd212d..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/feedforwardnn.h +++ /dev/null @@ -1,108 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_FEEDFORWARDNN_H__ -#define __LEMGA_FEEDFORWARDNN_H__ - -/** @file - * @brief Feed-forward neural network. - * - * $Id: feedforwardnn.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "learnmodel.h" -#include "nnlayer.h" - -namespace lemga { - -/** @todo documentation */ -class FeedForwardNN : public LearnModel { - void free_space (); - void forward (const Input& x) const { - assert(n_layer > 0 && x.size() == n_input()); - layer[1]->feed_forward(x, _y[1]); - for (UINT i = 2; i <= n_layer; ++i) - layer[i]->feed_forward(_y[i-1], _y[i]); - } - -public: - typedef std::vector WEIGHT; - enum TRAIN_METHOD { - GRADIENT_DESCENT, - LINE_SEARCH, - CONJUGATE_GRADIENT, - WEIGHT_DECAY, - ADAPTIVE_LEARNING_RATE - }; - -protected: - UINT n_layer; ///< # of layers == layer.size()-1. - std::vector layer; ///< layer pointers (layer[0] == 0). - mutable std::vector _y; ///< buffer for outputs. - mutable std::vector _dy;///< buffer for derivatives. - - bool online_learn; - TRAIN_METHOD train_method; - REAL learn_rate, min_cst; - UINT max_run; - -public: - FeedForwardNN (); - FeedForwardNN (const FeedForwardNN&); - explicit FeedForwardNN (std::istream& is) { is >> *this; } - virtual ~FeedForwardNN (); - const FeedForwardNN& operator= (const FeedForwardNN&); - - virtual const id_t& id () const; - virtual FeedForwardNN* create () const { return new FeedForwardNN(); } - virtual FeedForwardNN* clone () const { - return new FeedForwardNN(*this); } - - UINT size () const { return n_layer; } - const NNLayer& operator[] (UINT n) const { return *layer[n+1]; } - void add_top (const NNLayer&); - void add_bottom (const NNLayer&); - - void set_batch_mode (bool b = true) { online_learn = !b; } - void set_train_method (TRAIN_METHOD m) { train_method = m; } - /** @param lr learning rate. - * @param mincst minimal cost (error) need to be achieved during - * training. - * @param maxrun maximal # of epochs the training should take. - */ - void set_parameter (REAL lr, REAL mincst, UINT maxrun) { - learn_rate = lr; min_cst = mincst; max_run = maxrun; } - - virtual bool support_weighted_data () const { return true; } - virtual void initialize (); - virtual void train (); - virtual Output operator() (const Input&) const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); - - virtual REAL _cost (const Output& F, const Output& y) const { - return r_error(F, y); } - virtual Output _cost_deriv (const Output& F, const Output& y) const; - virtual void log_cost (UINT epoch, REAL err); - -public: - WEIGHT weight () const; - void set_weight (const WEIGHT&); - - REAL cost (UINT idx) const; - REAL cost () const; - WEIGHT gradient (UINT idx) const; - WEIGHT gradient () const; - void clear_gradient () const; - - bool stop_opt (UINT step, REAL cst); -}; - -} // namespace lemga - -#ifdef __FEEDFORWARDNN_H__ -#warning "This header file may conflict with another `feedforwardnn.h' file." -#endif -#define __FEEDFORWARDNN_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.cpp deleted file mode 100644 index a8625a0..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/** @file - * Register kernel creators. - * $Id: kernel.cpp 2551 2006-01-17 04:31:27Z ling $ - */ - -#include "kernel.h" - -REGISTER_CREATOR2(lemga::kernel::Linear, k_lin); -REGISTER_CREATOR2(lemga::kernel::Polynomial, k_pol); -REGISTER_CREATOR2(lemga::kernel::Stump, k_stu); -REGISTER_CREATOR2(lemga::kernel::Perceptron, k_per); -REGISTER_CREATOR2(lemga::kernel::RBF, k_rbf); -REGISTER_CREATOR2(lemga::kernel::Sigmoid, k_sig); - -namespace lemga { -namespace kernel { - -bool Kernel::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Object, os, vl, 1); - return os; -} - -bool Kernel::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Object, is, vl, 1, v); - assert(v > 0); - return true; -} - -bool Polynomial::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Kernel, os, vl, 1); - return (os << degree << ' ' << gamma << ' ' << coef0 << '\n'); -} - -bool -Polynomial::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Kernel, is, vl, 1, v); - assert(v > 0); - return (is >> degree >> gamma >> coef0) && (gamma > 0); -} - -bool RBF::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Kernel, os, vl, 1); - return (os << gamma << '\n'); -} - -bool RBF::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Kernel, is, vl, 1, v); - assert(v > 0); - return (is >> gamma) && (gamma > 0); -} - -bool Sigmoid::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Kernel, os, vl, 1); - return (os << gamma << ' ' << coef0 << '\n'); -} - -bool Sigmoid::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Kernel, is, vl, 1, v); - assert(v > 0); - return (is >> gamma >> coef0) && (gamma > 0); -} - -}} // namespace lemga::kernel diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.h deleted file mode 100644 index 46824d8..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/kernel.h +++ /dev/null @@ -1,204 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_KERNEL_H__ -#define __LEMGA_KERNEL_H__ - -/** @file - * @brief Kernels for SVM etc. - * - * $Id: kernel.h 2611 2006-02-02 21:53:30Z ling $ - */ - -#include -#include -#include "learnmodel.h" - -#define DOTPROD(x,y) std::inner_product(x.begin(), x.end(), y.begin(), .0) - -namespace lemga { - -// only for LIBSVM, see Kernel::set_params. -struct SVM_detail; - -namespace kernel { - -inline REAL norm_1 (const Input& u, const Input& v) { - REAL sum(0); - Input::const_iterator x = u.begin(), y = v.begin(); - for (; x != u.end(); ++x, ++y) - sum += std::fabs(*x - *y); - return sum; -} - -inline REAL norm_2 (const Input& u, const Input& v) { - REAL sum(0); - Input::const_iterator x = u.begin(), y = v.begin(); - for (; x != u.end(); ++x, ++y) { - REAL d = *x - *y; - sum += d * d; - } - return sum; -} - -/// The operator() gives the inner-product in the transformed space. -class Kernel : public Object { -public: - virtual Kernel* create () const = 0; - virtual Kernel* clone () const = 0; - - /// The inner-product of two input vectors. - virtual REAL operator() (const Input&, const Input&) const = 0; - /// Store a dataset in order to compute the kernel matrix. - virtual void set_data (const pDataSet& pd) { ptd = pd; } - /// The inner-product of two stored inputs with index @a i and @a j. - virtual REAL matrix (UINT i, UINT j) const - { return operator()(ptd->x(i), ptd->x(j)); } - - /// In order to keep the SVM interface simple and avoid - /// member functions specific to kernels (e.g., set_gamma()), - /// we use Kernel to pass kernel parameters to SVM_detail. - virtual void set_params (SVM_detail*) const = 0; - -protected: - pDataSet ptd; - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -/// %Linear kernel @f$\left@f$. -struct Linear : public Kernel { - Linear () {} - explicit Linear (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Linear* create () const { return new Linear(); } - virtual Linear* clone () const { return new Linear(*this); } - - virtual REAL operator() (const Input& a, const Input& b) const { - return DOTPROD(a, b); - } - virtual void set_params (SVM_detail*) const; -}; - -/// %Polynomial kernel @f$(\gamma*\left+c_0)^d@f$. -struct Polynomial : public Kernel { - UINT degree; - REAL gamma, coef0; - - Polynomial (UINT d = 3, REAL g = 0.5, REAL c0 = 0) - : degree(d), gamma(g), coef0(c0) {}; - explicit Polynomial (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Polynomial* create () const { return new Polynomial(); } - virtual Polynomial* clone () const { return new Polynomial(*this); } - - virtual REAL operator() (const Input& a, const Input& b) const { - return std::pow(gamma * DOTPROD(a, b) + coef0, (double) degree); - } - virtual void set_params (SVM_detail*) const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -/// %Stump kernel @f$-\left|u-v\right|_1@f$. -struct Stump : public Kernel { - Stump () {} - explicit Stump (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Stump* create () const { return new Stump(); } - virtual Stump* clone () const { return new Stump(*this); } - - virtual REAL operator() (const Input& a, const Input& b) const { - return -norm_1(a, b); - } - virtual void set_params (SVM_detail*) const; -}; - -/// %Perceptron kernel @f$-\left|u-v\right|_2@f$. -struct Perceptron : public Kernel { -protected: - std::vector x_norm2; ///< cached inner-product of data input - -public: - Perceptron () {} - explicit Perceptron (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Perceptron* create () const { return new Perceptron(); } - virtual Perceptron* clone () const { return new Perceptron(*this); } - - virtual REAL operator() (const Input& a, const Input& b) const { - return -std::sqrt(norm_2(a, b)); - } - - virtual void set_data (const pDataSet& pd) { - Kernel::set_data(pd); - const UINT n = ptd->size(); - x_norm2.resize(n); - for (UINT i = 0; i < n; ++i) - x_norm2[i] = DOTPROD(ptd->x(i), ptd->x(i)); - } - virtual REAL matrix (UINT i, UINT j) const { - REAL n2 = x_norm2[i] + x_norm2[j] - 2*DOTPROD(ptd->x(i), ptd->x(j)); - return (n2 > 0)? -std::sqrt(n2) : 0; // avoid -0.0 - } - - virtual void set_params (SVM_detail*) const; -}; - -/// %RBF (Gausssian) kernel @f$e^{-\gamma\left|u-v\right|_2^2}@f$. -struct RBF : public Perceptron { - REAL gamma; - explicit RBF (REAL g = 0.5) : gamma(g) {} - explicit RBF (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual RBF* create () const { return new RBF(); } - virtual RBF* clone () const { return new RBF(*this); } - - virtual REAL operator() (const Input& a, const Input& b) const { - return std::exp(-gamma * norm_2(a, b)); - } - - virtual REAL matrix (UINT i, UINT j) const { - REAL n2 = x_norm2[i] + x_norm2[j] - 2*DOTPROD(ptd->x(i), ptd->x(j)); - return std::exp(-gamma * n2); - } - - virtual void set_params (SVM_detail*) const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -/// %Sigmoid kernel @f$\tanh(\gamma\left+c_0)@f$. -struct Sigmoid : public Kernel { - REAL gamma, coef0; - Sigmoid (REAL g = 0.5, REAL c0 = 0) : gamma(g), coef0(c0) {}; - explicit Sigmoid (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Sigmoid* create () const { return new Sigmoid(); } - virtual Sigmoid* clone () const { return new Sigmoid(*this); } - - virtual REAL operator() (const Input& a, const Input& b) const { - return std::tanh(gamma * DOTPROD(a, b) + coef0); - } - virtual void set_params (SVM_detail*) const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -}} // namespace lemga::kernel - -#ifdef __KERNEL_H__ -#warning "This header file may conflict with another `kernel.h' file." -#endif -#define __KERNEL_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.cpp deleted file mode 100644 index 271ec57..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.cpp +++ /dev/null @@ -1,225 +0,0 @@ -/** @file - * $Id: learnmodel.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include -#include -#include -#include "learnmodel.h" - -namespace lemga { - -/** A local helper for load_data */ -static DataSet* -load_data (DataSet* pd, std::istream& is, UINT n, UINT in, UINT out) { - for (UINT i = 0; i < n; ++i) { - Input x(in); - Output y(out); - for (UINT j = 0; j < in; ++j) - if (!(is >> x[j])) return pd; - for (UINT j = 0; j < out; ++j) - if (!(is >> y[j])) return pd; - - pd->append(x, y); - } - return pd; -} - -/** Each sample consists of first the input and then the output. - * Numbers are separated by spaces. - * @param is the input stream - * @param n gives the number of samples - * @param in is the dimension of input - * @param out is the dimension of output - * @todo documentation: why separate function - */ -DataSet* load_data (std::istream& is, UINT n, UINT in, UINT out) { - DataSet* pd = new DataSet(); - return load_data(pd, is, n, in, out); -} - -/** An easier-to-use version, where the output dimension is fixed - * at 1, and the input dimension is auto-detected. This version - * requires that each row of stream @a is should be a sample. - */ -DataSet* load_data (std::istream& is, UINT n) { - assert(n > 0); - /* read the first line and infer the input dimension */ - Input x; - do { - char line[1024*10]; - is.getline(line, 1024*10); - std::istringstream iss(line); - REAL xi; - while (iss >> xi) - x.push_back(xi); - } while (x.empty() && !is.eof()); - if (x.empty()) return 0; - - Output y(1, x.back()); - x.pop_back(); - - DataSet* pd = new DataSet(); - pd->append(x, y); - return load_data(pd, is, n-1, x.size(), 1); -} - -/** @param n_in is the dimension of input. - * @param n_out is the dimension of output. */ -LearnModel::LearnModel (UINT n_in, UINT n_out) - : Object(), _n_in(n_in), _n_out(n_out), n_samples(0), logf(NULL) -{ /* empty */ } - -bool LearnModel::serialize (std::ostream& os, - ver_list& vl) const { - SERIALIZE_PARENT(Object, os, vl, 1); - return (os << _n_in << ' ' << _n_out << '\n'); -} - -bool LearnModel::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - assert(d == NIL_ID); - UNSERIALIZE_PARENT(Object, is, vl, 1, v); - _n_in = 0; _n_out = 0; - ptd = 0; ptw = 0; n_samples = 0; - if (v == 0) return true; - return (is >> _n_in >> _n_out); -} - -/** @param out is the output from the learned hypothesis. - * @param y is the real output. - * @return Regression error between @a out and @a y. - * A commonly used measure is the squared error. - */ -REAL LearnModel::r_error (const Output& out, const Output& y) const { - assert(out.size() == n_output()); - assert(y.size() == n_output()); - - REAL err = 0; - for (UINT i = 0; i < _n_out; ++i) { - REAL dif = out[i] - y[i]; - err += dif * dif; - } - return err / 2; -} - -/** @param out is the output from the learned hypothesis. - * @param y is the real output. - * @return Classification error between @a out and @a y. - * The error measure is not necessary symmetric. A commonly used - * measure is @a out != @a y. - */ -REAL LearnModel::c_error (const Output& out, const Output& y) const { - assert(n_output() == 1); - assert(std::fabs(std::fabs(y[0]) - 1) < INFINITESIMAL); - return (out[0]*y[0] <= 0); -} - -REAL LearnModel::train_r_error () const { - assert(ptw != 0); - REAL err = 0; - for (UINT i = 0; i < n_samples; ++i) - err += (*ptw)[i] * r_error(get_output(i), ptd->y(i)); - return err; -} - -REAL LearnModel::train_c_error () const { - assert(ptw != 0); - REAL err = 0; - for (UINT i = 0; i < n_samples; ++i) - err += (*ptw)[i] * c_error(get_output(i), ptd->y(i)); - return err; -} - -REAL LearnModel::test_r_error (const pDataSet& pd) const { - UINT n = pd->size(); - REAL err = 0; - for (UINT i = 0; i < n; ++i) - err += r_error((*this)(pd->x(i)), pd->y(i)); - return err / n; -} - -REAL LearnModel::test_c_error (const pDataSet& pd) const { - UINT n = pd->size(); - REAL err = 0; - for (UINT i = 0; i < n; ++i) - err += c_error((*this)(pd->x(i)), pd->y(i)); - return err / n; -} - -/** If the learning model/algorithm can only do training using uniform - * sample weight, i.e., support_weighted_data() returns @c false, a - * ``boostrapped'' copy of the original data set will be generated and - * used in the following training. The boostrapping is done by randomly - * pick samples (with replacement) w.r.t. the given weight @a pw. - * - * In order to make the life easier, when support_weighted_data() returns - * @c true, a null @a pw will be replaced by a uniformly distributed - * probability vector. So we have the following invariant - * @invariant support_weighted_data() == (@a ptw != 0) - * - * @param pd gives the data set. - * @param pw gives the sample weight, whose default value is 0. - * @sa support_weighted_data(), train() - */ -void LearnModel::set_train_data (const pDataSet& pd, const pDataWgt& pw) { - n_samples = pd->size(); - assert(n_samples > 0); - assert(!pw || n_samples == pw->size()); - if (support_weighted_data()) { - ptd = pd; - ptw = (pw != 0)? pw : new DataWgt(n_samples, 1.0 / n_samples); - } else { - ptd = (!pw)? pd : pd->random_sample(*pw, n_samples); - ptw = 0; - } - assert(support_weighted_data() == (ptw != 0)); -#ifndef NDEBUG - // assert: ptw is a probability vector - if (ptw != 0) { - REAL wsum = 0; - for (UINT i = 0; i < n_samples; i++) { - assert((*ptw)[i] >= 0); - wsum += (*ptw)[i]; - } - assert(wsum-1 > -EPSILON && wsum-1 < EPSILON); - } -#endif - if (!exact_dimensions(*pd)) { - std::cerr << id() << "::set_train_data: Error: " - "Wrong input/output dimensions.\n"; - exit(-1); - } -} - -void LearnModel::reset () { - _n_in = _n_out = 0; -} - -REAL LearnModel::margin_of (const Input&, const Output&) const { - OBJ_FUNC_UNDEFINED("margin_of"); -} - -REAL LearnModel::min_margin () const { - REAL min_m = INFINITY; - for (UINT i = 0; i < n_samples; ++i) { - // assume all examples count (in computing the minimum) - assert((*ptw)[i] > INFINITESIMAL); - REAL m = margin(i); - if (min_m > m) min_m = m; - } - return min_m; -} - -bool LearnModel::valid_dimensions (UINT nin, UINT nout) const { - return (nin == 0 || _n_in == 0 || nin == _n_in) && - (nout == 0 || _n_out == 0 || nout == _n_out); -} - -void LearnModel::set_dimensions (UINT nin, UINT nout) { - assert(valid_dimensions(nin, nout)); - if (nin > 0) _n_in = nin; - if (nout > 0) _n_out = nout; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.h deleted file mode 100644 index 433f4f3..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/learnmodel.h +++ /dev/null @@ -1,204 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_LEARNMODEL_H__ -#define __LEMGA_LEARNMODEL_H__ - -/** @file - * @brief Type definitions, @link lemga::DataSet DataSet@endlink, and - * @link lemga::LearnModel LearnModel@endlink class. - * @todo: add input_dim check to dataset and learnmodel - * - * $Id: learnmodel.h 2779 2006-05-09 06:49:33Z ling $ - */ - -#include -#include -#include "object.h" -#include "dataset.h" -#include "shared_ptr.h" - -#define VERBOSE_OUTPUT 1 - -namespace lemga { - -typedef std::vector Input; -typedef std::vector Output; - -typedef dataset DataSet; -typedef std::vector DataWgt; -typedef const_shared_ptr pDataSet; -typedef const_shared_ptr pDataWgt; - -/// Load a data set from a stream -DataSet* load_data (std::istream&, UINT, UINT, UINT); -DataSet* load_data (std::istream&, UINT); - -/** @brief A unified interface for learning models. - * - * I try to provide - * + r_error and c_error - * for regression problems, r_error should be defined; - * for classification problems, c_error should be defined; - * these two errors can both be present - * - * The training data is stored with the learning model (as a pointer) - * Say: why (the benefit of store with, a pointer); maybe not a pointer - * Say: what's the impact of doing this (what will be changed - * from normal implementation) - * Say: wgt: could be null if the model doesn't support ...otherwise shoud - * be a probability vector (randome_sample)... - * - * @anchor learnmodel_training_order - * The flowchart of the learning ... - * -# Create a new instance, load from a file, and/or reset an existing - * one lm->reset();. - * -# lm->set_train_data(sample_data);\n - * Specify the training data - * -# err = lm->train();\n - * Usually, the return value has no meaning - * -# y = (*lm)(x);\n - * Apply the learning model to new data. - * - * @todo documentation - * @todo Do we really need two errors? - */ -class LearnModel : public Object { -protected: - UINT _n_in; ///< input dimension of the model - UINT _n_out; ///< output dimension of the model - pDataSet ptd; ///< pointer to the training data set - pDataWgt ptw; ///< pointer to the sample weight (for training) - UINT n_samples; ///< equal to @c ptd->size() - - FILE* logf; ///< file to record train/validate error - -public: - LearnModel (UINT n_in = 0, UINT n_out = 0); - - //@{ @name Basic - virtual LearnModel* create () const = 0; - virtual LearnModel* clone () const = 0; - - UINT n_input () const { return _n_in; } - UINT n_output () const { return _n_out; } - - void set_log_file (FILE* f) { logf = f; } - //@} - - //@{ @name Training related - /** @brief Whether the learning model/algorithm supports unequally - * weighted data. - * @return @c true if supporting; @c false otherwise. The default - * is @c false, just for safety. - * @sa set_train_data() - */ - virtual bool support_weighted_data () const { return false; } - - /// Error measure for regression problems - virtual REAL r_error (const Output& out, const Output& y) const; - /// Error measure for classification problems - virtual REAL c_error (const Output& out, const Output& y) const; - - /// Training error (regression) - REAL train_r_error () const; - /// Training error (classification) - REAL train_c_error () const; - /// Test error (regression) - REAL test_r_error (const pDataSet&) const; - /// Test error (classification) - REAL test_c_error (const pDataSet&) const; - - virtual void initialize () { - std::cerr << "!!! initialize() is depreciated.\n" - << "!!! See the documentation of LearnModel and reset().\n"; - } - - /// Set the data set and sample weight to be used in training - virtual void set_train_data (const pDataSet&, const pDataWgt& = 0); - /// Return pointer to the embedded training data set - const pDataSet& train_data () const { return ptd; } - /* temporarily disabled; ptw in boosting base learners is reset - * after the training; disabled to be sure no one actually uses it - const pDataWgt& data_weight () const { return ptw; } - */ - - /** @brief Train with preset data set and sample weight. - */ - virtual void train () = 0; - - /// Cleaning up the learning model but keeping most settings. - /// @note This is probably needed after training or loading from file, - /// but before having another training. - virtual void reset (); - //@} - - virtual Output operator() (const Input&) const = 0; - - /** @brief Get the output of the hypothesis on the @a idx-th input. - * @note It is possible to cache results to save computational effort. - */ - virtual Output get_output (UINT idx) const { - assert(ptw != 0); // no data sampling - return operator()(ptd->x(idx)); } - - //@{ @name Margin related - /** @brief The normalization term for margins. - * - * The margin concept can be normalized or unnormalized. For example, - * for a perceptron model, the unnormalized margin would be the wegithed - * sum of the input features, and the normalized margin would be the - * distance to the hyperplane, and the normalization term is the norm - * of the hyperplane weight. - * - * Since the normalization term is usually a constant, it would be - * more efficient if it is precomputed instead of being calculated - * every time when a margin is asked for. The best way is to use a - * cache. Here I use a easier way: let the users decide when to - * compute the normalization term. - */ - virtual REAL margin_norm () const { return 1; } - /// Report the (unnormalized) margin of an example (@a x, @a y). - virtual REAL margin_of (const Input& x, const Output& y) const; - /** @brief Report the (unnormalized) margin of the example @a i. - * @note It is possible to cache results to save computational effort. - */ - virtual REAL margin (UINT i) const { - assert(ptw != 0); // no data sampling - return margin_of(ptd->x(i), ptd->y(i)); } - /// The minimal (unnormalized) in-sample margin. - REAL min_margin () const; - //@} - - bool valid_dimensions (UINT, UINT) const; - inline bool valid_dimensions (const LearnModel& l) const { - return valid_dimensions(l.n_input(), l.n_output()); } - - inline bool exact_dimensions (UINT i, UINT o) const { - return (i > 0 && o > 0 && valid_dimensions(i, o)); } - inline bool exact_dimensions (const LearnModel& l) const { - return exact_dimensions(l.n_input(), l.n_output()); } - inline bool exact_dimensions (const DataSet& d) const { - assert(d.size() > 0); - return exact_dimensions(d.x(0).size(), d.y(0).size()); } - -protected: - void set_dimensions (UINT, UINT); - inline void set_dimensions (const LearnModel& l) { - set_dimensions(l.n_input(), l.n_output()); } - inline void set_dimensions (const DataSet& d) { - assert(exact_dimensions(d)); - set_dimensions(d.x(0).size(), d.y(0).size()); } - - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -typedef var_shared_ptr pLearnModel; -typedef const_shared_ptr pcLearnModel; - -} // namespace lemga - -#ifdef __LEARNMODEL_H__ -#warning "This header file may conflict with another `learnmodel.h' file." -#endif -#define __LEARNMODEL_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-2.81-weight.patch b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-2.81-weight.patch deleted file mode 100644 index 9c83585..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-2.81-weight.patch +++ /dev/null @@ -1,356 +0,0 @@ -diff -ru libsvm-2.81.orig/svm.cpp libsvm-2.81/svm.cpp ---- libsvm-2.81.orig/svm.cpp 2005-10-10 20:17:45.000000000 -0700 -+++ libsvm-2.81/svm.cpp 2005-12-13 22:35:53.000000000 -0800 -@@ -376,6 +376,11 @@ - double r; // for Solver_NU - }; - -+ /** the solver with different C_i */ -+ void Solve(int l, const QMatrix& Q, const double *b_, const schar *y_, -+ double *alpha_, const double *C_, double eps, -+ SolutionInfo* si, int shrinking); -+ - void Solve(int l, const QMatrix& Q, const double *b_, const schar *y_, - double *alpha_, double Cp, double Cn, double eps, - SolutionInfo* si, int shrinking); -@@ -383,13 +388,21 @@ - int active_size; - schar *y; - double *G; // gradient of objective function -+ -+ /** the enum is removed to allow examples to be both at -+ the upper bound and at the lower bound (if C = 0) - enum { LOWER_BOUND, UPPER_BOUND, FREE }; -+ */ -+ static const char FREE = 0; -+ static const char LOWER_BOUND = 1; // bit 1 -+ static const char UPPER_BOUND = 2; // bit 2 -+ - char *alpha_status; // LOWER_BOUND, UPPER_BOUND, FREE - double *alpha; - const QMatrix *Q; - const Qfloat *QD; - double eps; -- double Cp,Cn; -+ double *C; // the weighted C_i - double *b; - int *active_set; - double *G_bar; // gradient, if we treat free variables as 0 -@@ -398,18 +411,19 @@ - - double get_C(int i) - { -- return (y[i] > 0)? Cp : Cn; -+ return C[i]; - } -+ /** the following modification is for maintaining the alpha_status */ - void update_alpha_status(int i) - { -- if(alpha[i] >= get_C(i)) -- alpha_status[i] = UPPER_BOUND; -- else if(alpha[i] <= 0) -- alpha_status[i] = LOWER_BOUND; -- else alpha_status[i] = FREE; -+ alpha_status[i] = FREE; -+ if (alpha[i] <= 0) -+ alpha_status[i] |= LOWER_BOUND; -+ if (alpha[i] >= get_C(i)) -+ alpha_status[i] |= UPPER_BOUND; - } -- bool is_upper_bound(int i) { return alpha_status[i] == UPPER_BOUND; } -- bool is_lower_bound(int i) { return alpha_status[i] == LOWER_BOUND; } -+ bool is_upper_bound(int i) { return alpha_status[i] & UPPER_BOUND; } -+ bool is_lower_bound(int i) { return alpha_status[i] & LOWER_BOUND; } - bool is_free(int i) { return alpha_status[i] == FREE; } - void swap_index(int i, int j); - void reconstruct_gradient(); -@@ -429,6 +443,7 @@ - swap(b[i],b[j]); - swap(active_set[i],active_set[j]); - swap(G_bar[i],G_bar[j]); -+ swap(C[i], C[j]); - } - - void Solver::reconstruct_gradient() -@@ -452,7 +467,7 @@ - } - - void Solver::Solve(int l, const QMatrix& Q, const double *b_, const schar *y_, -- double *alpha_, double Cp, double Cn, double eps, -+ double *alpha_, const double* C_, double eps, - SolutionInfo* si, int shrinking) - { - this->l = l; -@@ -461,8 +476,7 @@ - clone(b, b_,l); - clone(y, y_,l); - clone(alpha,alpha_,l); -- this->Cp = Cp; -- this->Cn = Cn; -+ clone(C, C_, l); - this->eps = eps; - unshrinked = false; - -@@ -705,13 +719,11 @@ - // or Q.swap_index(i,active_set[i]); - }*/ - -- si->upper_bound_p = Cp; -- si->upper_bound_n = Cn; -- - info("\noptimization finished, #iter = %d\n",iter); - - delete[] b; - delete[] y; -+ delete[] C; - delete[] alpha; - delete[] alpha_status; - delete[] active_set; -@@ -719,6 +731,19 @@ - delete[] G_bar; - } - -+void Solver::Solve(int l, const QMatrix& Q, const double *b_, const schar *y_, -+ double *alpha_, double Cp, double Cn, double eps, -+ SolutionInfo* si, int shrinking) -+{ -+ double* C_ = new double[l]; -+ for (int i = 0; i < l; ++i) -+ C_[i] = (y_[i] > 0 ? Cp : Cn); -+ Solve(l, Q, b_, y_, alpha_, C_, eps, si, shrinking); -+ si->upper_bound_p = Cp; -+ si->upper_bound_n = Cn; -+ delete[] C_; -+} -+ - // return 1 if already optimal, return 0 otherwise - int Solver::select_working_set(int &out_i, int &out_j) - { -@@ -1444,7 +1469,7 @@ - // - static void solve_c_svc( - const svm_problem *prob, const svm_parameter* param, -- double *alpha, Solver::SolutionInfo* si, double Cp, double Cn) -+ double *alpha, Solver::SolutionInfo* si, const double *C_) - { - int l = prob->l; - double *minus_ones = new double[l]; -@@ -1461,14 +1486,7 @@ - - Solver s; - s.Solve(l, SVC_Q(*prob,*param,y), minus_ones, y, -- alpha, Cp, Cn, param->eps, si, param->shrinking); -- -- double sum_alpha=0; -- for(i=0;il)); -+ alpha, C_, param->eps, si, param->shrinking); - - for(i=0;il; - double *alpha2 = new double[2*l]; - double *linear_term = new double[2*l]; -+ double *C2 = new double[2*l]; - schar *y = new schar[2*l]; - int i; - -@@ -1579,26 +1598,24 @@ - alpha2[i] = 0; - linear_term[i] = param->p - prob->y[i]; - y[i] = 1; -+ C2[i] = C_[i]; - - alpha2[i+l] = 0; - linear_term[i+l] = param->p + prob->y[i]; - y[i+l] = -1; -+ C2[i+l] = C_[i]; - } - - Solver s; - s.Solve(2*l, SVR_Q(*prob,*param), linear_term, y, -- alpha2, param->C, param->C, param->eps, si, param->shrinking); -+ alpha2, C2, param->eps, si, param->shrinking); - -- double sum_alpha = 0; - for(i=0;iC*l)); - - delete[] alpha2; - delete[] linear_term; -+ delete[] C2; - delete[] y; - } - -@@ -1655,21 +1672,39 @@ - { - double *alpha = Malloc(double,prob->l); - Solver::SolutionInfo si; -+ double *C_ = NULL; -+ -+ if (prob->W){ -+ C_ = new double[prob->l]; -+ for(int i=0;il;i++) C_[i] = prob->W[i] * param->C; -+ } -+ else if (param->svm_type == C_SVC){ -+ C_ = new double[prob->l]; -+ for(int i=0;il;i++) C_[i] = (prob->y[i] > 0 ? Cp : Cn); -+ } -+ else if (param->svm_type == EPSILON_SVR){ -+ C_ = new double[prob->l]; -+ for(int i=0;il;i++) C_[i] = param->C; -+ } -+ - switch(param->svm_type) - { - case C_SVC: -- solve_c_svc(prob,param,alpha,&si,Cp,Cn); -+ solve_c_svc(prob,param,alpha,&si,C_); - break; - case NU_SVC: -+ info("Warning!! weighted mode not supported for NU_SVC"); - solve_nu_svc(prob,param,alpha,&si); - break; - case ONE_CLASS: -+ info("Warning!! weighted mode not supported for ONE_CLASS"); - solve_one_class(prob,param,alpha,&si); - break; - case EPSILON_SVR: -- solve_epsilon_svr(prob,param,alpha,&si); -+ solve_epsilon_svr(prob,param,alpha,&si,C_); - break; - case NU_SVR: -+ info("Warning!! weighted mode not supported for NU_SVR"); - solve_nu_svr(prob,param,alpha,&si); - break; - } -@@ -1685,7 +1720,11 @@ - if(fabs(alpha[i]) > 0) - { - ++nSV; -- if(prob->y[i] > 0) -+ if(C_){ -+ if(fabs(alpha[i]) >= C_[i]) -+ ++nBSV; -+ } -+ else if(prob->y[i] > 0) - { - if(fabs(alpha[i]) >= si.upper_bound_p) - ++nBSV; -@@ -1698,6 +1737,7 @@ - } - } - -+ if (C_) delete[] C_; - info("nSV = %d, nBSV = %d\n",nSV,nBSV); - - decision_function f; -@@ -2159,9 +2199,14 @@ - // group training data of the same class - svm_group_classes(prob,&nr_class,&label,&start,&count,perm); - svm_node **x = Malloc(svm_node *,l); -+ double *W = NULL; -+ if (prob->W) W = Malloc(double, l); -+ - int i; -- for(i=0;ix[perm[i]]; -+ if (prob->W) W[i] = prob->W[perm[i]]; -+ } - - // calculate weighted C - -@@ -2204,16 +2249,20 @@ - sub_prob.l = ci+cj; - sub_prob.x = Malloc(svm_node *,sub_prob.l); - sub_prob.y = Malloc(double,sub_prob.l); -+ if (W) sub_prob.W = Malloc(double, sub_prob.l); -+ else sub_prob.W = NULL; - int k; - for(k=0;kprobability) -@@ -2228,6 +2277,7 @@ - nonzero[sj+k] = true; - free(sub_prob.x); - free(sub_prob.y); -+ if (sub_prob.W) free(sub_prob.W); - ++p; - } - -@@ -2323,6 +2373,7 @@ - free(count); - free(perm); - free(start); -+ if (W) free(W); - free(x); - free(weighted_C); - free(nonzero); -@@ -2417,18 +2468,21 @@ - subprob.l = l-(end-begin); - subprob.x = Malloc(struct svm_node*,subprob.l); - subprob.y = Malloc(double,subprob.l); -- -+ if (prob->W) subprob.W = Malloc(double,subprob.l); -+ else subprob.W = NULL; - k=0; - for(j=0;jx[perm[j]]; - subprob.y[k] = prob->y[perm[j]]; -+ if (subprob.W) subprob.W[k] = prob->W[perm[j]]; - ++k; - } - for(j=end;jx[perm[j]]; - subprob.y[k] = prob->y[perm[j]]; -+ if (subprob.W) subprob.W[k] = prob->W[perm[j]]; - ++k; - } - struct svm_model *submodel = svm_train(&subprob,param); -@@ -2446,6 +2500,7 @@ - svm_destroy_model(submodel); - free(subprob.x); - free(subprob.y); -+ if (subprob.W) free(subprob.W); - } - free(fold_start); - free(perm); -diff -ru libsvm-2.81.orig/svm.h libsvm-2.81/svm.h ---- libsvm-2.81.orig/svm.h 2004-03-05 23:06:42.000000000 -0800 -+++ libsvm-2.81/svm.h 2005-12-13 22:40:27.000000000 -0800 -@@ -16,6 +16,7 @@ - int l; - double *y; - struct svm_node **x; -+ double *W; - }; - - enum { C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR }; /* svm_type */ diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-infkernels.patch b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-infkernels.patch deleted file mode 100644 index fc2cc6a..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-infkernels.patch +++ /dev/null @@ -1,211 +0,0 @@ -diff -ru libsvm-2.6.orig/svm.cpp libsvm-2.6/svm.cpp ---- libsvm-2.6.orig/svm.cpp 2004-03-30 18:34:04.000000000 -0800 -+++ libsvm-2.6/svm.cpp 2005-11-13 01:17:25.000000000 -0800 -@@ -199,6 +199,8 @@ - const double coef0; - - static double dot(const svm_node *px, const svm_node *py); -+ static double norm_1(const svm_node *, const svm_node *); -+ static double norm_2(const svm_node *, const svm_node *); - double kernel_linear(int i, int j) const - { - return dot(x[i],x[j]); -@@ -215,6 +217,15 @@ - { - return tanh(gamma*dot(x[i],x[j])+coef0); - } -+ double kernel_stump(int i, int j) const -+ { -+ return -norm_1(x[i],x[j]); -+ } -+ double kernel_perceptron(int i, int j) const -+ { -+ double n2 = x_square[i]+x_square[j]-2*dot(x[i],x[j]); -+ return (n2>0)? -sqrt(n2) : 0; -+ } - }; - - Kernel::Kernel(int l, svm_node * const * x_, const svm_parameter& param) -@@ -235,11 +246,17 @@ - case SIGMOID: - kernel_function = &Kernel::kernel_sigmoid; - break; -+ case STUMP: -+ kernel_function = &Kernel::kernel_stump; -+ break; -+ case PERCEPTRON: -+ kernel_function = &Kernel::kernel_perceptron; -+ break; - } - - clone(x,x_,l); - -- if(kernel_type == RBF) -+ if(kernel_type == RBF || kernel_type == PERCEPTRON) - { - x_square = new double[l]; - for(int i=0;iindex != -1 && y->index != -1) -+ { -+ if (x->index == y->index) -+ { -+ sum += fabs(x->value - y->value); -+ ++x; -+ ++y; -+ } -+ else if (x->index > y->index) -+ { -+ sum += fabs(y->value); -+ ++y; -+ } -+ else -+ { -+ sum += fabs(x->value); -+ ++x; -+ } -+ } -+ while (x->index != -1) -+ { -+ sum += fabs(x->value); -+ ++x; -+ } -+ while (y->index != -1) -+ { -+ sum += fabs(y->value); -+ ++y; -+ } -+ return sum; -+} -+ -+double Kernel::norm_2(const svm_node *x, const svm_node *y) -+{ -+ double sum = 0; -+ while (x->index != -1 && y->index !=-1) -+ { -+ if (x->index == y->index) -+ { -+ double d = x->value - y->value; -+ sum += d*d; -+ ++x; -+ ++y; -+ } -+ else if (x->index > y->index) -+ { -+ sum += y->value * y->value; -+ ++y; -+ } -+ else -+ { -+ sum += x->value * x->value; -+ ++x; -+ } -+ } -+ while (x->index != -1) -+ { -+ sum += x->value * x->value; -+ ++x; -+ } -+ while (y->index != -1) -+ { -+ sum += y->value * y->value; -+ ++y; -+ } -+ return sum; -+} -+ - double Kernel::k_function(const svm_node *x, const svm_node *y, - const svm_parameter& param) - { -@@ -287,48 +375,13 @@ - case POLY: - return pow(param.gamma*dot(x,y)+param.coef0,param.degree); - case RBF: -- { -- double sum = 0; -- while(x->index != -1 && y->index !=-1) -- { -- if(x->index == y->index) -- { -- double d = x->value - y->value; -- sum += d*d; -- ++x; -- ++y; -- } -- else -- { -- if(x->index > y->index) -- { -- sum += y->value * y->value; -- ++y; -- } -- else -- { -- sum += x->value * x->value; -- ++x; -- } -- } -- } -- -- while(x->index != -1) -- { -- sum += x->value * x->value; -- ++x; -- } -- -- while(y->index != -1) -- { -- sum += y->value * y->value; -- ++y; -- } -- -- return exp(-param.gamma*sum); -- } -+ return exp(-param.gamma*norm_2(x,y)); - case SIGMOID: - return tanh(param.gamma*dot(x,y)+param.coef0); -+ case STUMP: -+ return -norm_1(x,y); -+ case PERCEPTRON: -+ return -sqrt(norm_2(x,y)); - default: - return 0; /* Unreachable */ - } -@@ -2350,7 +2403,7 @@ - - const char *kernel_type_table[]= - { -- "linear","polynomial","rbf","sigmoid",NULL -+ "linear","polynomial","rbf","sigmoid","stump","perceptron",NULL - }; - - int svm_save_model(const char *model_file_name, const svm_model *model) -@@ -2666,7 +2719,9 @@ - if(kernel_type != LINEAR && - kernel_type != POLY && - kernel_type != RBF && -- kernel_type != SIGMOID) -+ kernel_type != SIGMOID && -+ kernel_type != STUMP && -+ kernel_type != PERCEPTRON) - return "unknown kernel type"; - - // cache_size,eps,C,nu,p,shrinking -diff -ru libsvm-2.6.orig/svm.h libsvm-2.6/svm.h ---- libsvm-2.6.orig/svm.h 2004-03-05 23:07:17.000000000 -0800 -+++ libsvm-2.6/svm.h 2005-11-13 01:17:25.000000000 -0800 -@@ -19,7 +19,7 @@ - }; - - enum { C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR }; /* svm_type */ --enum { LINEAR, POLY, RBF, SIGMOID }; /* kernel_type */ -+enum { LINEAR, POLY, RBF, SIGMOID, STUMP, PERCEPTRON }; /* kernel_type */ - - struct svm_parameter - { diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-kernel.patch b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-kernel.patch deleted file mode 100644 index 917ffa3..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/libsvm-kernel.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff -ru libsvm-2.6.orig/svm.cpp libsvm-2.6/svm.cpp ---- libsvm-2.6.orig/svm.cpp 2004-03-30 18:34:04.000000000 -0800 -+++ libsvm-2.6/svm.cpp 2005-05-30 14:53:30.557366028 -0700 -@@ -277,6 +277,12 @@ - return sum; - } - -+double svm_kernel(const svm_node *x, const svm_node *y, -+ const svm_parameter& param) -+{ -+ return Kernel::k_function(x, y, param); -+} -+ - double Kernel::k_function(const svm_node *x, const svm_node *y, - const svm_parameter& param) - { -diff -ru libsvm-2.6.orig/svm.h libsvm-2.6/svm.h ---- libsvm-2.6.orig/svm.h 2004-03-05 23:07:17.000000000 -0800 -+++ libsvm-2.6/svm.h 2005-05-30 14:53:30.558365983 -0700 -@@ -63,6 +63,8 @@ - const char *svm_check_parameter(const struct svm_problem *prob, const struct svm_parameter *param); - int svm_check_probability_model(const struct svm_model *model); - -+double svm_kernel(const svm_node *, const svm_node *, const svm_parameter&); -+ - #ifdef __cplusplus - } - #endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.cpp deleted file mode 100644 index f3dca0d..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/** @file - * $Id: lpboost.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include -#include -#include "lpboost.h" -extern "C"{ -#include -} - -REGISTER_CREATOR(lemga::LPBoost); - -namespace lemga { - -#define U(i) ((i)+1) //U(0) to U(n_samples-1) -#define R(t) ((t)+1) //R(0) to R(T-1) - -// Compute the weighted training error, and -// Add one more constraint R(t) = -sum u_i y_i h_t(x_i) >= -1 -bool lp_add_hypothesis (LPX* lp, int* ndx, double* val, const LearnModel& lm, - const pDataWgt& pdw = 0, REAL maxe = 0) -{ - const pDataSet ptd = lm.train_data(); - assert(ptd != 0); - const UINT n = ptd->size(); - assert(pdw == 0 || pdw->size() == n); - - REAL err = 0; - for (UINT i = 0; i < n; ++i) { - bool e = (lm.c_error(lm.get_output(i), ptd->y(i)) > 0.1); - if (e && pdw) err += (*pdw)[i]; - ndx[i+1] = U(i); - val[i+1] = e? 1 : -1; // "discretize" the base learner to -1/1 - } - if (err >= maxe && pdw) // Cannot find better hypotheses - return false; - - lpx_add_rows(lp, 1); - int Rt = lpx_get_num_rows(lp); - lpx_set_mat_row(lp, Rt, n, ndx, val); - lpx_set_row_bnds(lp, Rt, LPX_LO, -1.0, 0.0); // R(t) >= -1 - - return true; -} - -// return the negative objective, sum of u -REAL lp_solve (LPX* lp, pDataWgt& pdw) { - lpx_simplex(lp); - REAL sumu = -lpx_get_obj_val(lp); - - // get the new sample weights - UINT n = lpx_get_num_cols(lp); - DataWgt* sample_wgt = new DataWgt(n); - for (UINT i = 0; i < n; ++i) { - double wgt = lpx_get_col_prim(lp, U(i)); - assert(wgt >= -EPSILON); - if (wgt < 0) wgt = 0; - (*sample_wgt)[i] = wgt / sumu; - } - pdw = sample_wgt; - return sumu; -} - -void LPBoost::train () { - assert(ptd != 0 && ptw != 0); - assert(lm_base != 0); // we need lm_base to create new hypotheses - assert(!grad_desc_view); - set_dimensions(*ptd); - - // Construct inner problem - LPX* lp = lpx_create_prob(); - lpx_add_cols(lp, n_samples); // u_i - for (UINT i = 0; i < n_samples; ++i) { - lpx_set_col_bnds(lp, U(i), LPX_DB, 0.0, - RegC * (*ptw)[i] * n_samples); // 0 <= u_i <= C_i - lpx_set_obj_coef(lp, U(i), -1); // obj: -sum u_i - } - lpx_set_obj_dir(lp, LPX_MIN); // min obj - - int* ndx = new int[n_samples+1]; double* val = new double[n_samples+1]; - - // adding existing hypotheses - for (UINT t = 0; t < size(); ++t) { - const pLearnModel p = lm[t]; - assert(p->train_data() == ptd); - lp_add_hypothesis(lp, ndx, val, *p); - } - n_in_agg = size(); - - REAL sumu = RegC * n_samples; // the largest possible sum of u - pDataWgt pdw = ptw; - for (UINT t = n_in_agg; t < max_n_model; ++t) { - if (t > 0) - sumu = lp_solve(lp, pdw); - if (sumu < EPSILON) { // we do not expect this to happen - std::cerr << "Warning: sum u is " << sumu << "; quit earlier.\n"; - break; - } - REAL besterr = (1 - 1 / sumu) / 2; - - const pLearnModel p = train_with_smpwgt(pdw); - assert(p->train_data() == ptd); - if (!lp_add_hypothesis(lp, ndx, val, *p, pdw, besterr-EPSILON)) - break; - - set_dimensions(*p); - lm.push_back(p); lm_wgt.push_back(0); - ++n_in_agg; - } - - lpx_simplex(lp); - - // Update hypothesis coefficients - assert((UINT) lpx_get_num_rows(lp) == R(n_in_agg-1)); - for (UINT k = 0; k < n_in_agg; ++k) { - lm_wgt[k] = lpx_get_row_dual(lp, R(k)); - assert(lm_wgt[k] > -EPSILON); - if (lm_wgt[k] < 0) lm_wgt[k] = 0; - } - - delete[] ndx; delete[] val; - lpx_delete_prob(lp); -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.h deleted file mode 100644 index 01033ee..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/lpboost.h +++ /dev/null @@ -1,87 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_LPBOOST_H__ -#define __LEMGA_AGGREGATING_LPBOOST_H__ - -/** @file - * @brief Declare @link lemga::LPBoost LPBoost@endlink class. - * - * $Id: lpboost.h 2696 2006-04-05 20:10:13Z ling $ - */ - -#include -#include "boosting.h" - -namespace lemga { - -/** @brief %LPBoost (Linear-Programming %Boosting). - * - * With a similar idea to the original %LPBoost [1], which solves - * \f{eqnarray*} - * \min && -\rho + D \sum_i \xi_i \\ - * \textrm{s.t.}&& y_i\left(\sum_t\alpha_t h_t(x_i)\right)\ge \rho-\xi_i, \\ - * && \xi_i\ge 0, \quad \alpha_t\ge 0, \quad \sum_t\alpha_t=1. - * \f} - * we instead implement the algorithm to solve - * \f{eqnarray*} - * \min && \sum_t \alpha_t + C \sum_i \xi_i \\ - * \textrm{s.t.}&& y_i \left(\sum_t \alpha_t h_t(x_i)\right) \ge 1 - \xi_i,\\ - * && \xi_i \ge 0, \quad \alpha_t \ge 0. - * \f} - * by column generation. Note that the dual problem is - * \f{eqnarray*} - * \max && \sum_i u_i \\ - * \textrm{s.t.}&& \sum_i u_i y_i h_t(x_i) \le 1, \qquad (*)\\ - * && 0 \le u_i \le C. - * \f} - * Column generation corresponds to generating the constraints (*). - * We actually use individual upper bound @f$C_i@f$ proportional to - * example's initial weight. - * - * If we treat @f$w_i@f$, the normalized version of @f$u_i@f$, as the - * sample weight, and @f$\Sigma_u = \sum_i u_i@f$ as the normalization - * constraint, (*) is the same as - * @f[ \Sigma_u (1 - 2 e(h_t, w)) \le 1, @f] - * which means - * @f[ e(h_t, w) \ge \frac12 (1 - \Sigma_u^{-1}).\qquad (**) @f] - * Assume that we have found @f$h_1, \dots, h_T@f$ so far, solving the dual - * problem with @f$T@f$ (*) constraints gives us @f$\Sigma_u@f$. If for - * every remaining @f$h@f$ in @f$\cal{H}@f$, - * @f$ e(h, w) \ge \frac12 (1 - \Sigma_u^{-1}),@f$ - * the duality condition tells us that even if we set @f$\alpha=0@f$ for those - * remaining @f$h@f$, the solution is still optimal. Thus, we can train the - * weak learner with sample weight @f$w@f$ in each iteration, and terminate - * if the best hypothesis has satisfied (**). - * - * [1] A. Demiriz, K. P. Bennett, and J. Shawe-Taylor. Linear programming - * boosting via column generation. Machine Learning, - * 46(1-3):225-254, 2002. - */ -class LPBoost : public Boosting { - REAL RegC; - -public: - explicit LPBoost () : Boosting(false) { set_C(1); } - LPBoost (const Boosting& s) : Boosting(s) { assert(!convex); set_C(1); } - explicit LPBoost (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual LPBoost* create () const { return new LPBoost(); } - virtual LPBoost* clone () const { return new LPBoost(*this); } - - /// @todo Implement something similar to that in CGBoost. - virtual bool set_aggregation_size (UINT) { return false; } - virtual void train (); - - /// The regularization constant C. - REAL C () const { return RegC; } - /// Set the regularization constant C. - void set_C (REAL _C) { assert(_C >= 0); RegC = _C; } -}; - -} // namespace lemga - -#ifdef __LPBOOST_H__ -#warning "This header file may conflict with another `lpboost.h' file." -#endif -#define __LPBOOST_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.cpp deleted file mode 100644 index e939c93..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.cpp +++ /dev/null @@ -1,25 +0,0 @@ -/** @file - * $Id: mgnboost.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "optimize.h" -#include "mgnboost.h" - -REGISTER_CREATOR(lemga::MgnBoost); - -namespace lemga { - -void MgnBoost::train () { - assert(!convex && grad_desc_view); - Boosting::train(); -} - -void MgnBoost::train_gd () { - _mgn_gd mgd(this); - iterative_optimize(_line_search<_mgn_gd,BoostWgt,REAL,REAL> - (&mgd, convex? 1.0 : 0.5)); - update_cost_l(); // optional, just to be consistent with the def. -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.h deleted file mode 100644 index 5db76d1..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/mgnboost.h +++ /dev/null @@ -1,94 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_AGGREGATING_MGNBOOST_H__ -#define __LEMGA_AGGREGATING_MGNBOOST_H__ - -/** @file - * @brief Declare @link lemga::MgnBoost MgnBoost@endlink class. - * - * $Id: mgnboost.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include "boosting.h" - -namespace lemga { - -/** @brief Cost proxy used in MgnBoost. - * - * This class is a proxy to any other ``base'' cost functor. The - * minimal margin @f$\lambda@f$ is subtracted from the margin of - * each examples, and then the base cost functor is called. - */ -struct MgnCost : public cost::Cost { - const cost::Cost& orig_cost; - REAL lambda; ///< current minimal margin @f$\lambda@f$ - REAL wsum; ///< current sum of weak learner weights - - MgnCost (const cost::Cost& c = cost::_cost) - : orig_cost(c), lambda(0), wsum(0) {} - virtual REAL cost (const REAL& F, const REAL& y) const { - return orig_cost.cost(F - lambda*wsum/y, y); - } - virtual REAL deriv1 (const REAL& F, const REAL& y) const { - return orig_cost.deriv1(F - lambda*wsum/y, y); - } -}; - -/** @brief %MgnBoost (margin maximizing boosting). - * - * %MgnBoost is an implementation of the arc-gv boosting algorithm - * [1]. We add a proxy MgnCost to modify any cost function, though - * arc-gv only uses the exponential cost (which is the default). - * (Thus the use of other cost functions is just experimental.) The - * minimal margin is updated only before the gradient calculation, - * but not during the line-search step. This is exactly how arc-gv - * works. I've tried updating the minimal margin also in the - * line-search step, but it didn't work well (minimal margin remains - * very negative). - * - * [1] L. Breiman. Prediction games and arcing algorithms. - * Neural Computation, 11(7):1493-1517, 1999. - */ -class MgnBoost : public Boosting { - friend struct _mgn_gd; - MgnCost shifted_cost; - void update_cost_l () { shifted_cost.lambda = min_margin()/margin_norm(); } - void update_cost_w () { shifted_cost.wsum = model_weight_sum(); } - -public: - explicit MgnBoost (bool cvx = false, const cost::Cost& c = cost::_cost) - : Boosting(cvx, shifted_cost), shifted_cost(c) - { use_gradient_descent(); } - explicit MgnBoost (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual MgnBoost* create () const { return new MgnBoost(); } - virtual MgnBoost* clone () const { return new MgnBoost(*this); } - - virtual void train (); - -protected: - virtual void train_gd (); -}; - -struct _mgn_gd : public _boost_gd { - MgnBoost* pm; - explicit _mgn_gd (MgnBoost* p) : _boost_gd(p), pm(p) {} - - void set_weight (const Boosting::BoostWgt& bw) { - _boost_gd::set_weight(bw); - pm->update_cost_w(); - } - - Boosting::BoostWgt gradient () const { - pm->update_cost_w(); pm->update_cost_l(); - return _boost_gd::gradient(); - } -}; - -} // namespace lemga - -#ifdef __MGNBOOST_H__ -#warning "This header file may conflict with another `mgnboost.h' file." -#endif -#define __MGNBOOST_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.cpp deleted file mode 100644 index bf7d396..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.cpp +++ /dev/null @@ -1,415 +0,0 @@ -/** @file - * $Id: multiclass_ecoc.cpp 2695 2006-04-05 05:21:47Z ling $ - */ - -#include -#include -#include -#include -#include "multiclass_ecoc.h" - -REGISTER_CREATOR(lemga::MultiClass_ECOC); - -#define LABEL_EQUAL(x,y) (std::fabs((x)-(y)) < EPSILON) -// search y in the current class labels -#define LABEL2INDEX(y) \ - std::lower_bound(labels.begin(), labels.end(), y) - labels.begin() - -namespace lemga { - -bool MultiClass_ECOC::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(Aggregating, os, vl, 1); - const UINT n = lm.size(); - - assert(lm_wgt.size() == n); - for (UINT i = 0; i < n; ++i) - os << lm_wgt[i] << ' '; - if (n) os << '\n'; - - os << ecoc.size() << '\n'; - for (UINT i = 0; i < ecoc.size(); ++i) { - assert(n <= ecoc[i].size()); - for (UINT j = 0; j < n; ++j) - os << ecoc[i][j] << ' '; - if (n) os << '\n'; - } - - os << nclass << '\n'; - assert(nclass == labels.size()); - for (UINT i = 0; i < nclass; ++i) - os << labels[i] << ' '; - if (nclass) os << '\n'; - return os; -} - -bool -MultiClass_ECOC::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(Aggregating, is, vl, 1, v); - - const UINT n = lm.size(); - lm_wgt.resize(n); - for (UINT i = 0; i < n; ++i) - if (!(is >> lm_wgt[i])) return false; - - UINT c; - if (!(is >> c)) return false; - ecoc.resize(c); - for (UINT i = 0; i < c; ++i) { - ecoc[i].resize(n); - for (UINT j = 0; j < n; ++j) - if (!(is >> ecoc[i][j])) return false; - } - - if (!(is >> nclass)) return false; - labels.resize(nclass); - for (UINT i = 0; i < nclass; ++i) - if (!(is >> labels[i])) return false; - - return true; -} - -void MultiClass_ECOC::set_ECOC_table (const ECOC_TABLE& e) { - assert(!e.empty()); - ecoc = e; - if (max_n_model == 0) // only set the max_n_model if it is 0 - set_max_models(ecoc[0].size()); -} - -void MultiClass_ECOC::set_ECOC_table (UINT i, const ECOC_VECTOR& p) { - for (UINT c = 0; c < p.size(); ++c) { - if (ecoc[c].size() <= i) { - assert(ecoc[c].size() == i); - ecoc[c].push_back(p[c]); - } - else ecoc[c][i] = p[c]; - } -} - -void MultiClass_ECOC::set_ECOC_table (ECOC_TYPE et) { - assert(et != NO_TYPE); - if (nclass == 0) { // data not loaded, - ecoc_type = et; // do this later in set_train_data() - return; - } - - ECOC_TABLE e; - - switch (et) { - case ONE_VS_ONE: { - UINT m = nclass*(nclass-1)/2; - e = ECOC_TABLE(nclass, ECOC_VECTOR(m, 0)); - UINT k = 0; - for (UINT i = 0; i < nclass; ++i) - for (UINT j = i+1; j < nclass; ++j, ++k) { - e[i][k] = 1; e[j][k] = -1; - } - } - break; - - case ONE_VS_ALL: - e = ECOC_TABLE(nclass, ECOC_VECTOR(nclass, -1)); - for (UINT i = 0; i < nclass; ++i) - e[i][i] = 1; - break; - - default: - assert(false); - } - - set_ECOC_table(e); -} - -// the default one in LearnModel is not suitable for multiclass -REAL MultiClass_ECOC::c_error (const Output& out, const Output& y) const { - assert(n_output() == 1); - return !LABEL_EQUAL(out[0], y[0]); -} - -void MultiClass_ECOC::reset () { - Aggregating::reset(); - lm_wgt.clear(); - /// @todo Keep the initial ECOC settings - ecoc.clear(); ecoc_type = NO_TYPE; -#if MULTICLASS_ECOC_OUTPUT_CACHE - if (nclass > 0) clear_cache(); -#endif -} - -void MultiClass_ECOC::set_train_data (const pDataSet& pd, - const pDataWgt& pw) { - pDataSet old_ptd = ptd; - Aggregating::set_train_data(pd, pw); - if (old_ptd == ptd) { - assert(ecoc_type == NO_TYPE); - return; - } - - std::set ls; - for (UINT i = 0; i < n_samples; ++i) - ls.insert(ptd->y(i)[0]); - - // compare with the old label information - std::vector new_labels(ls.begin(), ls.end()); - if (nclass > 0) { - bool same = (nclass == new_labels.size()); - for (UINT c = 0; same && c < nclass; ++c) - same = LABEL_EQUAL(labels[c], new_labels[c]); - if (!same) - std::cerr << "MultiClass_ECOC: Warning: " - "Class label mapping is changed\n"; - } - - // check whether labels are too close - labels.swap(new_labels); - nclass = labels.size(); - for (UINT c = 1; c < nclass; ++c) - if (LABEL_EQUAL(labels[c-1], labels[c])) { - std::cerr << "MultiClass_ECOC: Error: Labels (" - << labels[c-1] << " and " << labels[c] - << ") are too close. Alter LABEL_EQUAL?\n"; - std::exit(-1); - } - - // fill ex_class - ex_class.resize(n_samples); - for (UINT i = 0; i < n_samples; ++i) { - REAL y = ptd->y(i)[0]; - UINT c = LABEL2INDEX(y); - assert(c < nclass && LABEL_EQUAL(y, labels[c])); - ex_class[i] = c; - } - - if (ecoc_type != NO_TYPE) { - set_ECOC_table(ecoc_type); - ecoc_type = NO_TYPE; - } else - ecoc.resize(nclass); -#if MULTICLASS_ECOC_OUTPUT_CACHE - clear_cache(); -#endif - local_d.resize(nclass); -} - -void MultiClass_ECOC::train () { - assert(ptd != 0 && ptw != 0); - assert(lm_base != 0); // we need lm_base to create new hypotheses - assert(_n_out == 1); // currently only deal with one output - assert(nclass <= ecoc.size() && nclass == labels.size()); - - // start learning from the current set of hypotheses - n_in_agg = lm.size(); - assert(n_in_agg == lm_wgt.size()); - setup_aux(); - - while (n_in_agg < max_n_model) { - ECOC_VECTOR par; - if (!ECOC_partition(n_in_agg, par)) break; -#if VERBOSE_OUTPUT - std::cout << "*** " << id() << " #" << n_in_agg+1 - << " / " << max_n_model << " ***\n"; -#endif - // train with this partition (the partition may be altered) - const pLearnModel p = train_with_partition(par); - // decide the coefficient - const REAL w = assign_weight(par, *p); - if (w <= 0) break; - - set_dimensions(*p); - lm.push_back(p); lm_wgt.push_back(w); - set_ECOC_table(n_in_agg, par); - ++n_in_agg; - update_aux(par); - } -} - -// find the closest class -#define GET_BEST_CLASS(distance_to_class_c) \ - REAL dmin = INFINITY; UINT cmin = UINT(-1); \ - for (UINT c = 0; c < nclass; ++c) { \ - REAL dc = distance_to_class_c; \ - assert(dc < INFINITY/10); \ - if (dc < dmin) { dmin = dc; cmin = c; } \ - } -#define GET_MARGIN(distance_to_class_c,y) \ - REAL dy = 0, dmin = INFINITY; \ - for (UINT c = 0; c < nclass; ++c) { \ - REAL dc = distance_to_class_c; \ - assert(dc < INFINITY/10); \ - if (c == y) dy = dc; \ - else if (dc < dmin) dmin = dc; \ - } - -const std::vector& MultiClass_ECOC::distances (const Input& x) const { - assert(n_in_agg <= lm.size() && n_in_agg <= lm_wgt.size()); -#ifndef NDEBUG - for (UINT i = 0; i < n_in_agg; ++i) - assert(lm_wgt[i] >= 0); -#endif - - assert(n_output() == 1); - Output out(n_in_agg); - for (UINT i = 0; i < n_in_agg; ++i) { - assert(lm[i] != 0); - out[i] = (*lm[i])(x)[0]; - } - std::vector& d = local_d; - assert(d.size() == nclass); - for (UINT c = 0; c < nclass; ++c) - d[c] = ECOC_distance(out, ecoc[c]); - return d; -} - -const std::vector& MultiClass_ECOC::distances (UINT idx) const { -#if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache - assert(cache_n.size() == n_samples); - if (cache_n[idx] > n_in_agg) - clear_cache(idx); - std::vector& d = cache_d[idx]; - assert(d.size() == nclass); - for (UINT i = cache_n[idx]; i < n_in_agg; ++i) { - REAL o = lm[i]->get_output(idx)[0]; - for (UINT c = 0; c < nclass; ++c) - d[c] = ECOC_distance(o, ecoc[c][i], lm_wgt[i], d[c]); - } - cache_n[idx] = n_in_agg; -#else // compute the output first -#if MULTICLASS_ECOC_OUTPUT_CACHE == 1 - assert(cache_o.size() == n_samples); - Output& out = cache_o[idx]; - for (UINT i = out.size(); i < n_in_agg; ++i) { - assert(lm[i] != 0); - out.push_back(lm[i]->get_output(idx)[0]); - } -#elif !MULTICLASS_ECOC_OUTPUT_CACHE - Output out(n_in_agg); - for (UINT i = 0; i < n_in_agg; ++i) { - assert(lm[i] != 0); - out[i] = lm[i]->get_output(idx)[0]; - } -#else -#error "Wrong value of MULTICLASS_ECOC_OUTPUT_CACHE" -#endif - std::vector& d = local_d; - assert(d.size() == nclass); - for (UINT c = 0; c < nclass; ++c) - d[c] = ECOC_distance(out, ecoc[c]); -#endif - return d; -} - -Output MultiClass_ECOC::operator() (const Input& x) const { - const std::vector& d = distances(x); - GET_BEST_CLASS(d[c]); - return Output(1, labels[cmin]); -} - -Output MultiClass_ECOC::get_output (UINT idx) const { - assert(ptw != 0); // no data sampling - assert(nclass <= ecoc.size() && nclass == labels.size()); - - const std::vector& d = distances(idx); - GET_BEST_CLASS(d[c]); - // Due to the instability of real number comparison, the - // assertion below does not always hold. However, it holds if we - // replace (dc < dmin) in GET_BEST_CLASS with (dc+EPSILON < dmin) - //assert(LABEL_EQUAL(labels[cmin], (*this)(ptd->x(idx))[0])); - return Output(1, labels[cmin]); -} - -REAL MultiClass_ECOC::margin_of (const Input& x, const Output& l) const { - UINT y = LABEL2INDEX(l[0]); - assert(y < nclass && LABEL_EQUAL(l[0], labels[y])); - const std::vector& d = distances(x); - GET_MARGIN(d[c], y); - return dmin - dy; -} - -REAL MultiClass_ECOC::margin (UINT idx) const { - assert(ptw != 0); // no data sampling - UINT y = ex_class[idx]; - - const std::vector& d = distances(idx); - GET_MARGIN(d[c], y); - return dmin - dy; -} - -REAL MultiClass_ECOC::ECOC_distance (const Output& o, - const ECOC_VECTOR& cw) const { - assert(n_in_agg <= o.size() && n_in_agg <= cw.size()); - assert(n_in_agg <= lm_wgt.size()); - REAL ip = 0; - for (UINT i = 0; i < n_in_agg; ++i) - ip += lm_wgt[i] * (o[i]>0? -.5:.5) * cw[i]; - return ip; -} - -#if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache -REAL MultiClass_ECOC::ECOC_distance (REAL o, int cw, REAL w, REAL d) const { - return d + w * (o>0? -.5:.5) * cw; -} -#endif - -bool MultiClass_ECOC::is_full_partition (const ECOC_VECTOR& p) const { - assert(p.size() == nclass); - for (UINT c = 0; c < nclass; ++c) - if (p[c] != -1 && p[c] != 1) return false; - return true; -} - -bool MultiClass_ECOC::ECOC_partition (UINT i, ECOC_VECTOR& p) const { - assert(nclass <= ecoc.size()); - p.resize(nclass); - for (UINT c = 0; c < nclass; ++c) { - if (i >= ecoc[c].size()) return false; - p[c] = ecoc[c][i]; - } - return true; -} - -pLearnModel MultiClass_ECOC::train_with_partition (ECOC_VECTOR& p) const { - LearnModel *plm = lm_base->clone(); - assert(plm != 0); - - // generate the binary data - DataSet* btd = new DataSet(); - DataWgt* btw = new DataWgt(); - REAL wsum = 0; - for (UINT i = 0; i < n_samples; ++i) { - const Input& x = ptd->x(i); - int b = p[ex_class[i]]; - if (b != 0) { - assert(b == 1 || b == -1); - btd->append(x, Output(1, b)); - btw->push_back((*ptw)[i]); - wsum += (*ptw)[i]; - } - } - assert(wsum > 0); - for (UINT i = 0; i < btw->size(); ++i) - (*btw)[i] /= wsum; - - plm->set_train_data(btd, btw); - plm->train(); - // To save some memory, we put back the original set and weight - plm->set_train_data(ptd, ptw); - return plm; -} - -REAL MultiClass_ECOC::cost () const { - assert(ptd != 0 && ptw != 0); - REAL cst = 0; - for (UINT i = 0; i < n_samples; ++i) { - const std::vector& d = distances(i); - const UINT y = ex_class[i]; - REAL dy = d[y], csti = 0; - for (UINT c = 0; c < nclass; ++c) - if (c != y) { - csti += std::exp(dy - d[c]); - } - cst += (*ptw)[i] * csti; - } - return cst; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.h deleted file mode 100644 index b3a7037..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/multiclass_ecoc.h +++ /dev/null @@ -1,136 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_MULTICLASS_ECOC_H__ -#define __LEMGA_MULTICLASS_ECOC_H__ - -/** @file - * @brief Declare @link lemga::MultiClass_ECOC MultiClass_ECOC@endlink - * (Multiclass classification with Error-Correcting Output Code) class. - * - * $Id: multiclass_ecoc.h 2695 2006-04-05 05:21:47Z ling $ - */ - -#include -#include "aggregating.h" - -// 0 - no cache; 1 - cache output; 2 - cache distance -#define MULTICLASS_ECOC_OUTPUT_CACHE 2 - -namespace lemga { - -typedef std::vector ECOC_VECTOR; -typedef std::vector ECOC_TABLE; -enum ECOC_TYPE { - NO_TYPE, - ONE_VS_ONE, - ONE_VS_ALL -}; - -/** @brief Multiclass classification using error-correcting output code. - */ -class MultiClass_ECOC : public Aggregating { -protected: - std::vector lm_wgt; ///< hypothesis weight - ECOC_TABLE ecoc; ///< the ECC table - /// The type of the ECOC table, if there is some fixed type. - /// @note It is not serialized, at least for now. - ECOC_TYPE ecoc_type; - - // variables extracted from the training data - UINT nclass; ///< number of classes - std::vector labels; ///< class labels - std::vector ex_class; ///< class number of examples - -public: - MultiClass_ECOC () : Aggregating(), ecoc_type(NO_TYPE), nclass(0) - { set_dimensions(0, 1); } - explicit MultiClass_ECOC (std::istream& is) : ecoc_type(NO_TYPE) - { is >> *this; } - - virtual const id_t& id () const; - virtual MultiClass_ECOC* create () const { return new MultiClass_ECOC(); } - virtual MultiClass_ECOC* clone () const { - return new MultiClass_ECOC(*this); } - - REAL model_weight (UINT n) const { return lm_wgt[n]; } - const ECOC_TABLE& ECOC_table () const { return ecoc; } - void set_ECOC_table (const ECOC_TABLE&); - void set_ECOC_table (ECOC_TYPE); - void set_ECOC_table (UINT, const ECOC_VECTOR&); - UINT n_class () const { return nclass; } - - virtual bool support_weighted_data () const { return true; } - virtual REAL c_error (const Output& out, const Output& y) const; - virtual void set_train_data (const pDataSet&, const pDataWgt& = 0); - virtual void train (); - virtual void reset (); - virtual Output operator() (const Input&) const; - virtual Output get_output (UINT idx) const; - - virtual REAL margin (UINT) const; - virtual REAL margin_of (const Input&, const Output&) const; - /// The in-sample exponential cost defined in my paper. - REAL cost () const; - -#if MULTICLASS_ECOC_OUTPUT_CACHE -private: -#if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache - mutable std::vector cache_n; - mutable std::vector > cache_d; - inline void clear_cache (UINT i) const { - assert(i < n_samples && cache_n.size() == n_samples && nclass > 0); - cache_n[i] = 0; - std::vector cdi(nclass, 0); - cache_d[i].swap(cdi); - } - inline void clear_cache () const { - cache_d.resize(n_samples); - cache_n.resize(n_samples); - for (UINT i = 0; i < n_samples; ++i) - clear_cache(i); - } -#elif MULTICLASS_ECOC_OUTPUT_CACHE == 1 - mutable std::vector cache_o; - inline void clear_cache () const { - std::vector co(n_samples); - cache_o.swap(co); - } -#else -#error "Wrong value of MULTICLASS_ECOC_OUTPUT_CACHE in `multiclass_ecoc.h'." -#endif -#endif // MULTICLASS_ECOC_OUTPUT_CACHE - -protected: - virtual REAL ECOC_distance (const Output&, const ECOC_VECTOR&) const; -#if MULTICLASS_ECOC_OUTPUT_CACHE == 2 // distance cache - virtual REAL ECOC_distance (REAL, int, REAL, REAL = 0) const; -#endif - - mutable std::vector local_d; - /// @note It might be unsafe to use a reference to a local - /// variable as the return value - const std::vector& distances (const Input&) const; - const std::vector& distances (UINT) const; - - /// Does the partition only consist of -1 and +1? - bool is_full_partition (const ECOC_VECTOR&) const; - - /// Prepare auxiliary variables for current n_in_agg - virtual void setup_aux () {} - virtual bool ECOC_partition (UINT, ECOC_VECTOR&) const; - virtual pLearnModel train_with_partition (ECOC_VECTOR&) const; - virtual REAL assign_weight (const ECOC_VECTOR&, const LearnModel&) const - { return 1; } - /// Update those auxiliary variables after this round of learning - virtual void update_aux (const ECOC_VECTOR&) {} - - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __MULTICLASS_ECOC_H__ -#warning "This header file may conflict with another `multiclass_ecoc.h' file." -#endif -#define __MULTICLASS_ECOC_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.cpp deleted file mode 100644 index b176a06..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/** @file - * $Id: nnlayer.cpp 2550 2006-01-17 04:00:54Z ling $ - */ - -#include -#include -#include -#include "random.h" -#include "quickfun.h" -#include "nnlayer.h" - -REGISTER_CREATOR(lemga::NNLayer); - -namespace lemga { - -NNLayer::NNLayer (UINT n_in, UINT n_unit) - : LearnModel(n_in, n_unit), w_min(-1), w_max(1), - w(n_unit*(n_in+1)), dw(n_unit*(n_in+1)), sig_der(n_unit) -{ - quick_tanh_setup(); -} - -bool NNLayer::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - if (!(os << w_min << ' ' << w_max << '\n')) return false; - WVEC::const_iterator pw = w.begin(); - for (UINT i = 0; i < _n_out; ++i) { - for (UINT j = 0; j <= _n_in; ++j, ++pw) - if (!(os << *pw << ' ')) return false; - os << '\n'; - } - return true; -} - -bool NNLayer::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - - if (v == 0) // Take care of _n_in and _n_out - if (!(is >> _n_in >> _n_out)) return false; - if (!(is >> w_min >> w_max) || w_min >= w_max) return false; - - const UINT n_weights = _n_out * (_n_in+1); - w.resize(n_weights); - dw = WVEC(n_weights, 0); - sig_der = DVEC(_n_out); - - for (UINT i = 0; i < n_weights; ++i) - if (!(is >> w[i])) return false; - return true; -} - -void NNLayer::set_weight (const WVEC& wgt) { - assert(wgt.size() == _n_out * (_n_in+1)); - w = wgt; -} - -void NNLayer::clear_gradient () { - std::fill(dw.begin(), dw.end(), 0); -} - -void NNLayer::initialize () { - for (WVEC::iterator pw = w.begin(); pw != w.end(); ++pw) - *pw = w_min + randu() * (w_max-w_min); - clear_gradient(); -} - -REAL NNLayer::sigmoid (REAL x) const { - stored_sigmoid = quick_tanh(x); - return stored_sigmoid; -} - -REAL NNLayer::sigmoid_deriv (REAL x) const { - assert(stored_sigmoid == quick_tanh(x)); - return (1 - stored_sigmoid*stored_sigmoid); -} - -void NNLayer::feed_forward (const Input& x, Output& y) const { - assert(x.size() == n_input()); - assert(y.size() == n_output()); - - WVEC::const_iterator pw = w.begin(); - for (UINT i = 0; i < _n_out; ++i) { - const REAL th = *pw; - const REAL s = std::inner_product(x.begin(), x.end(), ++pw, th); - pw += _n_in; - - y[i] = sigmoid(s); - sig_der[i] = sigmoid_deriv(s); - } -} - -void NNLayer::back_propagate (const Input& x, const DVEC& dy, DVEC& dx) { - assert(x.size() == n_input()); - assert(dy.size() == n_output()); - assert(dx.size() == n_input()); - - std::fill(dx.begin(), dx.end(), 0); - - WVEC::const_iterator pw = w.begin(); - DVEC::iterator pdw = dw.begin(); - for (UINT i = 0; i < _n_out; ++i) { - const REAL delta = dy[i] * sig_der[i]; - *pdw += delta; ++pdw; ++pw; - - DVEC::iterator pdx = dx.begin(); - Input::const_iterator px = x.begin(); - for (UINT j = _n_in; j; --j) { - *pdw++ += delta * (*px++); - *pdx++ += delta * (*pw++); - } - } -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.h deleted file mode 100644 index ad6480c..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/nnlayer.h +++ /dev/null @@ -1,110 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_NNLAYER_H__ -#define __LEMGA_NNLAYER_H__ - -/** @file - * @brief Neural network layer @link lemga::NNLayer NNLayer@endlink. - * - * $Id: nnlayer.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "learnmodel.h" - -namespace lemga { - -/** @brief A layer in a neural network. - * - * This class simulates a layer of neurons. - * - * Here's usage information. - * - * Here's some details. - * - * Say, we have @a n neurons and @a m inputs. The output from - * the neuron is - * @f[ y_i=\theta(s_i),\quad s_i=\sum_{j=0}^m w_{ij}x_j, @f] - * where @f$s_i@f$ is the weighted sum of inputs, @f$x_0\equiv1@f$, - * and @f$w_{i0}@f$ is sort of the threshold. feed_forward() does - * this calculation and saves @f$\theta'(s_i)@f$ for future use. - * - * The ``chain rule'' for back-propagation says the derative w.r.t. - * the input @a x can be calculated from that to the output @a y. - * Define @f[ - * \delta_i \stackrel\Delta= \frac{\partial E}{\partial s_i} - * = \frac{\partial E}{\partial y_i}\frac{\partial y_i}{\partial s_i} - * = \frac{\partial E}{\partial y_i}\theta'(s_i). @f] - * We have @f[ - * \frac{\partial E}{\partial w_{ij}} - * = \frac{\partial E}{\partial s_i} - * \frac{\partial s_i}{\partial w_{ij}} - * = \delta_i x_j,\quad - * \frac{\partial E}{\partial x_j} - * = \sum_{i=1}^n \frac{\partial E}{\partial s_i} - * \frac{\partial s_i}{\partial x_j} - * = \sum_{i=1}^n \delta_i w_{ij}. @f] - * These equations consitute the essense of back_propagate(). - * - * Modifying sigmoid() (which computes @f$\theta@f$) and - * sigmoid_deriv() (which computes @f$\theta'@f$) is usually enough - * to get a different type of layer. - * @todo documentation - */ -class NNLayer : public LearnModel { - mutable REAL stored_sigmoid; - -public: - typedef std::vector WVEC; ///< weight vector - typedef std::vector DVEC; ///< derivative vector - -protected: - REAL w_min, w_max; - WVEC w; ///< weights and thresholds - WVEC dw; ///< deravatives: @a w -= lr * @a dw - mutable DVEC sig_der; // f'(wsum) - -public: - explicit NNLayer (UINT n_in = 0, UINT n_unit = 0); - explicit NNLayer (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual NNLayer* create () const { return new NNLayer(); } - virtual NNLayer* clone () const { return new NNLayer(*this); } - - UINT size () const { return n_output(); } - - void set_weight_range (REAL min, REAL max) { - assert(min < max); - w_min = min; w_max = max; - } - const WVEC& weight () const { return w; } - void set_weight (const WVEC&); - - const WVEC& gradient () const { return dw; } - void clear_gradient (); - - virtual void initialize (); - virtual void train () { OBJ_FUNC_UNDEFINED("train"); } - virtual Output operator() (const Input& x) const { - Output y(n_output()); - feed_forward(x, y); - return y; - } - - void feed_forward (const Input&, Output&) const; - void back_propagate (const Input&, const DVEC&, DVEC&); - -protected: - virtual REAL sigmoid (REAL) const; - virtual REAL sigmoid_deriv (REAL) const; - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __NNLAYER_H__ -#warning "This header file may conflict with another `nnlayer.h' file." -#endif -#define __NNLAYER_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.cpp deleted file mode 100644 index 0dd0786..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/** @file - * $Id: object.cpp 2554 2006-01-18 02:55:21Z ling $ - */ - -#include -#include -#include -#include "object.h" - -const Object::id_t Object::NIL_ID; - -typedef std::map _creator_map_t; - -/** @brief Wrapper for the static creator map. - * - * The wrapper is to ensure @a cmap is initialized before it is used. - * @see C++ FAQ Lite 10.11, 10.12, and 10.13 - */ -static _creator_map_t& _creator_map () { - static _creator_map_t cmap; - return cmap; -} - -/** @return @c true if @id is already registered. */ -inline static bool _creator_registered (const Object::id_t& id) { - return _creator_map().find(id) != _creator_map().end(); -} - -/** @brief Register @a id with creator @a cf. - * @param id String of class identification. - * @param cf Function pointer to the creator. - */ -_register_creator::_register_creator (const Object::id_t& id, _creator_t cf) { - assert(!_creator_registered(id)); - _creator_map()[id] = cf; -} - -/** Serialization ``writes'' the object to an output stream, which - * helps to store, transport, and transform the object. See - * C++ FAQ Lite 35 for a technical overview. - * - * @param os Output stream - * @param vl A list of version numbers (from its descendants) - * @sa operator<<(std::ostream&, const Object&) - * - * The first part of serialization must be the class name in full - * (with namespaces if any) followed by a list of version numbers - * from itself and its ancestors. A version number gives the - * version of the serialization format, but not the version - * of an object. - * - * A class that adds some data to its parent's serialization should - * call serialize() of its parent with its own version number appended - * to @a vl. Eventually Object::serialize() is called to output the - * class id and the list of versions. - * - * @note A zero version number is reserved for pre-0.1 format. - * @todo serialize() should both return false and set os's state - * when fail - */ -bool Object::serialize (std::ostream& os, ver_list& vl) const { - assert(vl.size() > 0); - ver_list::const_iterator p = vl.begin(); - os << "# " << id() << " v" << *p; - for (++p; p != vl.end(); ++p) - os << '.' << *p; - os << '\n'; - return true; -} - -static bool _get_id_ver (std::istream& is, Object::id_t& id, - std::vector& vl) { - assert(vl.empty()); - const UINT ignore_size = 512; - - // get the comment char # - char c; - if (!(is >> c)) return false; - if (c != '#') { - std::cerr << "_get_id_ver: Warning: # expected\n"; - is.ignore(ignore_size, '#'); - } - - // get the rest of the line - char line[ignore_size]; - is.getline(line, ignore_size); - std::istringstream iss(line); - - // get the identity (class name) - if (!(iss >> id)) { - std::cerr << "_get_id_ver: Error: no class id?\n"; - return false; - } - - // get the version - vl.push_back(0); // we use 0 for unspecified versions - iss.ignore(ignore_size, 'v'); - UINT v; - while (iss >> v) { - assert(v > 0); - vl.push_back(v); - iss.ignore(1); - } - if (vl.size() == 1) { - std::cerr << "_get_id_ver: Warning: pre-0.1 file format\n"; - if (id == "AdaBoost.M1") id = "AdaBoost"; - if (id.substr(0, 7) != "lemga::") id.insert(0, "lemga::"); - } - - return true; -} - -/** - * @todo better exception; documentation - * @sa _register_creator, Object::unserialize, C++ FAQ Lite 35.8 - */ -Object* Object::create (std::istream& is) { - id_t _id; - ver_list vl; - if (!_get_id_ver(is, _id, vl)) return 0; - assert(vl[0] == 0); - - if (_creator_registered(_id)) { - Object* p = _creator_map()[_id](); - bool loaded = p->unserialize(is, vl, _id); - if (loaded && vl.size() == 1) - return p; - else if (loaded) - std::cerr << _id << "::create: newer format encountered\n"; - delete p; - } else - std::cerr << "Class (" << _id << ") not regiestered\n"; - - return 0; -} - -std::istream& operator>> (std::istream& is, Object& obj) { - Object::id_t id; - Object::ver_list vl; - - // unserialize() judges whether id and obj are compatible - if (!_get_id_ver(is, id, vl) || !obj.unserialize(is, vl, id)) { - std::cerr << "operator>>: something wrong...\n"; - is.setstate(std::ios::badbit); - } else if (vl.size() > 1) { - std::cerr << "operator>>: newer format encountered\n"; - is.setstate(std::ios::badbit); - } - assert(vl[0] == 0); - - return is; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.h deleted file mode 100644 index 669882c..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/object.h +++ /dev/null @@ -1,182 +0,0 @@ -// -*- C++ -*- -#ifndef __COMMON_TYPES_OBJECT_H__ -#define __COMMON_TYPES_OBJECT_H__ - -/** @file - * @brief Object class, _register_creator, and other common types. - * - * $Id: object.h 2554 2006-01-18 02:55:21Z ling $ - */ - -#ifdef size_t -typedef size_t UINT; -#else -typedef unsigned long UINT; -#endif - -typedef double REAL; - -#define INFINITESIMAL (8e-16) ///< almost equal to 0, 1+x/2-1>0 -#define EPSILON (1e-9) ///< accumulated numerical error -#undef INFINITY -/// Should avoid using INFINITY since it is a lazy trick. -#define INFINITY (5e30) - -#ifdef __cplusplus - -#include -#include -#include -#include - -/** @brief Stop the program and warn when an undefined function is called. - * - * Some member functions may be @em optional for child classes. There - * are two cases for an optional function: - * -# It is declared as an interface; Some child classes may implement - * it but others may not. This macro should be called in the interface - * class. See NNLayer::train for an example. - * -# It is defined and implemented but may be inappropriate for some - * child classes. To prevent misuse of this function, use this macro - * to redefine it in those child classes. - * - * This macro will stop the program with a ``function undefined" error. - * - * @note This approach differs from defining functions as pure virtual, - * since using the latter way, a child class cannot be instantiated without - * implementing all pure virtual functions, while some of them may not - * be desired. - * - * @todo A better macro name; Auto-extraction of function name (could be - * done by __PRETTY_FUNCTION__ but is not compatible between compilers) - */ -#define OBJ_FUNC_UNDEFINED(fun) \ - std::cerr<<__FILE__":"<<__LINE__<<": "<> (std::istream&, Object&); - friend std::ostream& operator<< (std::ostream&, const Object&); - -public: - virtual ~Object () {} - - typedef std::string id_t; - /** @return Class ID string (class name) */ - virtual const id_t& id () const = 0; - - /** @brief Create a new object using the default constructor. - * - * The code for a derived class Derived is always - * @code return new Derived(); @endcode - */ - virtual Object* create () const = 0; - - /// Create a new object from an input stream. - static Object* create (std::istream&); - - /** @brief Create a new object by replicating itself. - * @return A pointer to the new copy. - * - * The code for a derived class Derived is always - * @code return new Derived(*this); @endcode - * Though seemingly redundant, it helps to copy an object without - * knowing the real type of the object. - * @sa C++ FAQ Lite 20.6 - */ - virtual Object* clone () const = 0; - -protected: - static const id_t NIL_ID; - typedef UINT ver_t; - typedef std::vector ver_list; - /** @brief Serialize the object to an output stream. - * @sa Macro SERIALIZE_PARENT, unserialize() - * - * Each class should either have its own serialize() and a positive - * version number, or ensure that all its descendants don't overload - * serialize() thus have 0 as the version number. This way we can - * safely assume, if we go down in the object hierarchy tree from - * Object to the any class, the versions are positive numbers followed - * by 0's. In other words, the version list is preceded by some 0's. - */ - virtual bool serialize (std::ostream&, ver_list&) const; - /** @brief Unserialize from an input stream. - * @sa Macro UNSERIALIZE_PARENT, serialize() - * - * The ID and version list are used for sanity check. For any parent - * classes, the ID will be set to @a NIL_ID. Thus any classes that - * is definitely some parent should assert ID == @a NIL_ID. - * - * A class which previously did not have its own serialize() or - * unserialize(), usually because it had nothing to store, can have - * its own serialize() and unserialize() later on, as long as it takes - * care of the 0 version number case in unserialize(). - */ - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID) - { return true; } -}; - -inline std::ostream& operator<< (std::ostream& os, const Object& obj) { - Object::ver_list vl_empty; - obj.serialize(os, vl_empty); - return os; -} - -typedef Object* (*_creator_t) (); - -/** Unserialization from an input stream will create an object of a now - * unknown class. To facilitate this process, each instantiable derived - * class (that is, a class with all pure virtual functions implemented) - * must have a creator and add it to this map. The way is to declare a - * static _register_creator object in the .cpp file. - * @code static const _register_creator _("Derived", _creator); @endcode - * @sa C++ FAQ Lite 35.8 - */ -struct _register_creator { - _register_creator(const Object::id_t&, _creator_t); -}; - -/// Use a prefix @a p to allow several classes in a same .cpp file -#define REGISTER_CREATOR2(cls,p) \ - static const Object::id_t p##_id_ = #cls; \ - static Object* p##_c_ () { return new cls; } \ - const Object::id_t& cls::id () const { return p##_id_; } \ - static const _register_creator p##_(p##_id_, p##_c_) -#define REGISTER_CREATOR(cls) REGISTER_CREATOR2(cls,) - -/** @brief Serialize parents (ancestors) part. - * @param pcls Parent class. - * @param os Output stream. - * @param def_ver Current (default) version. - */ -#define SERIALIZE_PARENT(pcls,os,vl,def_ver) \ - vl.push_back(def_ver); pcls::serialize(os, vl) - -/** @brief Unserialize parents and set current format number @a ver. - * @param pcls Parent class (input). - * @param is Input stream (input). - * @param ver Version number for this class (output). - */ -#define UNSERIALIZE_PARENT(pcls,is,vl,def_ver,ver) \ - if (!pcls::unserialize(is, vl)) return false; \ - const ver_t ver = vl.back(); \ - if (ver > def_ver) { \ - std::cerr << this->id() \ - << ": unserialize: format newer than the program\n"; \ - return false; \ - } \ - if (ver > 0) vl.pop_back() - -#endif // __cplusplus - -#ifdef __OBJECT_H__ -#warning "This header file may conflict with another `object.h' file." -#endif -#define __OBJECT_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/optimize.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/optimize.h deleted file mode 100644 index df5bb8a..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/optimize.h +++ /dev/null @@ -1,380 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_OPTIMIZE_H__ -#define __LEMGA_OPTIMIZE_H__ - -/** @file - * @brief Some generic optimization algorithms. - * - * $Id: optimize.h 1917 2005-01-07 01:25:30Z ling $ - */ - -#include -#include -#include -#include "vectorop.h" - -namespace lemga { - -using namespace op; - -/** @brief Interface used in iterative optimization algorithms. - * - * This template is used in joint with the generic optimization - * algorithm iterative_optimize(). It describes functions that a - * class have to provide in order to use iterative_optimize(). - * - * Some arithmetic operations on gradient are needed for optimization. - * The full set required consists of - * - @a u += @a v - * - -@a v - * - @a v *= @a r - * - inner_product<@a typename> (@a u, @a v) - * where @a u and @a v are weights/gradients and @a r the step length. - * - * Using such interface improves the flexibility of code. However, - * there are always concerns about the performance. Here are - * several considerations for the sake of performance: - * - * - The weight(s) is the variable to be ``searched'' in order to - * optimize the cost. It is assumed to be stored somewhere (say, - * in a derived class) and is not passed as a parameter to these - * functions. - * - direction() returns a const reference to a class member. - * - The inheritance of different methods doesn't mean the parent - * class could be changed without taking care of the classes - * derived from it. This is just used as a way to share functions. - * - step_lenth() return non-positive step length if the direction - * is improper. (It could have returned a pair.) - * - * @todo Does making these functions pure virtual sacrifice - * the performance? - */ -template -struct _search { - typedef Dir direction_type; - typedef Step step_length_type; - - /// Initialize local variables. - void initialize () {} - /// Search direction at @a w. - const Dir& direction () { return dir; } - /// Should we go in direction @a d? How far?. - std::pair step_length (const Dir& d) { - return std::make_pair(false, 0); } - /// Update the weight. - void update_weight (const Dir& d, const Step& s) {} - /// Stopping criteria. - bool satisfied () { return true; } -protected: - Dir dir; -}; - -/// Main search routine. -template -void iterative_optimize (SEARCH s) { - s.initialize(); - while (!s.satisfied()) { - const typename SEARCH::direction_type& - pd = s.direction(); - const std::pair - stp = s.step_length(pd); - - if (!stp.first) break; - s.update_weight(pd, stp.second); - } -} - -/** @brief Gradient descent. - * @todo Add options for on-line or batch mode; documentation - * Why use get_weight() and set_weight(), not update_weight()? - * (how much is the performance decrease) - */ -template -struct _gradient_descent : public _search { - LM* plm; - Step learning_rate; - - _gradient_descent (LM* lm, const Step& lr) - : _search(), plm(lm), learning_rate(lr) {} - - void initialize () { stp_cnt = 0; w = plm->weight(); } - - const Dir& direction () { - using namespace op; - return (this->dir = -plm->gradient()); - } - - std::pair step_length (const Dir&) { - return std::make_pair(true, learning_rate); - } - - void update_weight (Dir d, const Step& s) { - using namespace op; - w += (d *= s); - ++stp_cnt; plm->set_weight(w); - } - - bool satisfied () { return plm->stop_opt(stp_cnt, plm->cost()); } - -protected: - Dir w; - unsigned int stp_cnt; -}; - -/** @brief Gradient descent with weight decay. - * - * The cost function includes a regularization term which prefers - * smaller weight values. That is, the cost to be minimized is - * @f[ \hat{E} = E + \frac{\lambda}2 \Vert w\Vert^2. @f] - * The updating rule thus becomes @f[ - * w \leftarrow w - \eta \frac{\partial\hat{E}}{\partial w} - * = (1-\eta\lambda)w - \eta\frac{\partial E}{\partial w}. - * @f] - * @f$\lambda@f$ usually ranges from 0.01 to 0.1. - * - * @todo Well, I guess weight decay is more a term in the cost - * than an optimization technique. Though it is easy to add it - * here (as an optimization technique). - */ -template -struct _gd_weightdecay : public _gradient_descent { - Step decay; - - _gd_weightdecay (LM* lm, const Step& lr, const Step& dcy) - : _gradient_descent(lm,lr), decay(dcy) {} - - void update_weight (Dir d, const Step& s) { - using namespace op; - assert(s*decay < 1); - this->w *= (1 - s*decay); - this->w += (d *= s); - ++this->stp_cnt; this->plm->set_weight(this->w); - } -}; - -/** @brief Gradient descent with momentum. - * - * With momentum, the weight change is accumulated: @f[ - * \Delta w\leftarrow-\eta\frac{\partial E}{\partial w}+\mu\Delta w. - * @f] - * When the derivative is roughly constant, this speeds up the - * training by using an effectively larger learning rate @f[ - * \Delta w \approx -\frac{\eta}{1-\mu}\frac{\partial E}{\partial w}. - * @f] - * When the derivative somehow ``fluctuates'', the momentum stablizes - * the optimization to some exent by effectively decreasing the - * learning rate. - */ -template -struct _gd_momentum : public _gradient_descent { - Step momentum; - - _gd_momentum (LM* lm, const Step& lr, const Step& m) - : _gradient_descent(lm,lr), momentum(m) {} - - const Dir& direction () { - assert(momentum >= 0 && momentum < 1); - using namespace op; - if (this->stp_cnt > 0) { - this->dir *= momentum; - this->dir += -this->plm->gradient(); - } - else this->dir = -this->plm->gradient(); - return this->dir; - } -}; - -template -struct _gd_adaptive : public _gradient_descent { - using _gradient_descent::plm; - using _gradient_descent::learning_rate; - using _gradient_descent::w; - using _gradient_descent::stp_cnt; - Step alpha, beta; - - _gd_adaptive (LM* lm, const Step& lr, const Step& a, const Step& b) - : _gradient_descent(lm,lr), alpha(a), beta(b) {} - - void initialize () { - _gradient_descent::initialize(); - old_cost = plm->cost(); - } - - std::pair step_length (Dir d) { - assert(alpha >= 1 && beta < 1); - using namespace op; - - Step lr = learning_rate; - - d *= learning_rate; - Dir wd = w; - plm->set_weight(wd += d); - Cost c = plm->cost(); - if (c < old_cost) - learning_rate *= alpha; - else { - do { - learning_rate *= beta; - d *= beta; wd = w; - plm->set_weight(wd += d); - c = plm->cost(); - } while (!(c < old_cost) && learning_rate > 1e-6); - lr = learning_rate; - } - - const bool cont = (c < old_cost); - if (cont) old_cost = c; - else plm->set_weight(w); - return std::make_pair(cont, lr); - } - - void update_weight (Dir d, const Step& s) { - ++stp_cnt; - using namespace op; - w += (d *= s); - // DO NOT set_weight; done in step_length() - } - - bool satisfied () { return plm->stop_opt(stp_cnt, old_cost); } - -protected: - Cost old_cost; -}; - -namespace details { - -template -Step line_search (LM& lm, const Dir& w, Cost& cst3, - const Dir& dir, Step step) { - using namespace op; - assert(w == lm.weight()); - cst3 = lm.cost(); - Step stp3 = 0; - - Dir d = dir, wd = w; d *= step; - lm.set_weight(wd += d); Cost cst5 = lm.cost(); - while (cst5 > cst3 && step > 2e-7) { - //std::cout << '-'; - step *= 0.5; d *= 0.5; wd = w; - lm.set_weight(wd += d); cst5 = lm.cost(); - } - - if (cst5 > cst3) { - std::cerr << "\tWarning: not a descending direction\n"; - lm.set_weight(w); // undo - return 0; - } - - Step stp1, stp5 = step; - do { - //std::cout << '*'; - step += step; - stp1 = stp3; - stp3 = stp5; cst3 = cst5; - stp5 += step; - d = dir; d *= stp5; wd = w; - lm.set_weight(wd += d); cst5 = lm.cost(); - } while (cst5 < cst3); - - while (stp3 > stp1*1.01 || stp5 > stp3*1.01) { - //std::cout << '.'; - Step stp2 = (stp1 + stp3) / 2; - Step stp4 = (stp3 + stp5) / 2; - d = dir; d *= stp2; wd = w; - lm.set_weight(wd += d); Cost cst2 = lm.cost(); - d = dir; d *= stp4; wd = w; - lm.set_weight(wd += d); Cost cst4 = lm.cost(); - - if (cst4 < cst2 && cst4 < cst3) { - stp1 = stp3; - stp3 = stp4; cst3 = cst4; - } - else if (cst2 < cst3) { - stp5 = stp3; - stp3 = stp2; cst3 = cst2; - } - else { - stp1 = stp2; - stp5 = stp4; - } - } - //std::cout << "\tcost = " << cst3 << ", step = " << stp3 << '\n'; - return stp3; -} - -} // namespace lemga::details - -template -struct _line_search : public _gradient_descent { - using _gradient_descent::plm; - using _gradient_descent::learning_rate; - using _gradient_descent::w; - using _gradient_descent::stp_cnt; - - _line_search (LM* lm, const Step& lr) - : _gradient_descent(lm,lr) {} - - void initialize () { - _gradient_descent::initialize(); - cost_w = plm->cost(); - } - - std::pair step_length (const Dir& d) { - const Step stp = - details::line_search(*plm, w, cost_w, d, learning_rate); - return std::make_pair((stp>0), stp); - } - - bool satisfied () { return plm->stop_opt(stp_cnt, cost_w); } - -protected: - Cost cost_w; -}; - -template -struct _conjugate_gradient : public _line_search { - using _gradient_descent::plm; - using _gradient_descent::w; - using _gradient_descent::stp_cnt; - using _search::dir; - - _conjugate_gradient (LM* lm, const Step& lr) - : _line_search(lm,lr) {} - - const Dir& direction () { - // get g - const Dir g = plm->gradient(); - const Step g_norm = op::inner_product(g, g); - - using namespace op; - // get d - if (stp_cnt == 0) - dir = -g; - else { - const Step g_dot_old = op::inner_product(g, g_old); - assert(g_norm_old > 0); - Step beta = (g_norm - g_dot_old) / g_norm_old; - if (beta < 0) beta = 0; - - dir *= beta; - dir += -g; - } - - g_old = g; - g_norm_old = g_norm; - - return dir; - } - -private: - Dir g_old; - Step g_norm_old; -}; - -} // namespace lemga - -#ifdef __OPTIMIZE_H__ -#warning "This header file may conflict with another `optimize.h' file." -#endif -#define __OPTIMIZE_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.cpp deleted file mode 100644 index aa120d6..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.cpp +++ /dev/null @@ -1,320 +0,0 @@ -/** @file - * $Id: ordinal_ble.cpp 2782 2006-05-15 19:25:35Z ling $ - */ - -#include -#include "ordinal_ble.h" - -REGISTER_CREATOR(lemga::Ordinal_BLE); - -namespace lemga { - -#define OUT2RANK(y) ((UINT) ((y) - .5)) -#define RANK2OUT(r) ((r) + 1) -#define VALIDRANK(y) ((y) > .9 && std::fabs((y)-1-OUT2RANK(y)) < INFINITESIMAL) -#define nrank (out_tab.size()) -#define n_hyp (ext_tab.size()) - -bool Ordinal_BLE::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - - if (!(os << nrank << ' ' << n_hyp << ' ' << n_ext << ' ' - << full_ext << ' ' << (lm != 0) << '\n')) - return false; - - for (UINT i = 0; i < nrank; ++i) { - assert(out_tab[i].size() == n_hyp); - for (UINT j = 0; j < n_hyp; ++j) - os << out_tab[i][j] << ' '; - if (n_hyp) os << '\n'; - } - for (UINT i = 0; i < n_hyp; ++i) { - assert(ext_tab[i].size() == n_ext); - for (UINT j = 0; j < n_ext; ++j) - os << ext_tab[i][j] << ' '; - if (n_ext) os << '\n'; - } - - if (lm != 0) { - assert(_n_in == 0 || lm->valid_dimensions(_n_in+n_ext, 1)); - return (os << *lm); - } - return os; -} - -bool Ordinal_BLE::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - assert(v > 0); - - lm = 0; ptd = 0; ptw = 0; ext_d = 0; ext_w = 0; - d_nrank = 0; reset_data = true; - - UINT nr, nh, fe, tr; - if (!(is >> nr >> nh >> n_ext >> fe >> tr) || fe > 1 || tr > 1) - return false; - full_ext = fe; - - out_tab.resize(nr); ext_tab.resize(nh); - for (UINT i = 0; i < nrank; ++i) { - out_tab[i].resize(n_hyp); - for (UINT j = 0; j < n_hyp; ++j) - is >> out_tab[i][j]; - } - for (UINT i = 0; i < n_hyp; ++i) { - ext_tab[i].resize(n_ext); - for (UINT j = 0; j < n_ext; ++j) - is >> ext_tab[i][j]; - } - - if (tr) { - lm = (LearnModel*) Object::create(is); - if (lm == 0 || !(_n_in == 0 || lm->valid_dimensions(_n_in+n_ext, 1))) - return false; - } - return true; -} - -Ordinal_BLE::Ordinal_BLE (const Ordinal_BLE& o) - : LearnModel(o), lm(0), full_ext(o.full_ext), - out_tab(o.out_tab), ext_tab(o.ext_tab), n_ext(o.n_ext), - ext_d(o.ext_d), ext_w(o.ext_w), d_nrank(o.d_nrank), - reset_data(o.reset_data) -{ - if (o.lm != 0) lm = o.lm->clone(); -} - -const Ordinal_BLE& Ordinal_BLE::operator= (const Ordinal_BLE& o) { - if (&o == this) return *this; - - LearnModel::operator=(o); - lm = 0; full_ext = o.full_ext; - out_tab = o.out_tab; ext_tab = o.ext_tab; n_ext = o.n_ext; - ext_d = o.ext_d; ext_w = o.ext_w; d_nrank = o.d_nrank; - reset_data = o.reset_data; - if (o.lm != 0) lm = o.lm->clone(); - - return *this; -} - -void Ordinal_BLE::set_model (const LearnModel& l) { - lm = l.clone(); - reset_data = true; -} - -void Ordinal_BLE::set_full_extension (bool f) { - assert(f); //??? only deal with full-extension for now - if (full_ext ^ f) { // full_ext will be changed - ext_d = 0; ext_w = 0; - } - full_ext = f; -} - -void Ordinal_BLE::set_tables (const ECOC_TABLE& ecc, const EXT_TABLE& ext) { - out_tab = ecc; ext_tab = ext; - assert(nrank > 1 && n_hyp > 0); - n_ext = ext_tab[0].size(); -#ifndef NDEBUG - for (UINT i = 0; i < nrank; ++i) - assert(out_tab[i].size() == n_hyp); - for (UINT i = 0; i < n_hyp; ++i) - assert(ext_tab[i].size() == n_ext); -#endif - local_d.resize(nrank); -} - -void Ordinal_BLE::set_tables (BLE_TYPE bt, UINT nr) { - ECOC_TABLE ecc(nr); - EXT_TABLE ext; - - switch (bt) { - case MULTI_THRESHOLD: - assert(nr > 1); - for (UINT i = 0; i < nr; ++i) { - ecc[i].resize(nr-1, -1); - for (UINT j = 0; j < i; ++j) - ecc[i][j] = 1; - } - ext.resize(nr-1); - for (UINT i = 0; i < nr-1; ++i) { - ext[i].resize(nr-1, 0); - ext[i][i] = 1; - } - break; - - default: - assert(false); - } - - set_tables(ecc, ext); -} - -REAL Ordinal_BLE::c_error (const Output& out, const Output& y) const { - assert(n_output() == 1 && VALIDRANK(out[0]) && VALIDRANK(y[0])); - return OUT2RANK(out[0]) != OUT2RANK(y[0]); -} - -REAL Ordinal_BLE::r_error (const Output& out, const Output& y) const { - assert(n_output() == 1 && VALIDRANK(out[0]) && VALIDRANK(y[0])); - return std::fabs(out[0] - y[0]); -} - -void Ordinal_BLE::set_train_data (const pDataSet& pd, const pDataWgt& pw) { - pDataSet old_ptd = ptd; - LearnModel::set_train_data(pd, pw); - if (old_ptd == ptd) return; - - ext_d = 0; ext_w = 0; - UINT old_nr = d_nrank; - - // let's be sure that the labels are 1-K (nrank) - std::vector has_example; - UINT nr = 0; - for (UINT i = 0; i < n_samples; ++i) { - REAL y = ptd->y(i)[0]; - if (!VALIDRANK(y)) { - std::cerr << "Ordinal_BLE: Error: " - << "Label (" << y << ") is not a valid rank.\n"; - std::exit(-1); - } - UINT r = OUT2RANK(y); - if (r >= has_example.size()) - has_example.resize(r+1, false); - nr += !has_example[r]; - has_example[r] = true; - } - d_nrank = has_example.size(); - if (nr < d_nrank) { - std::cerr << "Ordinal_BLE: Warning: " << "Missing rank(s) "; - for (UINT r = 0; r < d_nrank; ++r) - if (!has_example[r]) { - std::cerr << RANK2OUT(r); - if (++nr < d_nrank) std::cerr << ", "; - } - std::cerr << ".\n"; - } - - if (old_nr > 0 && old_nr != d_nrank) - std::cerr << "Ordinal_BLE: Warning: " - << "Number of ranks changed from " << old_nr - << " to " << d_nrank << ".\n"; -} - -void Ordinal_BLE::extend_data () { - //??? full extension only - assert(n_hyp > 0 && ptd != 0 && n_samples > 0); - - DataSet* rd = new DataSet; - DataWgt* rw = new DataWgt; - rw->reserve(n_samples * n_hyp); - - // don't assume _n_in has been set - UINT nin = ptd->x(0).size(); - Input rx(nin + n_ext); - for (UINT i = 0; i < n_samples; ++i) { - const Input& x = ptd->x(i); - const UINT r = OUT2RANK(ptd->y(i)[0]); - assert(x.size() == nin && r < nrank); - const REAL wgt = (*ptw)[i] / n_hyp; - - for (UINT j = 0; j < n_hyp; ++j) { - REAL ry; - extend_example(x, r, j, rx, ry); - rd->append(rx, Output(1, ry)); - rw->push_back(wgt); - } - } - - ext_d = rd; ext_w = rw; - reset_data = true; -} - -void Ordinal_BLE::train () { - assert(ptd != 0 && ptw != 0); - assert(lm != 0); - if (nrank == 0) // set the default tables - set_tables(BLE_DEFAULT, d_nrank); - - assert(nrank > 0 && n_hyp > 0); - if (d_nrank > nrank) { - std::cerr << "Ordinal_BLE: Error: " - << "More ranks in the data than in the ECC matrix.\n"; - std::exit(-1); - } else if (d_nrank < nrank) - std::cerr << "Ordinal_BLE: Warning: " - << "Less ranks in the data than in the ECC matrix.\n"; - - set_dimensions(*ptd); - if (ext_d == 0) extend_data(); - assert(ext_d != 0 && ext_w != 0); - - if (reset_data) - lm->set_train_data(ext_d, ext_w); - reset_data = false; - lm->train(); -} - -void Ordinal_BLE::reset () { - LearnModel::reset(); - if (lm != 0) lm->reset(); -} - -#define GET_BEST_RANK(distance_to_rank_r) \ - REAL dmin = INFINITY; UINT rmin = UINT(-1); \ - for (UINT r = 0; r < nrank; ++r) { \ - REAL dr = distance_to_rank_r; \ - assert(dr < INFINITY/10); \ - if (dr < dmin) { dmin = dr; rmin = r; } \ - } - -Output Ordinal_BLE::operator() (const Input& x) const { - assert(valid_dimensions(x.size(), 1)); - const std::vector d = distances(x); - GET_BEST_RANK(d[r]); - return Output(1, RANK2OUT(rmin)); -} - -/** @arg t: hypothesis index */ -void Ordinal_BLE::extend_input (const Input& x, UINT t, Input& ext_x) const { - UINT n_in = x.size(); - assert(t < n_hyp && ext_tab[t].size() == n_ext); - assert(ext_x.size() == n_in + n_ext); - //ext_x.resize(n_in + n_ext); - std::copy(x.begin(), x.end(), ext_x.begin()); - std::copy(ext_tab[t].begin(), ext_tab[t].end(), ext_x.begin()+n_in); -} - -void Ordinal_BLE::extend_example (const Input& x, UINT r, UINT t, - Input& ext_x, REAL& ext_y) const -{ - assert(r < nrank && out_tab[r].size() == n_hyp); - extend_input(x, t, ext_x); - ext_y = out_tab[r][t]; -} - -REAL Ordinal_BLE::ECOC_distance (const Output& o, - const ECOC_VECTOR& cw) const { - assert(o.size() == n_hyp && n_hyp <= cw.size()); - REAL d = 0; - for (UINT i = 0; i < n_hyp; ++i) - d += std::exp(- o[i] * cw[i]); - return d; -} - -const std::vector& Ordinal_BLE::distances (const Input& x) const { - UINT nin = x.size(); - assert(lm != 0 && lm->exact_dimensions(nin+n_ext, 1)); - Input rx(nin + n_ext); - Output out(n_hyp); - for (UINT j = 0; j < n_hyp; ++j) { - extend_input(x, j, rx); - out[j] = (*lm)(rx)[0]; - } - std::vector& d = local_d; - assert(local_d.size() == nrank); - for (UINT i = 0; i < nrank; ++i) - d[i] = ECOC_distance(out, out_tab[i]); - return d; -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.h deleted file mode 100644 index 7f43380..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/ordinal_ble.h +++ /dev/null @@ -1,91 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_ORDINAL_BLE_H__ -#define __LEMGA_ORDINAL_BLE_H__ - -/** @file - * @brief Declare @link lemga::Ordinal_BLE Ordinal_BLE@endlink class. - * - * $Id: ordinal_ble.h 2782 2006-05-15 19:25:35Z ling $ - */ - -#include "learnmodel.h" -#include "multiclass_ecoc.h" // ECOC_VECTOR & ECOC_TABLE - -namespace lemga { -typedef std::vector > EXT_TABLE; -enum BLE_TYPE { - MULTI_THRESHOLD, - BLE_DEFAULT = MULTI_THRESHOLD -}; - -/** @brief Ordinal regression via binary learning on extended examples - */ -class Ordinal_BLE : public LearnModel { -protected: - pLearnModel lm; ///< the learning model - bool full_ext; ///< use the full extension or the partial one? - ECOC_TABLE out_tab; ///< K (nrank) by T (n_hyp) output ECC matrix - EXT_TABLE ext_tab; ///< T (n_hyp) by E (n_ext) extension matrix - UINT n_ext; - -public: - Ordinal_BLE () : full_ext(true), n_ext(0), d_nrank(0) - { set_dimensions(0, 1); } - Ordinal_BLE (const Ordinal_BLE&); - const Ordinal_BLE& operator= (const Ordinal_BLE&); - explicit Ordinal_BLE (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Ordinal_BLE* create () const { return new Ordinal_BLE(); } - virtual Ordinal_BLE* clone () const { return new Ordinal_BLE(*this); } - - /// set the underlying learning model - void set_model (const LearnModel&); - /// the underlying model - const LearnModel& model () const { assert(lm != 0); return *lm; } - - //@{ Settings on how to extend the original examples - bool full_extension () const { return full_ext; } - void set_full_extension (bool = true); - const ECOC_TABLE& ECOC_table () const { return out_tab; } - const EXT_TABLE& extension_table () const { return ext_tab; } - void set_tables (const ECOC_TABLE&, const EXT_TABLE&); - void set_tables (BLE_TYPE, UINT); - //@} - - /// the number of ranks from the training set - UINT n_rank () const { return d_nrank; } - - virtual bool support_weighted_data () const { return true; } - virtual REAL c_error (const Output& out, const Output& y) const; - virtual REAL r_error (const Output& out, const Output& y) const; - virtual void set_train_data (const pDataSet&, const pDataWgt& = 0); - virtual void train (); - virtual void reset (); - virtual Output operator() (const Input&) const; - -protected: - pDataSet ext_d; ///< the extended dataset - pDataWgt ext_w; ///< the weight for the extended set - UINT d_nrank; ///< number of ranking levels of the training set - bool reset_data; ///< whether to reset the training set for lm - - void extend_input (const Input&, UINT, Input&) const; - void extend_example (const Input&, UINT, UINT, Input&, REAL&) const; - void extend_data (); - - virtual REAL ECOC_distance (const Output&, const ECOC_VECTOR&) const; - mutable std::vector local_d; - const std::vector& distances (const Input&) const; - - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __ORDINAL_BLE_H__ -#warning "This header file may conflict with another `ordinal_ble.h' file." -#endif -#define __ORDINAL_BLE_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.cpp deleted file mode 100644 index 32bcf73..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.cpp +++ /dev/null @@ -1,644 +0,0 @@ -/** @file - * $Id: perceptron.cpp 2891 2006-11-08 03:17:31Z ling $ - */ - -// "Fixed bias" means we still use the bias term, but do not change it. -// This is to simulate a perceptron without the bias term. An alternative -// is to modify the dset_distract (for RCD) and those update rules (for -// other algorithms) to not add "1" to input vectors. Although this approach -// is simpler (to implement), it causes extra troubles with saving/loading -// or importing from SVM. - -#define STUMP_DOUBLE_DIRECTIONS 0 - -#include -#include -#include -#include "random.h" -#include "utility.h" -#include "stump.h" -#include "perceptron.h" - -REGISTER_CREATOR(lemga::Perceptron); - -typedef std::vector RVEC; -typedef std::vector RMAT; - -#define DOTPROD(x,y) std::inner_product(x.begin(), x.end(), y.begin(), .0) -// The version without bias -#define DOTPROD_NB(x,y) std::inner_product(x.begin(),x.end()-1,y.begin(),.0) - -inline RVEC randvec (UINT n) { - RVEC x(n); - for (UINT i = 0; i < n; ++i) - x[i] = 2*randu() - 1; - return x; -} - -inline RVEC coorvec (UINT n, UINT c) { - RVEC x(n, 0); - x[c] = 1; - return x; -} - -inline void normalize (RVEC& v, REAL thr = 0) { - REAL s = DOTPROD(v, v); - assert(s > 0); - if (s <= thr) return; - s = 1 / std::sqrt(s); - for (RVEC::iterator p = v.begin(); p != v.end(); ++p) - *p *= s; -} - -RMAT randrot (UINT n, bool bias_row = false, bool conjugate = true) { - RMAT m(n); - if (bias_row) - m.back() = coorvec(n, n-1); - - for (UINT i = 0; i+bias_row < n; ++i) { - m[i] = randvec(n); - if (bias_row) m[i][n-1] = 0; - if (!conjugate) continue; - - // make m[i] independent - for (UINT k = 0; k < i; ++k) { - REAL t = DOTPROD(m[i], m[k]); - for (UINT j = 0; j < n; ++j) - m[i][j] -= t * m[k][j]; - } - // normalize m[i] - normalize(m[i]); - } - return m; -} - -namespace lemga { - -/// Update the weight @a wgt along the direction @a dir. -/// If necessary, the whole @a wgt will be negated. -void update_wgt (RVEC& wgt, const RVEC& dir, const RMAT& X, const RVEC& y) { - const UINT dim = wgt.size(); - assert(dim == dir.size()); - const UINT n_samples = X.size(); - assert(n_samples == y.size()); - - RVEC xn(n_samples), yn(n_samples); -#if STUMP_DOUBLE_DIRECTIONS - REAL bias = 0; -#endif - for (UINT i = 0; i < n_samples; ++i) { - assert(X[i].size() == dim); - const REAL x_d = DOTPROD(X[i], dir); - REAL x_new = 0, y_new = 0; - if (x_d != 0) { - y_new = (x_d > 0)? y[i] : -y[i]; - REAL x_w = DOTPROD(X[i], wgt); - x_new = x_w / x_d; - } -#if STUMP_DOUBLE_DIRECTIONS - else - bias += (DOTPROD(X[i], wgt) > 0)? y[i] : -y[i]; -#endif - xn[i] = x_new; yn[i] = y_new; - } - -#if STUMP_DOUBLE_DIRECTIONS - REAL th1, th2; - bool ir, pos; - Stump::train_1d(xn, yn, bias, ir, pos, th1, th2); - const REAL w_d_new = -(th1 + th2) / 2; -#else - const REAL w_d_new = - Stump::train_1d(xn, yn); -#endif - - for (UINT j = 0; j < dim; ++j) - wgt[j] += w_d_new * dir[j]; - -#if STUMP_DOUBLE_DIRECTIONS - if (!pos) - for (UINT j = 0; j < dim; ++j) - wgt[j] = -wgt[j]; -#endif - - // use a big threshold to reduce the # of normalization - normalize(wgt, 1e10); -} - -void dset_extract (const pDataSet& ptd, RMAT& X, RVEC& y) { - UINT n = ptd->size(); - X.resize(n); y.resize(n); - for (UINT i = 0; i < n; ++i) { - X[i] = ptd->x(i); - X[i].push_back(1); - y[i] = ptd->y(i)[0]; - } -} - -inline void dset_mult_wgt (const pDataWgt& ptw, RVEC& y) { - UINT n = y.size(); - assert(ptw->size() == n); - for (UINT i = 0; i < n; ++i) - y[i] *= (*ptw)[i]; -} - -Perceptron::Perceptron (UINT n_in) - : LearnModel(n_in, 1), wgt(n_in+1,0), resample(0), - train_method(RCD_BIAS), learn_rate(0.002), min_cst(0), - max_run(1000), with_fld(false), fixed_bias(false) -{ -} - -Perceptron::Perceptron (const SVM& s) - : LearnModel(s), wgt(_n_in+1,0), resample(0), - train_method(RCD_BIAS), learn_rate(0.002), min_cst(0), - max_run(1000), with_fld(false), fixed_bias(false) -{ - assert(s.kernel().id() == kernel::Linear().id()); - - const UINT nsv = s.n_support_vectors(); - for (UINT i = 0; i < nsv; ++i) { - const Input sv = s.support_vector(i); - const REAL coef = s.support_vector_coef(i); - assert(sv.size() == _n_in); - for (UINT k = 0; k < _n_in; ++k) - wgt[k] += coef * sv[k]; - } - wgt.back() = s.bias(); -} - -bool Perceptron::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - for (UINT i = 0; i <= _n_in; ++i) - if (!(os << wgt[i] << ' ')) return false; - return (os << '\n'); -} - -bool Perceptron::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - - wgt.resize(_n_in+1); - for (UINT i = 0; i <= _n_in; ++i) - if (!(is >> wgt[i])) return false; - return true; -} - -void Perceptron::set_weight (const WEIGHT& w) { - assert(w.size() > 1); - set_dimensions(w.size()-1, 1); - wgt = w; -} - -void Perceptron::initialize () { - if (ptd != 0) set_dimensions(*ptd); - assert(_n_in > 0); - wgt.resize(_n_in + 1); - for (UINT i = 0; i < _n_in + (fixed_bias? 0:1); ++i) - wgt[i] = randu() * 0.1 - 0.05; // small random numbers -} - -Perceptron::WEIGHT Perceptron::fld () const { - assert(ptd != 0 && ptw != 0); - - // get the mean - Input m1(_n_in, 0), m2(_n_in, 0); - REAL w1 = 0, w2 = 0; - for (UINT i = 0; i < n_samples; ++i) { - const Input& x = ptd->x(i); - REAL y = ptd->y(i)[0]; - REAL w = (*ptw)[i]; - - if (y > 0) { - w1 += w; - for (UINT k = 0; k < _n_in; ++k) - m1[k] += w * x[k]; - } else { - w2 += w; - for (UINT k = 0; k < _n_in; ++k) - m2[k] += w * x[k]; - } - } - assert(w1 > 0 && w2 > 0 && std::fabs(w1+w2-1) < EPSILON); - for (UINT k = 0; k < _n_in; ++k) { - m1[k] /= w1; m2[k] /= w2; - } - - // get the covariance - RMAT A(_n_in, RVEC(_n_in, 0)); - RVEC diff(_n_in); - for (UINT i = 0; i < n_samples; ++i) { - const Input& x = ptd->x(i); - REAL y = ptd->y(i)[0]; - REAL w = (*ptw)[i]; - - if (y > 0) - for (UINT k = 0; k < _n_in; ++k) - diff[k] = x[k] - m1[k]; - else - for (UINT k = 0; k < _n_in; ++k) - diff[k] = x[k] - m2[k]; - // we only need the upper triangular part - for (UINT j = 0; j < _n_in; ++j) - for (UINT k = j; k < _n_in; ++k) - A[j][k] += w * diff[j] * diff[k]; - } - - for (UINT k = 0; k < _n_in; ++k) - diff[k] = m1[k] - m2[k]; - - RVEC w; - bool not_reg = true; - while (!ldivide(A, diff, w)) { - assert(not_reg); not_reg = false; // should only happen once - REAL tr = 0; - for (UINT j = 0; j < _n_in; ++j) - tr += A[j][j]; - const REAL gamma = 1e-10; // see [Friedman 1989] - const REAL adj = gamma/(1-gamma) * tr/_n_in; - for (UINT j = 0; j < _n_in; ++j) - A[j][j] += adj; -#if VERBOSE_OUTPUT - std::cerr << "LDA: The class covariance matrix estimate is " - "regularized by\n\tan eigenvalue shinkage parameter " - << gamma << '\n'; -#endif - } - - w.push_back(- (DOTPROD(m1,w) * w2 + DOTPROD(m2,w) * w1)); - return w; -} - -inline UINT randcdf (REAL r, const RVEC& cdf) { - UINT b = 0, e = cdf.size()-1; - while (b+1 < e) { - UINT m = (b + e) / 2; - if (r < cdf[m]) e = m; - else b = m; - } - return b; -} - -void Perceptron::train () { - assert(ptd != 0 && ptw != 0 && ptd->size() == n_samples); - set_dimensions(*ptd); - const UINT dim = _n_in+1; - // but what is updated might be of a smaller size - const UINT udim = _n_in + (fixed_bias? 0:1); - - RVEC cdf; - if (resample) { - cdf.resize(n_samples+1); - cdf[0] = 0; - for (UINT i = 0; i < n_samples; ++i) - cdf[i+1] = cdf[i] + (*ptw)[i]; - assert(cdf[n_samples]-1 > -EPSILON && cdf[n_samples]-1 < EPSILON); - } - - wgt.resize(dim); - const REAL bias_save = wgt.back(); - if (with_fld) wgt = fld(); - if (fixed_bias) - wgt.back() = bias_save; - - RMAT X; RVEC Y; - dset_extract(ptd, X, Y); - -#define RAND_IDX() (resample? randcdf(randu(),cdf) : UINT(randu()*n_samples)) -#define SAMPWGT(i) (resample? 1 : (*ptw)[i]*n_samples) -#define GET_XYO(i) \ - const Input& x = X[i]; \ - const REAL y = Y[i]; \ - const REAL o = DOTPROD(wgt,x) - - log_error(0); - switch (train_method) { - case PERCEPTRON: - case ADALINE: - for (UINT i = 0; i < max_run; ++i) { - for (UINT j = 0; j < n_samples; ++j) { - const UINT idx = RAND_IDX(); - GET_XYO(idx); - if (y * o > 0) continue; - REAL deriv = (train_method == PERCEPTRON? y : (y - o)); - REAL adj = learn_rate * SAMPWGT(idx) * deriv; - - for (UINT k = 0; k < udim; ++k) - wgt[k] += adj * x[k]; - } - log_error(i+1); - } - break; - - case POCKET_RATCHET: - case POCKET: { - bool ratchet = (train_method == POCKET_RATCHET); - RVEC best_w(wgt); - REAL run = 0, err = train_c_error(); - bool err_valid = true; - REAL best_run = run, best_err = err; - for (UINT i = 0; i < max_run; ++i) { - for (UINT j = 0; j < n_samples; ++j) { - const UINT idx = RAND_IDX(); - GET_XYO(idx); - - if (y * o > 0) { - run += SAMPWGT(idx); - if (run > best_run) { - if (!err_valid) err = train_c_error(); - err_valid = true; - if (!ratchet || err < best_err) { - best_run = run; - best_err = err; - best_w = wgt; - } - if (err <= 0) break; - } - } else { - run = 0; - err_valid = false; - - const REAL adj = SAMPWGT(idx) * y; - for (UINT k = 0; k < udim; ++k) - wgt[k] += adj * x[k]; - } - } - wgt.swap(best_w); log_error(i+1, best_err); wgt.swap(best_w); - } - wgt.swap(best_w); - } - break; - - case AVE_PERCEPTRON_RAND: - case AVE_PERCEPTRON: { - assert(train_method != AVE_PERCEPTRON || !resample); - RVEC ave_wgt(dim, 0); - REAL run = 0; - for (UINT i = 0; i < max_run; ++i) { - for (UINT j = 0; j < n_samples; ++j) { - const UINT idx = (train_method == AVE_PERCEPTRON)? - j : RAND_IDX(); - GET_XYO(idx); - if (y * o > 0) - run += SAMPWGT(idx); - else { - for (UINT k = 0; k < dim; ++k) - ave_wgt[k] += run * wgt[k]; - - const REAL adj = SAMPWGT(idx) * y; - for (UINT k = 0; k < udim; ++k) - wgt[k] += adj * x[k]; - run = SAMPWGT(idx); - } - } - RVEC tmp_wgt(ave_wgt); - for (UINT k = 0; k < dim; ++k) - tmp_wgt[k] += run * wgt[k]; - wgt.swap(tmp_wgt); log_error(i+1); wgt.swap(tmp_wgt); - } - for (UINT k = 0; k < dim; ++k) - wgt[k] = ave_wgt[k] + run * wgt[k]; - } - break; - - case ROMMA_AGG_RAND: - case ROMMA_AGG: - case ROMMA_RAND: - case ROMMA: { - bool fixed = (train_method == ROMMA || train_method == ROMMA_AGG); - assert(!fixed || !resample); - REAL bnd = (train_method == ROMMA || train_method == ROMMA_RAND)? - 0 : (1-EPSILON); - for (UINT i = 0; i < max_run; ++i) { - for (UINT j = 0; j < n_samples; ++j) { - const UINT idx = fixed? j : RAND_IDX(); - GET_XYO(idx); const REAL& w_x = o; - if (y * w_x > bnd) continue; - - REAL w_w = DOTPROD(wgt, wgt); - REAL x_x = 1 + DOTPROD(x, x); - REAL x2w2 = x_x * w_w; - REAL deno = x2w2 - w_x*w_x; - REAL c = (x2w2 - y*w_x) / deno; - REAL d = w_w * (y - w_x) / deno; - - wgt[0] = c*wgt[0] + d; - for (UINT k = 0; k < _n_in; ++k) - wgt[k+1] = c*wgt[k+1] + d*x[k]; - } - log_error(i+1); - } - } - break; - - case SGD_HINGE: - case SGD_MLSE: { - const REAL C = 0; // C is lambda - - for (UINT i = 0; i < max_run; ++i) { - for (UINT j = 0; j < n_samples; ++j) { - const UINT idx = RAND_IDX(); - GET_XYO(idx); - - if (y*o < 1) { - REAL shrink = 1 - C * learn_rate; - REAL deriv = (train_method == SGD_HINGE? y : (y - o)); - REAL adj = learn_rate * SAMPWGT(idx) * deriv; - for (UINT k = 0; k < udim; ++k) - wgt[k] = shrink * wgt[k] + adj * x[k]; - } - } - log_error(i+1); - } - } - break; - -#undef RAND_IDX -#undef SAMPWGT -#define CYCLE(r) (((r)+dim-1) % udim) -#define UPDATE_WGT(d) update_wgt(wgt, d, X, Y) -#define INIT_RCD() { \ - dset_mult_wgt(ptw, Y); \ -} - - case COORDINATE_DESCENT: { - INIT_RCD(); - for (UINT r = 0; r < max_run; ++r) { - UPDATE_WGT(coorvec(dim, CYCLE(r))); - log_error(r+1); - } - } - break; - - case FIXED_RCD: - case FIXED_RCD_CONJ: - case FIXED_RCD_BIAS: - case FIXED_RCD_CONJ_BIAS: { - bool bias_row = (train_method == FIXED_RCD_BIAS || - train_method == FIXED_RCD_CONJ_BIAS); - bool conjugate = (train_method == FIXED_RCD_CONJ || - train_method == FIXED_RCD_CONJ_BIAS); - RMAT A = randrot(dim, bias_row || fixed_bias, conjugate); - - INIT_RCD(); - for (UINT r = 0; r < max_run; ++r) { - UPDATE_WGT(A[CYCLE(r)]); - log_error(r+1); - } - } - break; - - case RCD: - case RCD_CONJ: - case RCD_BIAS: - case RCD_CONJ_BIAS: { - bool bias_row = (train_method == RCD_BIAS || - train_method == RCD_CONJ_BIAS); - bool conjugate = (train_method == RCD_CONJ || - train_method == RCD_CONJ_BIAS); - RMAT A; - - INIT_RCD(); - for (UINT r = 0; r < max_run; ++r) { - const UINT c = CYCLE(r); - if (c == CYCLE(0)) - A = randrot(dim, bias_row || fixed_bias, conjugate); - UPDATE_WGT(A[c]); - log_error(r+1); - } - } - break; - - case RCD_GRAD_BATCH_RAND: - case RCD_GRAD_BATCH: - case RCD_GRAD_RAND: - case RCD_GRAD: { - bool online = (train_method == RCD_GRAD || - train_method == RCD_GRAD_RAND); - bool wrand = (train_method == RCD_GRAD_RAND || - train_method == RCD_GRAD_BATCH_RAND); - // gradient of sum weight*y* over all unsatisfied examples - INIT_RCD(); - for (UINT r = 0; r < max_run; ++r) { - RVEC dir(dim, 0); - if (r % 5 == 0 && wrand) { - dir = randvec(dim); - } else if (online) { - UINT idx, cnt = 0; - REAL o; - do { - ++cnt; - idx = UINT(randu() * n_samples); - o = DOTPROD(wgt, X[idx]); - } while (Y[idx] * o > 0 && cnt < 2*n_samples); - // if we've tried too many times, just use any X - dir = X[idx]; - } else { - bool no_err = true; - for (UINT j = 0; j < n_samples; ++j) { - GET_XYO(j); - if (y * o > 0) continue; - no_err = false; - for (UINT k = 0; k < udim; ++k) - dir[k] += y * x[k]; - } - if (no_err) break; - } - - if (fixed_bias) - dir.back() = 0; - UPDATE_WGT(dir); - log_error(r+1); - } - } - break; - - case RCD_GRAD_MIXED_BATCH_INITRAND: - case RCD_GRAD_MIXED_BATCH: - case RCD_GRAD_MIXED_INITRAND: - case RCD_GRAD_MIXED: { - bool online = (train_method == RCD_GRAD_MIXED || - train_method == RCD_GRAD_MIXED_INITRAND); - bool init_rand = (train_method == RCD_GRAD_MIXED_INITRAND || - train_method == RCD_GRAD_MIXED_BATCH_INITRAND); - - INIT_RCD(); - for (UINT r = 0; r < max_run; ++r) { - RVEC dir(dim, 0); - if (init_rand) - dir = randvec(dim); - UINT cnt = 0; - - for (UINT j = 0; j < n_samples; ++j) { - UINT idx = (online? UINT(randu() * n_samples) : j); - GET_XYO(idx); - if (y * o > 0) continue; - ++cnt; - REAL adj = y*n_samples * randu(); - for (UINT k = 0; k < udim; ++k) - dir[k] += adj * x[k]; - } - //if (cnt == 0 && !online) break; - - if (cnt == 0 && !init_rand) - dir = randvec(dim); - if (fixed_bias) - dir.back() = 0; - UPDATE_WGT(dir); - log_error(r+1); - } - } - break; - - case RCD_MIXED: { - INIT_RCD(); - RMAT A; - for (UINT r = 0; r < max_run; ++r) { - UINT c = r % (2*udim); - RVEC dir; - if (c < udim) - dir = coorvec(dim, CYCLE(c)); - else { - if (c == udim) A = randrot(dim, fixed_bias, false); - dir = A[c-udim]; // CYCLE doesn't change anything - } - UPDATE_WGT(dir); - log_error(r+1); - } - } - break; - - default: - assert(false); - } - - assert(!fixed_bias || train_method == AVE_PERCEPTRON_RAND || - train_method == AVE_PERCEPTRON || wgt.back() == bias_save); -} - -#define INPUT_SUM(w,x) \ - std::inner_product(x.begin(), x.end(), w.begin(), w.back()) - -Output Perceptron::operator() (const Input& x) const { - assert(x.size() == n_input()); - REAL sum = INPUT_SUM(wgt, x); - return Output(1, (sum>=0)? 1 : -1); -} - -REAL Perceptron::margin_of (const Input& x, const Output& y) const { - assert(std::fabs(std::fabs(y[0]) - 1) < INFINITESIMAL); - return INPUT_SUM(wgt, x) * y[0]; -} - -REAL Perceptron::w_norm () const { - REAL s = DOTPROD_NB(wgt, wgt); - return std::sqrt(s); -} - -void Perceptron::log_error (UINT, REAL err) const { - if (logf != NULL) { - if (err < 0) err = train_c_error(); - fprintf(logf, "%g ", err); - } -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.h deleted file mode 100644 index a77d5c5..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/perceptron.h +++ /dev/null @@ -1,136 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_PERCEPTRON_H__ -#define __LEMGA_PERCEPTRON_H__ - -/** @file - * @brief @link lemga::Perceptron Perceptron@endlink class. - * - * $Id: perceptron.h 2891 2006-11-08 03:17:31Z ling $ - */ - -#include -#include "learnmodel.h" -#include "svm.h" - -namespace lemga { - -/** @brief %Perceptron models a type of artificial neural network that - * consists of only one neuron, invented by Frank Rosenblatt in 1957. - * - * We use the convention that @f$w_0@f$ is the negative threshold, - * or equivalently, letting @f$x_0=1@f$. When presented with input - * @f$x@f$, the perceptron outputs - * @f[ o = \theta(s),\quad s=\sum_i w_ix_i, @f] - * where @f$\theta(\cdot)@f$ is usually the sign function, - * - * The learning algorithm updates the weight according to - * @f[ w^{(t+1)} = w^{(t)} + \eta (y-o) x, @f] - * where @f$y@f$ is the desired output. If @f$\theta@f$ is the sign, - * the learning rate @f$\eta@f$ can be omitted since it just scales - * @f$w@f$. - * - * @note If @f$o@f$ is replaced by @f$s@f$, the sum before the function - * @f$\theta@f$, the algorithm is then called the ADALINE learning. - */ -class Perceptron : public LearnModel { -public: - typedef std::vector WEIGHT; - enum TRAIN_METHOD { - // These are known algorithms - PERCEPTRON, // Rosenblatt's learning rule - ADALINE, - POCKET, // Gallant's pocket algorithm - POCKET_RATCHET, - AVE_PERCEPTRON, // Freund's average-perceptron - ROMMA, - ROMMA_AGG, - SGD_HINGE, // Zhang's stochastic gradient descent - SGD_MLSE, // on SVM hinge loss or modified least square - // These are the recommended ones of my algorithms - RCD, - RCD_BIAS, - RCD_GRAD, - // Below are just for research comparison - AVE_PERCEPTRON_RAND, - ROMMA_RAND, - ROMMA_AGG_RAND, - COORDINATE_DESCENT, - FIXED_RCD, - FIXED_RCD_CONJ, - FIXED_RCD_BIAS, - FIXED_RCD_CONJ_BIAS, - RCD_CONJ, - RCD_CONJ_BIAS, - RCD_GRAD_BATCH, - RCD_GRAD_RAND, - RCD_GRAD_BATCH_RAND, - RCD_MIXED, - RCD_GRAD_MIXED, - RCD_GRAD_MIXED_INITRAND, - RCD_GRAD_MIXED_BATCH, - RCD_GRAD_MIXED_BATCH_INITRAND, - // Old definitions - RAND_COOR_DESCENT = RCD, - RAND_COOR_DESCENT_BIAS = RCD_BIAS, - RAND_CONJ_DESCENT = RCD_CONJ, - RAND_CONJ_DESCENT_BIAS = RCD_CONJ_BIAS, - GRADIENT_COOR_DESCENT_ONLINE = RCD_GRAD - }; - -protected: - WEIGHT wgt; ///< wgt.back() is the bias - // only for online-type algorithms; RCD always uses reweighting - bool resample; ///< reweighting or resampling - TRAIN_METHOD train_method; - REAL learn_rate, min_cst; - UINT max_run; - bool with_fld; ///< start training with FLD? - bool fixed_bias; ///< using a fixed bias? - -public: - explicit Perceptron (UINT n_in = 0); - Perceptron (const SVM&); - explicit Perceptron (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Perceptron* create () const { return new Perceptron(); } - virtual Perceptron* clone () const { return new Perceptron(*this); } - - WEIGHT weight () const { return wgt; } - void set_weight (const WEIGHT&); - - void start_with_fld (bool b = true) { with_fld = b; } - void set_fixed_bias (bool b = false) { fixed_bias = b; } - void use_resample (bool s = true) { resample = true; } - void set_train_method (TRAIN_METHOD m) { train_method = m; } - /** @param lr learning rate. - * @param mincst minimal cost (error) need to be achieved during - * training. - * @param maxrun maximal # of epochs the training should take. - */ - void set_parameter (REAL lr, REAL mincst, UINT maxrun) { - learn_rate = lr; min_cst = mincst; max_run = maxrun; } - - virtual bool support_weighted_data () const { return true; } - virtual void initialize (); - WEIGHT fld () const; - virtual void train (); - virtual Output operator() (const Input&) const; - - virtual REAL margin_norm () const { return w_norm(); } - virtual REAL margin_of (const Input&, const Output&) const; - REAL w_norm () const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); - virtual void log_error (UINT, REAL = -1) const; -}; - -} // namespace lemga - -#ifdef __PERCEPTRON_H__ -#warning "This header file may conflict with another `perceptron.h' file." -#endif -#define __PERCEPTRON_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.cpp deleted file mode 100644 index e3e0587..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.cpp +++ /dev/null @@ -1,184 +0,0 @@ -/** @file - * $Id: pulse.cpp 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include -#include -#include -#include -#include "pulse.h" - -REGISTER_CREATOR(lemga::Pulse); - -template -bool serialize (std::ostream& os, const II& b, const II& e, bool l = true) { - if (l) if (!(os << (e - b) << '\n')) return false; - for (II i = b; i != e; ++i) - if (!(os << *i << ' ')) return false; - if (b != e) os << '\n'; - return true; -} - -template -bool unserialize (std::istream& is, const II& b, const II& e) { - for (II i = b; i != e; ++i) - if (!(is >> *i)) return false; - return true; -} - -namespace lemga { - -bool Pulse::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - if (!(os << idx << ' ' << th.size() << ' ' << (dir? '-':'+') << '\n')) - return false; - return ::serialize(os, th.begin(), th.end(), false); -} - -bool Pulse::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - assert(v > 0); - - UINT nl; - if (!(is >> idx >> nl)) return false; - char c; - if (!(is >> c) || (c != '-' && c != '+')) return false; - dir = (c == '-'); - - th.resize(nl); - return ::unserialize(is, th.begin(), th.end()); -} - -void Pulse::set_threshold (const std::vector& t) { - assert(t.size() <= max_l); -#ifndef NDEBUG - // assert t is sorted (std::is_sorted is an SGL extension) - for (UINT i = 1; i < t.size(); ++i) - assert(t[i-1] <= t[i]); -#endif - th = t; -} - -void Pulse::train () { - const UINT N = n_samples; - assert(ptd != 0 && ptw != 0 && ptd->size() == N); - set_dimensions(*ptd); - - std::vector yw(N); - for (UINT i = 0; i < N; ++i) - yw[i] = ptd->y(i)[0] * (*ptw)[i]; - - REAL minerr = 2; // a number large enough (> 1) - std::vector thi; // threshold index - std::vector xb(N); // backup for sorted x - - // no reallocation within loops - std::vector x(N), ysum(N); - for (UINT d = 0; d < _n_in; ++d) { - // extract the dimension d info, collapse data with same x - std::map xy; - for (UINT i = 0; i < N; ++i) - xy[ptd->x(i)[d]] += yw[i]; - REAL sum = 0; - int last_sign = 0; // 1: pos, 2: neg, 3: zero - std::vector::iterator px = x.begin(), py = ysum.begin(); - for (std::map::const_iterator - p = xy.begin(); p != xy.end(); ++p) { - static REAL last_x; - const int cur_sign = (p->second > 0)? 1:((p->second < 0)? 2:3); - if (last_sign != cur_sign && last_sign != 0) { - /** @note we can also save the threshold info. as lower - * and upper @a x values, which is a bit faster (less - * operations) but needs more memory space. */ - *px = last_x + p->first; *py = sum * 2; - ++px; ++py; - } - last_sign = cur_sign; - last_x = p->first; - sum += p->second; - assert(-1.01 < sum && sum < 1.01); - } - *py = sum * 2; - const UINT n = py - ysum.begin(); - - std::vector e0(n+1, 0); // error of pulses ending with -1 - std::vector e1(n+1, 0); // error of pulses ending with +1 - std::vector > t0(n+1), t1(n+1); // transitions idx - - // dynamic programming: compute err for level 1--max_l - // e0 and e1 at the begining of loop l are - // e0[i] = 2*best_e_{i,l} - 1 - sum(w*y), - // e1[i] = 2*best_e_{i,l} - 1 + sum(w*y). - // where best_e_{i,l} is the lowest error if l transitions - // happens before or at position i. - for (UINT l = 0; l < max_l; ++l) { - // swap e0 & e1, t0 & t1 to get rid of the sign change - e0.swap(e1); t0.swap(t1); - - // compute errors for level (l+1) - std::vector::iterator p0 = e0.begin(), p1 = e1.begin(); - std::vector::iterator ps = ysum.begin(); - for (UINT i = 0; i <= n; ++p0, ++p1, ++ps, ++i) { - *p0 -= *ps; *p1 += *ps; - } - assert(p0 == e0.end()); - - std::vector >::iterator - pt0 = t0.begin(), pt1 = t1.begin(); - REAL bst0 = 3, bst1 = 3; // a number large enough (> 2) - p0 = e0.begin(); p1 = e1.begin(); - for (UINT i = 0; i <= n; ++p0, ++p1, ++pt0, ++pt1, ++i) { - static std::vector tb0, tb1; // always the best - assert(-2.01 < *p0 && *p0 < 2.01); - assert(-2.01 < *p1 && *p1 < 2.01); - - if (*p0 < bst0) { - bst0 = *p0; tb0.swap(*pt0); // => tb0 = *pt0; - if (i < n) tb0.push_back(i); - } - *p0 = bst0; *pt0 = tb0; - - if (*p1 < bst1) { - bst1 = *p1; tb1.swap(*pt1); // => tb1 = *pt1; - if (i < n) tb1.push_back(i); - } - *p1 = bst1; *pt1 = tb1; - } - assert(p0 == e0.end()); - } - - e0[n] += ysum[n] / 2; - e1[n] -= ysum[n] / 2; - if (e0[n] <= e1[n] && e0[n] < minerr) { - minerr = e0[n]; idx = d; dir = !(max_l & 1); - thi.swap(t0[n]); xb.swap(x); - } else if (e1[n] < minerr) { - minerr = e1[n]; idx = d; dir = (max_l & 1); - thi.swap(t1[n]); xb.swap(x); - } - } - - th.clear(); - for (UINT i = 0; i < thi.size(); /* empty */) { - UINT ind = thi[i]; ++i; - if (i < thi.size() && ind == thi[i]) - ++i; - else - th.push_back(xb[ind] / 2); - } -} - -Output Pulse::operator() (const Input& x) const { - assert(idx < n_input() && x.size() == n_input()); - - if (th.empty()) - return Output(1, dir? -1 : 1); - - const UINT i = - std::lower_bound(th.begin(), th.end(), x[idx]) - th.begin(); - return Output(1, ((i & 1) ^ dir)? -1 : 1); -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.h deleted file mode 100644 index af70116..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/pulse.h +++ /dev/null @@ -1,68 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_PULSE_H__ -#define __LEMGA_PULSE_H__ - -/** @file - * @brief @link lemga::Pulse Pulse function@endlink class - * - * $Id: pulse.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include -#include "learnmodel.h" - -namespace lemga { - -/** @brief Multi-transition pulse functions (step functions). - * @todo Documentation - */ -class Pulse : public LearnModel { - UINT idx; ///< which dimenstion? - std::vector th; ///< transition points - bool dir; ///< @c true: start with -1 - UINT max_l; ///< Maximal # of transitions - -public: - explicit Pulse (UINT n_in = 0) : LearnModel(n_in, 1), idx(0), max_l(1) {} - explicit Pulse (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Pulse* create () const { return new Pulse(); } - virtual Pulse* clone () const { return new Pulse(*this); } - - /** @return The dimension picked for classification. */ - UINT index () const { return idx; } - /// Set the dimension picked for classification. - void set_index (UINT i) { assert(i < n_input()); idx = i; } - /** @return a vector of transition points. */ - const std::vector& threshold () const { return th; } - /// Set the transition points. - void set_threshold (const std::vector&); - /** @return @c true if the pulse starts with -1. */ - bool direction () const { return dir; } - /// Set the pulse direction (true if starting with -1). - void set_direction (bool d) { dir = d; } - /** @return the maximal number of transitions the pulse can have. */ - UINT max_transitions () { return max_l; } - /// Set the maximal number of transitions the pulse can have. - void set_max_transitions (UINT ml) { max_l = ml; } - // TODO: resize th? - - virtual bool support_weighted_data () const { return true; } - - virtual void train (); - - virtual Output operator() (const Input&) const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __PULSE_H__ -#warning "This header file may conflict with another `pulse.h' file." -#endif -#define __PULSE_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.c b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.c deleted file mode 100644 index f9542de..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.c +++ /dev/null @@ -1,50 +0,0 @@ -/** @file - * $Id: quickfun.c 1789 2004-04-23 05:15:44Z ling $ - */ - -#include -#include -#include "quickfun.h" - -#define round(x) ((UINT)(x+0.5)) - -#define TANH_RANGE0 1.84 /* tanh(1.84) = 0.95079514... */ -#define TANH_RANGE1 4.5 /* tanh(4.5) = 0.99975321... */ -#define TANH_STEP0 0.001 -#define TANH_STEP1 0.005 - -#define TANH_FACTOR0 (1/TANH_STEP0) -#define TANH_FACTOR1 (1/TANH_STEP1) -#define TANH_SIZE0 (round(TANH_FACTOR0*TANH_RANGE0)+1) -#define TANH_SIZE1 (round(TANH_FACTOR1*(TANH_RANGE1-TANH_RANGE0))+1) - -/* note: C doesn't have boolean type */ -static int tanh_table_ready = 0; -static REAL tanh_table0[TANH_SIZE0], tanh_table1[TANH_SIZE1]; - -void quick_tanh_setup (void) { - UINT i; - if (tanh_table_ready) return; - - for (i = 0; i < TANH_SIZE0; ++i) - tanh_table0[i] = tanh(i * TANH_STEP0); - for (i = 0; i < TANH_SIZE1; ++i) - tanh_table1[i] = tanh(i * TANH_STEP1 + TANH_RANGE0); - tanh_table_ready = 1; -} - -static REAL quick_tanh_p (REAL x) { - if (x <= TANH_RANGE0) - return tanh_table0[round(x * TANH_FACTOR0)]; - else if (x <= TANH_RANGE1) - return tanh_table1[round((x - TANH_RANGE0) * TANH_FACTOR1)]; - return 0.9998; -} - -REAL quick_tanh (REAL x) { - assert(tanh_table_ready); - - if (x < 0) - return -quick_tanh_p(-x); - return quick_tanh_p(x); -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.h deleted file mode 100644 index 425f729..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/quickfun.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef __QUICKFUN_H__ -#define __QUICKFUN_H__ -#define __COMMON_TYPES_QUICKFUN_H__ - -#include "object.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void quick_tanh_setup (void); -REAL quick_tanh (REAL x); - -#ifdef __cplusplus -} -#endif - -#else /* def __QUICKFUN_H__ */ - -#ifndef __COMMON_TYPES_QUICKFUN_H__ - #error This header file conflicts with another "quickfun.h" file. -#endif - -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.c b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.c deleted file mode 100644 index 1187b7e..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.c +++ /dev/null @@ -1,40 +0,0 @@ -#include -#include -#include -#include -#include -#include "random.h" - -void set_seed (unsigned int seed) -{ - if (seed == 0 || seed > RAND_MAX) { - /* use time() to set the seed */ - time_t t = time(NULL); - assert(t > 0); - seed = t; - } - fprintf(stderr, "random seed = %u\n", seed); - srand(seed); -} - -REAL randn () { - static int saved = 0; - static REAL saved_val; - REAL rsq, v1, v2, ratio; - - if (saved) { - saved = 0; - return saved_val; - } - - do { - v1 = 2*randu()-1; - v2 = 2*randu()-1; - rsq = v1*v1 + v2*v2; - } while (rsq >= 1 || rsq == 0); - ratio = sqrt(-2 * log(rsq)/rsq); - - saved_val = v1 * ratio; - saved = 1; - return v2 * ratio; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.h deleted file mode 100644 index 85230f7..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/random.h +++ /dev/null @@ -1,36 +0,0 @@ -/* random.h -- Define several random distributions - * - * Copyright (C) 2001 Ling Li - * ling@caltech.edu - */ - -#ifndef __RANDOM_H__ -#define __RANDOM_H__ -#define __COMMON_TYPES_RANDOM_H__ - -#include -#include "object.h" - -typedef REAL PROBAB; - -#ifdef __cplusplus -extern "C" { -#endif - -void set_seed (unsigned int seed); -/* randu: [0, 1); randuc: [0, 1] */ -#define randu() (rand() / (RAND_MAX + 1.0)) -#define randuc() (rand() / (PROBAB) RAND_MAX) -REAL randn (); - -#ifdef __cplusplus -} -#endif - -#else /* __RANDOM_H__ */ - -#ifndef __COMMON_TYPES_RANDOM_H__ - #error This header file conflicts with another "random.h" file. -#endif - -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/shared_ptr.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/shared_ptr.h deleted file mode 100644 index 5583f15..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/shared_ptr.h +++ /dev/null @@ -1,103 +0,0 @@ -// -*- C++ -*- -#ifndef __CPP_ADDON_SHARED_PTR_H__ -#define __CPP_ADDON_SHARED_PTR_H__ - -/** @file - * @brief Shared pointers with reference count. - * - * $Id: shared_ptr.h 1907 2004-12-11 00:51:14Z ling $ - */ - -/** @warning Do not use this class. Use const_shared_ptr or var_shared_ptr. - * @note I guess it is not thread-safe. - * @note See boost.org for a better and more complicated implementation. - */ -template -class _shared_ptr { -protected: - T* ptr; - UINT* pcnt; - - void delete_this () { - if (valid() && !(--*pcnt)) { - delete ptr; delete pcnt; - ptr = 0; pcnt = 0; - } - } - - /// @c false if the pointer is null. - bool valid () const { - assert((!ptr && !pcnt) || (ptr && pcnt && *pcnt > 0)); - return (ptr != 0); - } - - UINT use_count () const { assert(valid()); return *pcnt; } - bool unique () const { return (use_count() == 1); } - -public: - explicit _shared_ptr (T* p = 0) : ptr(p), pcnt(0) { - if (p) { pcnt = new UINT(1); assert(valid()); } } - _shared_ptr (const _shared_ptr& s) : ptr(s.ptr), pcnt(s.pcnt) { - if (valid()) ++(*pcnt); } - ~_shared_ptr () { delete_this(); } - - const _shared_ptr& operator= (const _shared_ptr& s) { - if (this == &s) return *this; - delete_this(); - ptr = s.ptr; pcnt = s.pcnt; - if (valid()) ++(*pcnt); - return *this; - } - bool operator!= (const T* p) const { return ptr != p; } - bool operator== (const T* p) const { return ptr == p; } - bool operator!= (const _shared_ptr& p) const { return ptr != p.ptr; } - bool operator== (const _shared_ptr& p) const { return ptr == p.ptr; } - bool operator! () const { return !ptr; } - - typedef bool (_shared_ptr::*implicit_bool_type) () const; - operator implicit_bool_type () const { - return ptr? &_shared_ptr::valid : 0; - } -}; - -/// Shared pointers (whose content can not be changed). -template -class const_shared_ptr : public _shared_ptr { - using _shared_ptr::valid; - using _shared_ptr::ptr; -public: - const_shared_ptr () {} - const_shared_ptr (T* p) : _shared_ptr(p) {} - const_shared_ptr (const const_shared_ptr& s) : _shared_ptr(s) {} - - const const_shared_ptr& operator= (const const_shared_ptr& s) { - _shared_ptr::operator=(s); - return *this; - } - const T* operator-> () const { assert(valid()); return ptr; } - const T& operator* () const { assert(valid()); return *ptr; } -}; - -/// Shared pointers (whose content can be changed). -template -class var_shared_ptr : public _shared_ptr { - using _shared_ptr::valid; - using _shared_ptr::ptr; -public: - var_shared_ptr () {} - var_shared_ptr (T* p) : _shared_ptr(p) {} - var_shared_ptr (const var_shared_ptr& s) : _shared_ptr(s) {} - - const var_shared_ptr& operator= (const var_shared_ptr& s) { - _shared_ptr::operator=(s); - return *this; - } - T* operator-> () const { assert(valid()); return ptr; } - T& operator* () const { assert(valid()); return *ptr; } -}; - -#ifdef __SHARED_PTR_H__ -#warning "This header file may conflict with another `shared_ptr.h' file." -#endif -#define __SHARED_PTR_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.cpp deleted file mode 100644 index 95b0c0f..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.cpp +++ /dev/null @@ -1,207 +0,0 @@ -/** @file - * $Id: stump.cpp 2891 2006-11-08 03:17:31Z ling $ - */ - -#include -#include -#include -#include -#include -#include "stump.h" - -REGISTER_CREATOR(lemga::Stump); - -namespace lemga { - -bool Stump::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 2); - return (os << idx << ' ' << bd1 << ' ' << bd2 << ' ' - << (dir? 'P':'N') << '\n'); -} - -bool Stump::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 2, v); - - if (v == 0) { /* Take care of _n_in and _n_out */ - if (!(is >> _n_in)) return false; - assert(_n_out == 1); - } - /* common part for ver 0, 1, and 2 */ - if (!(is >> idx >> bd1)) return false; - bd2 = bd1; - if (v >= 2) - if (!(is >> bd2) || bd1 > bd2) return false; - char c; - if (!(is >> c) || (c != 'P' && c != 'N')) return false; - dir = (c == 'P'); - return true; -} - -typedef std::map::iterator MI; - -REAL Stump::train_1d (const std::vector& x, const std::vector& yw) -{ - const UINT N = x.size(); - assert(yw.size() == N); - - // combine examples with same input, and sort them - std::map xy; - for (UINT i = 0; i < N; ++i) - xy[x[i]] += yw[i]; - UINT n = xy.size(); - xy[xy.begin()->first-2] = 1; // insert a "very small" x - assert(xy.size() == n+1); - for (MI p, pn = xy.begin(); p = pn++, p != xy.end();) - if (p->second > -INFINITESIMAL && p->second < INFINITESIMAL) - xy.erase(p); - - n = xy.size(); - xy[xy.rbegin()->first+2] = 1; // a "very large" x - assert(n > 0 && xy.size() == n+1); - const MI xyb = xy.begin(); - - REAL minthr = 0, mine = INFINITY, e = - xyb->second; - REAL cur_x = xyb->first; - for (MI p = xyb; n; --n) { - e += p->second; - assert(p != xyb || e == 0); - REAL nxt_x = (++p)->first; - REAL cur_thr = (cur_x + nxt_x) / 2; - cur_x = nxt_x; - - if (e < mine) { - mine = e; minthr = cur_thr; - } - } - - return minthr; -} - -/* yw_inf is yw[infinity] */ -REAL Stump::train_1d (const std::vector& x, const std::vector& yw, - REAL yw_inf, bool& ir, bool& mind, REAL& th1, REAL& th2) -{ - const UINT N = x.size(); - assert(yw.size() == N); - - // combine examples with same input, and sort them - std::map xy; - for (UINT i = 0; i < N; ++i) - xy[x[i]] += yw[i]; -#ifndef NDEBUG - UINT n = xy.size(); -#endif - xy[xy.begin()->first-2] = 1; // insert a "very small" x - assert(xy.size() == n+1); - for (MI p, pn = xy.begin(); p = pn++, p != xy.end();) - if (p->second > -INFINITESIMAL && p->second < INFINITESIMAL) - xy.erase(p); - -#ifndef NDEBUG - // check whether a constant function is enough - bool all_the_same = true; - MI pb = xy.begin(); ++pb; - for (MI p = pb; p != xy.end(); ++p) - if (p->second < -INFINITESIMAL) { all_the_same = false; break; } - - if (!all_the_same) { - all_the_same = true; - for (MI p = pb; p != xy.end(); ++p) - if (p->second > INFINITESIMAL) { all_the_same = false; break; } - } - - if (all_the_same) - std::cerr << "Stump: Warning: all y's are the same.\n"; -#endif - const MI xyb = xy.begin(), xye = xy.end(); - - REAL mine = 0, maxe = 0, e = -1; - MI mint, maxt, p; - for (mint = maxt = p = xyb; p != xye; ++p) { - e += p->second; - assert(p != xyb || e == 0); - if (e < mine) { - mine = e; mint = p; - } else if (e > maxe) { - maxe = e; maxt = p; - } - // we prefer middle indices - if (mint == xyb && e == 0) mint = p; - if (maxt == xyb && e == 0) maxt = p; - } - e = (1-e-yw_inf) / 2;// error of y = sgn(x > -Inf) - mine += e; // starting with y = -1 - maxe = 1 - (maxe+e); // starting with y = 1 - - // unify the solution to mind, mine, and mint - if (std::fabs(mine - maxe) < EPSILON) { - MI nxtt = mint; ++nxtt; - mind = (mint != xyb && nxtt != xye); - } - else mind = (mine < maxe); - if (!mind) { - mine = maxe; mint = maxt; - } - - MI nxtt = mint; ++nxtt; - ir = (mint != xyb && nxtt != xye); - assert(!ir || !all_the_same); - th1 = mint->first; - if (ir) - th2 = nxtt->first; - else - th2 = th1 + 2; - - return mine; -} - -/* Find the optimal dimension and threshold */ -void Stump::train () { - const UINT N = n_samples; - assert(ptd != 0 && ptw != 0 && ptd->size() == N); - set_dimensions(*ptd); - - // weight the examples - std::vector yw(N); - for (UINT i = 0; i < N; ++i) - yw[i] = ptd->y(i)[0] * (*ptw)[i]; - - REAL minerr = 2; - bool minir = true; - - std::vector x(N); - for (UINT d = 0; d < _n_in; ++d) { - for (UINT i = 0; i < N; ++i) - x[i] = ptd->x(i)[d]; - - bool mind, ir; - REAL th1, th2; - REAL mine = train_1d(x, yw, 0, ir, mind, th1, th2); - - if (mine < minerr) { - minerr = mine; - minir = ir; - dir = mind; idx = d; - bd1 = th1; bd2 = th2; - } - } - - if (!minir) - std::cerr << "Stump: Warning: threshold out of range.\n"; -} - -Output Stump::operator() (const Input& x) const { - assert(idx < n_input() && x.size() == n_input()); - assert(bd2 >= bd1); - REAL y = x[idx]*2 - (bd1 + bd2); - if (hard || bd1 == bd2) - y = (y < 0)? -1 : 1; - else { - y /= bd2 - bd1; - y = (y<-1)? -1 : (y>1)? 1 : y; - } - return Output(1, dir? y : -y); -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.h deleted file mode 100644 index a7235b6..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/stump.h +++ /dev/null @@ -1,61 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_STUMP_H__ -#define __LEMGA_STUMP_H__ - -/** @file - * @brief @link lemga::Stump Decision stump@endlink class. - * - * $Id: stump.h 2891 2006-11-08 03:17:31Z ling $ - */ - -#include "learnmodel.h" - -namespace lemga { - -/** @brief Decision stump. - * @todo Documentation - */ -class Stump : public LearnModel { - UINT idx; - REAL bd1, bd2;///< threshold is (bd1+bd2)/2 - bool dir; ///< \c true: sgn(x[idx] - th); \c false: -sgn(x[idx] - th). - bool hard; ///< use the hard threshold? - -public: - explicit Stump (UINT n_in = 0) - : LearnModel(n_in, 1), idx(0), bd1(0), bd2(0), hard(true) {} - explicit Stump (std::istream& is) { is >> *this; } - - virtual const id_t& id () const; - virtual Stump* create () const { return new Stump(); } - virtual Stump* clone () const { return new Stump(*this); } - - UINT index () const { return idx; } - REAL threshold () const { return (bd1+bd2)/2; } - bool direction () const { return dir; } - bool soft_threshold () const { return !hard; } - void use_soft_threshold (bool s = true) { hard = !s; } - - virtual bool support_weighted_data () const { return true; } - - virtual void train (); - /// Find the optimal threshold and direction (prefer the middle thresholds) - static REAL train_1d (const std::vector&, const std::vector&, - REAL, bool&, bool&, REAL&, REAL&); - /// Find the optimal threshold for positive direction - static REAL train_1d (const std::vector&, const std::vector&); - - virtual Output operator() (const Input&) const; - -protected: - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __STUMP_H__ -#warning "This header file may conflict with another `stump.h' file." -#endif -#define __STUMP_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.cpp deleted file mode 100644 index 2e4575a..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.cpp +++ /dev/null @@ -1,428 +0,0 @@ -/** @file - * $Id: svm.cpp 2782 2006-05-15 19:25:35Z ling $ - */ - -#include -#include -#include "../../libsvm-2.81/svm.h" -#include "svm.h" - -REGISTER_CREATOR(lemga::SVM); - -// In order to access nSV, we have to copy svm_model here. -// Comments removed. Please see svm.cpp for details. -// Any direct use of svm_model is marked with /* direct access svm_model */ -struct svm_model { - svm_parameter param; - int nr_class; - int l; - svm_node **SV; - double **sv_coef; - double *rho; - double *probA; - double *probB; - int *label; - int *nSV; - int free_sv; -}; - -namespace lemga { - -typedef struct svm_node* p_svm_node; - -#ifndef NDEBUG -static bool param_equal (const struct svm_parameter& a, - const struct svm_parameter& b) { - struct svm_parameter a2 = a, b2 = b; - a2.C = b2.C = 0; - return !std::memcmp(&a2, &b2, sizeof(struct svm_parameter)); -} -#endif - -struct SVM_detail { - struct svm_parameter param; - struct svm_problem prob; - struct svm_model *model; - struct svm_node *x_space; - UINT n_class; - int *labels; - bool trained; // the wrapper was not loaded from file (ONLY FOR DEBUG) - - SVM_detail (); - ~SVM_detail () { - clean_model(); clean_data(); svm_destroy_param(¶m); } - bool fill_svm_problem (const pDataSet&, const pDataWgt&); - bool train (const pDataSet&, const pDataWgt&); - void clean_model (); - void clean_data (); -}; - -SVM_detail::SVM_detail () : model(0), x_space(0), trained(false) { - // default LIBSVM parameters, copied from svm-train.c - param.svm_type = C_SVC; - param.degree = 3; - param.gamma = 0; - param.coef0 = 0; - param.nu = 0.5; - param.cache_size = 128; - param.eps = 1e-3; - param.p = 0.1; - param.shrinking = 1; - param.probability = 0; - param.nr_weight = 0; - param.weight_label = NULL; - param.weight = NULL; -} - -void SVM_detail::clean_model () { - trained = false; - if (!model) return; - svm_destroy_model(model); - delete[] labels; - model = 0; -} - -void SVM_detail::clean_data () { - if (!x_space) return; - delete[] prob.x; delete[] prob.y; - delete[] prob.W; - delete[] x_space; x_space = 0; -} - -p_svm_node fill_svm_node (const Input& x, struct svm_node *pool) { - for (UINT j = 0; j < x.size(); ++j, ++pool) { - pool->index = j+1; - pool->value = x[j]; - } - pool->index = -1; - return ++pool; -} - -bool SVM_detail::fill_svm_problem (const pDataSet& ptd, const pDataWgt& ptw) { - assert(ptd->size() == ptw->size()); - const UINT n_samples = ptd->size(); - if (n_samples == 0 || ptd->y(0).size() != 1) return false; - const UINT n_in = ptd->x(0).size(); - - clean_data(); - prob.l = n_samples; - prob.x = new p_svm_node[n_samples]; - prob.y = new double[n_samples]; - prob.W = new double[n_samples]; - x_space = new struct svm_node[n_samples*(n_in+1)]; - if (!x_space) return false; - - struct svm_node *psn = x_space; - for (UINT i = 0; i < n_samples; ++i) { - prob.x[i] = psn; - prob.y[i] = ptd->y(i)[0]; - psn = fill_svm_node(ptd->x(i), psn); - prob.W[i] = (*ptw)[i] * n_samples; - } - return true; -} - -bool SVM_detail::train (const pDataSet& ptd, const pDataWgt& ptw) { - if (!fill_svm_problem(ptd, ptw)) { - std::cerr << "Error in filling SVM problem (training data)\n"; - return false; - } - const char* error_msg = svm_check_parameter(&prob,¶m); - if (error_msg) { - std::cerr << "Error: " << error_msg << '\n'; - return false; - } - - clean_model(); - model = svm_train(&prob, ¶m); - trained = true; - n_class = svm_get_nr_class(model); - labels = new int[n_class]; - svm_get_labels(model, labels); - return true; -} - -namespace kernel { - -void Linear::set_params (SVM_detail* sd) const { - sd->param.kernel_type = ::LINEAR; -} - -void Polynomial::set_params (SVM_detail* sd) const { - sd->param.kernel_type = ::POLY; - sd->param.degree = degree; - sd->param.gamma = gamma; - sd->param.coef0 = coef0; -} - -void RBF::set_params (SVM_detail* sd) const { - sd->param.kernel_type = ::RBF; - sd->param.gamma = gamma; -} - -void Sigmoid::set_params (SVM_detail* sd) const { - sd->param.kernel_type = ::SIGMOID; - sd->param.gamma = gamma; - sd->param.coef0 = coef0; -} - -void Stump::set_params (SVM_detail* sd) const { - sd->param.kernel_type = ::STUMP; -} - -void Perceptron::set_params (SVM_detail* sd) const { - sd->param.kernel_type = ::PERCEPTRON; -} - -} // namespace kernel - -bool SVM::serialize (std::ostream& os, ver_list& vl) const { - SERIALIZE_PARENT(LearnModel, os, vl, 1); - - // we will not save the detail - assert(sv.size() == coef.size()); - assert(ker != 0 || sv.size() == 0); - if (!(os << (ker != 0) << ' ' << regC << ' ' << sv.size() << '\n')) - return false; - // the kernel - if (ker != 0) - if (!(os << *ker)) return false; - // support vectors and coefficients - for (UINT i = 0; i < sv.size(); ++i) { - assert(sv[i].size() == _n_in); - for (UINT j = 0; j < _n_in; ++j) - if (!(os << sv[i][j] << ' ')) return false; - if (!(os << coef[i] << '\n')) return false; - } - // the bias - return (os << coef0 << '\n'); -} - -bool SVM::unserialize (std::istream& is, ver_list& vl, const id_t& d) { - if (d != id() && d != NIL_ID) return false; - UNSERIALIZE_PARENT(LearnModel, is, vl, 1, v); - assert(v > 0); - - assert(detail != 0); - reset_model(); - if (ker != 0) delete ker; ker = 0; - - UINT has_kernel, nsv; - if (!(is >> has_kernel >> regC >> nsv) || has_kernel > 1 || regC < 0) - return false; - if (!has_kernel && nsv > 0) return false; - - if (has_kernel) { - kernel::Kernel* k = (kernel::Kernel*) Object::create(is); - if (k == 0) return false; - set_kernel(*k); // don't directly assign the kernel - delete k; - } - sv.resize(nsv); coef.resize(nsv); - for (UINT i = 0; i < nsv; ++i) { - Input svi(_n_in); - for (UINT j = 0; j < _n_in; ++j) - if (!(is >> svi[j])) return false; - sv[i].swap(svi); - if (!(is >> coef[i])) return false; - } - return (is >> coef0); -} - -SVM::SVM (UINT n_in) : LearnModel(n_in, 1), ker(0), regC(1), coef0(0) { - detail = new struct SVM_detail; -} - -SVM::SVM (const kernel::Kernel& k, UINT n_in) - : LearnModel(n_in, 1), ker(0), regC(1), coef0(0) { - detail = new struct SVM_detail; - set_kernel(k); -} - -SVM::SVM (const SVM& s) : LearnModel(s), - ker(0), regC(s.regC), sv(s.sv), coef(s.coef), coef0(s.coef0) { - detail = new struct SVM_detail; - if (s.ker != 0) set_kernel(*s.ker); - assert(param_equal(detail->param, s.detail->param)); -} - -SVM::SVM (std::istream& is) : LearnModel(), ker(0) { - detail = new struct SVM_detail; - is >> *this; -} - -SVM::~SVM () { - delete detail; - if (ker != 0) delete ker; -} - -const SVM& SVM::operator= (const SVM& s) { - if (&s == this) return *this; - assert(ker != s.ker); - - delete detail; - if (ker != 0) delete ker; ker = 0; - - LearnModel::operator=(s); - regC = s.regC; - sv = s.sv; coef = s.coef; coef0 = s.coef0; - detail = new struct SVM_detail; - if (s.ker != 0) set_kernel(*s.ker); - assert(param_equal(detail->param, s.detail->param)); - return *this; -} - -REAL SVM::kernel (const Input& x1, const Input& x2) const { - assert(ker != 0); -#ifndef NDEBUG - assert(detail); - struct svm_node sx1[n_input()+1], sx2[n_input()+1]; - fill_svm_node(x1, sx1); - fill_svm_node(x2, sx2); - REAL svmk = svm_kernel(sx1, sx2, detail->param); -#endif - REAL k = (*ker)(x1, x2); - assert(std::fabs(svmk - k) < EPSILON); - return k; -} - -void SVM::set_kernel (const kernel::Kernel& k) { - if (&k == ker) return; - - if (ker != 0) { - delete ker; - reset_model(); // since kernel is also used for testing - } - ker = k.clone(); - ker->set_params(detail); -} - -void SVM::initialize () { - reset_model(); -} - -void SVM::reset_model () { - detail->clean_model(); detail->clean_data(); - sv.clear(); coef.clear(); coef0 = 0; -} - -void SVM::train () { - assert(ptd && n_samples == ptd->size()); - assert(ker != 0); - set_dimensions(*ptd); - - reset_model(); - detail->param.C = regC; - if (!detail->train(ptd, ptw)) exit(-1); - assert(detail && detail->model && detail->x_space); - assert(detail->n_class > 0 && detail->n_class <= 2); - - // copy the trained model to local - /* direct access svm_model */ - const UINT nsv = detail->model->l; - sv.resize(nsv); coef.resize(nsv); - bool flip = false; // whether to flip coefficients and bias - if (detail->n_class == 1) { - assert(nsv == 0); - coef0 = detail->labels[0]; - } else { - flip = (detail->labels[0] < detail->labels[1]); - REAL rho = detail->model->rho[0]; - coef0 = flip? rho : -rho; - } - for (UINT i = 0; i < nsv; ++i) { - svm_node *SVi = detail->model->SV[i]; - Input svi(_n_in, 0); - for (; SVi->index != -1; ++SVi) { - assert(SVi->index > 0 && (UINT) SVi->index <= _n_in); - svi[SVi->index-1] = SVi->value; - } - sv[i].swap(svi); - REAL ci = detail->model->sv_coef[0][i]; - coef[i] = flip? -ci : ci; - } - -#ifdef NDEBUG - // destroy the original model (but keep the param) to save memory - detail->clean_model(); detail->clean_data(); -#endif -} - -REAL SVM::margin_of (const Input& x, const Input& y) const { - assert(std::fabs(std::fabs(y[0]) - 1) < INFINITESIMAL); - return signed_margin(x) * y[0]; -} - -REAL SVM::signed_margin (const Input& x) const { - assert(x.size() == n_input()); - assert(ker != 0); - - const UINT nsv = n_support_vectors(); - REAL sum = bias(); - for (UINT i = 0; i < nsv; ++i) - sum += support_vector_coef(i) * (*ker)(support_vector(i), x); - -#ifndef NDEBUG - assert(detail); - if (!detail->trained) return sum; - - assert(detail->model && detail->x_space); - struct svm_node sx[n_input()+1]; - fill_svm_node(x, sx); - double m = detail->labels[0]; - if (nsv > 0) { - svm_predict_values(detail->model, sx, &m); - if (detail->labels[0] < detail->labels[1]) m = -m; - } - assert(std::fabs(sum - m) < EPSILON); -#endif - - return sum; -} - -REAL SVM::w_norm () const { - assert(ker != 0); - const UINT nsv = n_support_vectors(); - - REAL sum = 0; - for (UINT i = 0; i < nsv; ++i) { - for (UINT j = i; j < nsv; ++j) { - REAL ip = (*ker)(support_vector(i), support_vector(j)) - * support_vector_coef(i) * support_vector_coef(j); - sum += ip + (i==j? 0 : ip); -#ifndef NDEBUG - assert(detail); - if (!detail->trained) continue; - - assert(detail->model && detail->x_space); - /* direct access svm_model */ - REAL ve = svm_kernel(detail->model->SV[i], - detail->model->SV[j], detail->param) - * detail->model->sv_coef[0][i] * detail->model->sv_coef[0][j]; - assert(std::fabs(ip - ve) < EPSILON); -#endif - } - } - assert(sum >= 0); - return std::sqrt(sum); -} - -Output SVM::operator() (const Input& x) const { - assert(x.size() == n_input()); - - REAL y = (signed_margin(x) > 0? 1 : -1); -#ifndef NDEBUG - assert(detail); - if (!detail->trained) return Output(1, y); - - assert(detail->model && detail->x_space); - struct svm_node sx[n_input()+1]; - fill_svm_node(x, sx); - REAL l = svm_predict(detail->model, sx); - assert(std::fabs(y - l) < INFINITESIMAL); -#endif - - return Output(1, y); -} - -} // namespace lemga diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.h deleted file mode 100644 index 855a97c..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/svm.h +++ /dev/null @@ -1,76 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_SVM_H__ -#define __LEMGA_SVM_H__ - -/** @file - * @brief A LEMGA interface to LIBSVM. - * - * $Id: svm.h 2664 2006-03-07 19:50:51Z ling $ - */ - -#include "learnmodel.h" -#include "kernel.h" - -namespace lemga { - -/// Hide LIBSVM details so other files don't need to include LIBSVM header. -struct SVM_detail; - -class SVM : public LearnModel { -private: - SVM_detail *detail; - kernel::Kernel *ker; - // Why not var_shared_ptr? because kernel is not meant to be shared - REAL regC; // regularization parameter C - std::vector sv; // support vectors - std::vector coef; // @f$y_i\alpha_i@f$ - REAL coef0; // the bias - -public: - explicit SVM (UINT n_in = 0); - explicit SVM (const kernel::Kernel&, UINT n_in = 0); - SVM (const SVM&); - explicit SVM (std::istream&); - virtual ~SVM (); - const SVM& operator= (const SVM&); - - virtual const id_t& id () const; - virtual SVM* create () const { return new SVM(); } - virtual SVM* clone () const { return new SVM(*this); } - - REAL C () const { return regC; } - void set_C (REAL c) { regC = c; } - UINT n_support_vectors () const { return sv.size(); } - const Input& support_vector (UINT i) const { return sv[i]; } - /// @return @f$y_i\alpha_i@f$ - REAL support_vector_coef (UINT i) const { return coef[i]; } - REAL bias () const { return coef0; } - const kernel::Kernel& kernel () const { return *ker; } - REAL kernel (const Input&, const Input&) const; - void set_kernel (const kernel::Kernel&); - - virtual bool support_weighted_data () const { return true; } - virtual void initialize (); - virtual void train (); - - virtual Output operator() (const Input&) const; - virtual REAL margin_norm () const { return w_norm(); } - virtual REAL margin_of (const Input&, const Output&) const; - REAL w_norm () const; - -protected: - /// Gives the signed belief for 2-class problems - /// (positive belief means the larger label) - REAL signed_margin (const Input&) const; - void reset_model (); - virtual bool serialize (std::ostream&, ver_list&) const; - virtual bool unserialize (std::istream&, ver_list&, const id_t& = NIL_ID); -}; - -} // namespace lemga - -#ifdef __SVM_H__ -#warning "This header file may conflict with another `svm.h' file." -#endif -#define __SVM_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.cpp deleted file mode 100644 index 5c3424e..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/** @file - * $Id: utility.cpp 2631 2006-02-08 21:58:14Z ling $ - */ - -#include -#include -#include "utility.h" - -typedef std::vector RVEC; -typedef std::vector RMAT; - -// from the Numerical Recipes in C -bool Cholesky_decomp (RMAT& A, RVEC& p) { - const UINT n = A.size(); - assert(p.size() == n); - for (UINT i = 0; i < n; ++i) { - for (UINT j = i; j < n; ++j) { - REAL sum = A[i][j]; - for (UINT k = 0; k < i; ++k) - sum -= A[i][k] * A[j][k]; - if (i == j) { - if (sum <= 0) - return false; - p[i] = std::sqrt(sum); - } else - A[j][i] = sum / p[i]; - } - } - return true; -} - -// from the Numerical Recipes in C -void Cholesky_linsol (const RMAT& A, const RVEC& p, const RVEC& b, RVEC& x) { - const UINT n = A.size(); - assert(p.size() == n && b.size() == n); - x.resize(n); - - for (UINT i = 0; i < n; ++i) { - REAL sum = b[i]; - for (UINT k = 0; k < i; ++k) - sum -= A[i][k] * x[k]; - x[i] = sum / p[i]; - } - for (UINT i = n; i--;) { - REAL sum = x[i]; - for (UINT k = i+1; k < n; ++k) - sum -= A[k][i] * x[k]; - x[i] = sum / p[i]; - } -} - -bool ldivide (RMAT& A, const RVEC& b, RVEC& x) { - const UINT n = A.size(); - assert(b.size() == n); - RVEC p(n); - if (!Cholesky_decomp(A, p)) return false; - Cholesky_linsol(A, p, b, x); - return true; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.h deleted file mode 100644 index 9d3f0d3..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/utility.h +++ /dev/null @@ -1,56 +0,0 @@ -// -*- C++ -*- -#ifndef __LEMGA_UTILITY_H__ -#define __LEMGA_UTILITY_H__ - -/** @file - * @brief Some utility functions. - * - * $Id: utility.h 2631 2006-02-08 21:58:14Z ling $ - */ - -#include -#include "object.h" - -/** Solve inv(@a A) * @a b, when @a A is symmetric and positive-definite. - * Actually we only need the upper triangular part of @a A. - */ -bool ldivide (std::vector >& A, - const std::vector& b, std::vector& x); - -/// Gray code: start from all 0's, and iteratively go through all numbers -template -bool gray_next (std::vector& v, typename std::vector::size_type& p) { - typename std::vector::size_type n = v.size(); - assert(n > 0); - - // by default we alter the last bit - p = n - 1; - bool more = true; - - // find the largest n such that v[n] == 1 - while (--n && v[n] == 0) /* empty */; - - if (n > 0) { - typename std::vector::size_type j = n; - bool xor_sum = true; - while (n--) - xor_sum = xor_sum ^ (v[n] != 0); - if (xor_sum) p = j - 1; - } else if (v[0] != 0) { - p = 0; more = false; - } - v[p] = (v[p] == 0); - return more; -} - -template -bool gray_next (std::vector& v) { - typename std::vector::size_type p; - return gray_next(v, p); -} - -#ifdef __UTILITY_H__ -#warning "This header file may conflict with another `utility.h' file." -#endif -#define __UTILITY_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/vectorop.h b/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/vectorop.h deleted file mode 100644 index 3b93f4a..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/lemga/vectorop.h +++ /dev/null @@ -1,88 +0,0 @@ -// -*- C++ -*- -#ifndef __STL_ADDON_VECTOROP_H__ -#define __STL_ADDON_VECTOROP_H__ - -/** @file - * @brief Add basic arithmetic support to @c vector. - * - * Let @a u and @a v be two vectos and @a r be a scalar. This file defines - * the following operators: - * - @a u += @a v - * - -\a v - * - @a v *= @a r - * - inner_product<@a typename> (@a u, @a v) - * - * @note The template can `recursively' apply itself to vectors of vectors, - * such as two matrices of real numbers could add up (if they have the - * same size). - * - * @warning @a u, @a v, and @a r can have different base types. For example, - * if @a u is a double vector and @a v an integer vector. When an - * inappropriate coercion happens, the compiler may complain (with @c gcc, - * turn options @c -Wconversion or @c -Wall on). - * - * $Id: vectorop.h 1907 2004-12-11 00:51:14Z ling $ - */ - -#include -#include - -namespace lemga { -namespace op { - -using std::vector; - -/// @a u += @a v. -template -vector& operator+= (vector& u, const vector& v) { - assert(u.size() == v.size()); - typename vector::iterator x = u.begin(); - typename vector::const_iterator y = v.begin(); - for (; x != u.end(); ++x, ++y) *x += *y; - return u; -} - -/// -@a v. -template -vector operator- (const vector& v) { - vector s(v); - for (typename vector::iterator x = s.begin(); x != s.end(); ++x) - *x = -*x; - return s; -} - -/** @brief Inner product of @a u and @a v. - * - * The return type has to be explicitly specified, e.g.,\n - * double ip = inner_product(u, v);\n - * The compiler will still complain if there is `downward' coercion. - */ -template -inline R inner_product (const R& u, const N& v) { - return u * v; -} -template -RET inner_product (const vector& u, const vector& v) { - assert(u.size() == v.size()); - RET s(0); - typename vector::const_iterator x = u.begin(); - typename vector::const_iterator y = v.begin(); - for (; x != u.end(); ++x, ++y) s += inner_product(*x, *y); - return s; -} - -/// @a v *= @a r. -template -vector& operator*= (vector& v, const N& r) { - for (typename vector::iterator x = v.begin(); x != v.end(); ++x) - *x *= r; - return v; -} - -}} // namespace lemga::op - -#ifdef __VECTOROP_H__ -#warning "This header file may conflict with another `vectorop.h' file." -#endif -#define __VECTOROP_H__ -#endif diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/Makefile b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/Makefile deleted file mode 100644 index 500600b..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/Makefile +++ /dev/null @@ -1,130 +0,0 @@ -# $Id: Makefile 2614 2006-02-04 04:42:26Z ling $ - -LEMGA = .. - -objs = nnet bag adabst showlm testsvm lpbst multi - -.PHONY: test debug clean $(addsuffix _clean, $(objs)) -test: $(addsuffix r, $(objs)) -debug: $(addsuffix d, $(objs)) -clean: $(addsuffix _clean, $(objs)) - -include ${LEMGA}/lemga/Makefile -VPATH = ${LEMGA}/lemga -CC_FLAGS := -I${LEMGA} ${CC_FLAGS} - -nnet_src = nnet.cpp $(random_h) $(nnlayer_h) $(feedforwardnn_h) -rnnet.o: $(nnet_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dnnet.o: $(nnet_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -nnet_o := $(random_o) $(nnlayer_o) $(feedforwardnn_o) nnet.o -nnet_ro := $(addprefix r, $(nnet_o)) -nnet_do := $(addprefix d, $(nnet_o)) -nnetr: $(nnet_ro) - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -nnetd: $(nnet_do) - $(CC) $(CC_FLAGS) $(CC_DEBUG) $(CC_LIB_FLAGS) -o $@ $^ -nnet_clean: - $(RM) $(nnet_ro) $(nnet_do) nnetr nnetd - -bag_src = bag.cpp $(stump_h) $(bagging_h) -rbag.o: $(bag_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dbag.o: $(bag_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -bag_o := $(stump_o) $(bagging_o) bag.o -bag_ro := $(addprefix r, $(bag_o)) -bag_do := $(addprefix d, $(bag_o)) -bagr: $(bag_ro) - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -bagd: $(bag_do) - $(CC) $(CC_FLAGS) $(CC_DEBUG) $(CC_LIB_FLAGS) -o $@ $^ -bag_clean: - $(RM) $(bag_ro) $(bag_do) bagr bagd - -adabst_src = adabst.cpp $(pulse_h) $(adaboost_h) $(mgnboost_h) -radabst.o: $(adabst_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dadabst.o: $(adabst_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -adabst_o := $(pulse_o) $(adaboost_o) $(mgnboost_o) adabst.o -adabst_ro := $(addprefix r, $(adabst_o)) -adabst_do := $(addprefix d, $(adabst_o)) -adabstr: $(adabst_ro) - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -adabstd: $(adabst_do) - $(CC) $(CC_FLAGS) $(CC_DEBUG) $(CC_LIB_FLAGS) -o $@ $^ -adabst_clean: - $(RM) $(adabst_ro) $(adabst_do) adabstr adabstd - -showlm_src = showlm.cpp $(object_h) -rshowlm.o: $(showlm_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dshowlm.o: $(showlm_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -showlm_o := $(stump_o) $(pulse_o) $(perceptron_o) $(feedforwardnn_o) \ - $(bagging_o) $(adaboost_o) $(cgboost_o) $(mgnboost_o) $(lpboost_o) \ - $(multiclass_ecoc_o) $(adaboost_ecoc_o) $(crossval_o) \ - $(kernel_o) $(svm_o) \ - showlm.o -showlm_ro := $(addprefix r, $(showlm_o)) -showlm_do := $(addprefix d, $(showlm_o)) -showlmr: $(showlm_ro) $(lpboost_lib) - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -showlmd: $(showlm_do) $(lpboost_lib) - $(CC) $(CC_FLAGS) $(CC_DEBUG) $(CC_LIB_FLAGS) -o $@ $^ -showlm_clean: - $(RM) $(showlm_ro) $(showlm_do) showlmr showlmd - -testsvm_src = testsvm.cpp $(random_h) $(crossval_h) $(svm_h) -rtestsvm.o: $(testsvm_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dtestsvm.o: $(testsvm_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -testsvm_o := $(random_o) $(crossval_o) $(svm_o) testsvm.o -testsvm_ro := $(addprefix r, $(testsvm_o)) -testsvm_do := $(addprefix d, $(testsvm_o)) -testsvmr: $(testsvm_ro) - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -testsvmd: $(testsvm_do) - $(CC) $(CC_FLAGS) $(CC_DEBUG) $(CC_LIB_FLAGS) -o $@ $^ -testsvm_clean: - $(RM) $(testsvm_ro) $(testsvm_do) testsvmr testsvmd - -lpbst_src = lpbst.cpp $(stump_h) $(adaboost_h) $(lpboost_h) -rlpbst.o: $(lpbst_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dlpbst.o: $(lpbst_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -lpbst_o := $(stump_o) $(adaboost_o) $(lpboost_o) lpbst.o -lpbst_ro := $(addprefix r, $(lpbst_o)) -lpbst_do := $(addprefix d, $(lpbst_o)) -lpbstr: $(lpbst_ro) $(lpboost_lib) - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -lpbstd: $(lpbst_do) $(lpboost_lib) - $(CC) $(CC_FLAGS) $(CC_DEBUG) $(CC_LIB_FLAGS) -o $@ $^ -lpbst_clean: - $(RM) $(lpbst_ro) $(lpbst_do) lpbstr lpbstd - -multi_src = multi.cpp $(perceptron_h) $(adaboost_h) $(multiclass_ecoc_h) -rmulti.o: $(multi_src) - $(CC) $(CC_FLAGS) $(CC_FINAL) -c -o $@ $< -dmulti.o: $(multi_src) - $(CC) $(CC_FLAGS) $(CC_DEBUG) -c -o $@ $< - -multi_o := $(perceptron_o) $(adaboost_o) $(multiclass_ecoc_o) multi.o -multi_ro := $(addprefix r, $(multi_o)) -multi_do := $(addprefix d, $(multi_o)) -multir: $(multi_ro) - $(CC) $(CC_FLAGS) $(CC_FINAL) $(CC_LIB_FLAGS) -o $@ $^ -multid: $(multi_do) - $(CC) $(CC_FLAGS) $(CC_DEBUG) $(CC_LIB_FLAGS) -o $@ $^ -multi_clean: - $(RM) $(multi_ro) $(multi_do) multir multid diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/README b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/README deleted file mode 100644 index f5f7076..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/README +++ /dev/null @@ -1,20 +0,0 @@ -README for lemga/test -===================== - -Files here are meant to demonstrate the usage of Lemga. - - adabst.cpp AdaBoost and arc-gv using pulse functions - bag.cpp Bagging using decision stumps - lpbst.cpp Both AdaBoost and LPBoost with decision stumps - multi.cpp Multiclass classification with AdaBoost-perceptrons. - nnet.cpp Neural networks - showlm.cpp A model file loader/checker -testsvm.cpp SVM classification with several kernels + cross-validation - -To compile all modules with "release" and "debug" settings, type - make test debug - -To compile a single module with "debug" setting, type - make "module name appended by 'd'" -For example, - make nnetd diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/adabst.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/adabst.cpp deleted file mode 100644 index aba3ed0..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/adabst.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* $Id: adabst.cpp 2664 2006-03-07 19:50:51Z ling $ */ - -#include -#include -#include -#include -#include - -int main (unsigned int argc, char* argv[]) { - if (argc < 5) { - std::cerr << "Usage: " << argv[0] << " datafile n_train n_test" - << " #_AdaBoost\n"; - return -1; - } - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": data file (" - << argv[1] << ") open error\n"; - return -2; - } - - /* load training and test data */ - lemga::pDataSet trd = lemga::load_data(fd, atoi(argv[2])); - lemga::pDataSet ted = lemga::load_data(fd, atoi(argv[3])); - std::cout << trd->size() << " training samples and " - << ted->size() << " test samples loaded\n"; - fd.close(); - - /* set base model */ - lemga::Pulse st; - st.set_max_transitions(3); - lemga::AdaBoost ada; - lemga::MgnBoost mgn; - ada.set_base_model(st); - mgn.set_base_model(st); - - /* train the AdaBoost of pulses */ - ada.set_max_models(atoi(argv[4])); - ada.set_train_data(trd); - ada.train(); - - /* train the MgnBoost of pulses */ - mgn.set_max_models(atoi(argv[4])); - mgn.set_train_data(trd); - mgn.train(); - - std::cout << "Minimal margin: AdaBoost " - << ada.min_margin() / ada.margin_norm() - << ", arc-gv " - << mgn.min_margin() / mgn.margin_norm() << '\n'; - - /* save the AdaBoost to a file */ - std::ofstream fw("adapulse.lm"); - if (!(fw << ada)) { - std::cerr << argv[0] << ": AdaBoost file save error\n"; - return -3; - } - std::cout << "AdaBoost saved\n"; - fw.close(); - - /* load the AdaBoost back */ - lemga::AdaBoost ad2; - std::ifstream fr("adapulse.lm"); - if (!(fr >> ad2)) { - std::cerr << argv[0] << ": AdaBoost file load error\n"; - return -3; - } - std::cout << "AdaBoost loaded\n"; - fr.close(); - - /* test the AdaBoost */ - double ada_tre(0), ada_tee(0), mgn_tre(0), mgn_tee(0); - for (UINT i = 0; i < trd->size(); ++i) { - ada_tre += st.c_error(ad2(trd->x(i)), trd->y(i)); - mgn_tre += st.c_error(mgn(trd->x(i)), trd->y(i)); - } - for (UINT i = 0; i < ted->size(); ++i) { - ada_tee += st.c_error(ad2(ted->x(i)), ted->y(i)); - mgn_tee += st.c_error(mgn(ted->x(i)), ted->y(i)); - } - std::cout << "training error: AdaBoost " << ada_tre / trd->size() - << ",\tarc-gv " << mgn_tre / trd->size() << '\n'; - std::cout << " test error: AdaBoost " << ada_tee / ted->size() - << ",\tarc-gv " << mgn_tee / ted->size() << '\n'; - - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/bag.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/bag.cpp deleted file mode 100644 index 815b8f3..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/bag.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* $Id: bag.cpp 2664 2006-03-07 19:50:51Z ling $ */ - -#include -#include -#include -#include - -int main (unsigned int argc, char* argv[]) { - if (argc < 5) { - std::cerr << "Usage: " << argv[0] << " datafile n_train n_test" - << " #_bagging\n"; - return -1; - } - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": data file (" - << argv[1] << ") open error\n"; - return -2; - } - - /* load training and test data */ - lemga::pDataSet trd = lemga::load_data(fd, atoi(argv[2])); - lemga::pDataSet ted = lemga::load_data(fd, atoi(argv[3])); - std::cout << trd->size() << " training samples and " - << ted->size() << " test samples loaded\n"; - fd.close(); - - /* set base model */ - lemga::Stump st; - lemga::Bagging bag; - bag.set_base_model(st); - - /* train the bagging of stumps */ - bag.set_max_models(atoi(argv[4])); - bag.set_train_data(trd); - bag.train(); - - /* save the bagging to a file */ - std::ofstream fw("bagstump.lm"); - if (!(fw << bag)) { - std::cerr << argv[0] << ": bagging file save error\n"; - return -3; - } - std::cout << "bagging saved\n"; - fw.close(); - - /* load the bagging back */ - lemga::Bagging bag2; - std::ifstream fr("bagstump.lm"); - if (!(fr >> bag2)) { - std::cerr << argv[0] << ": bagging file load error\n"; - return -3; - } - std::cout << "bagging loaded\n"; - fr.close(); - - /* test the bagging */ - double tre = 0, tee = 0; - for (UINT i = 0; i < trd->size(); ++i) - tre += st.c_error(bag(trd->x(i)), trd->y(i)); - for (UINT i = 0; i < ted->size(); ++i) - tee += st.c_error(bag2(ted->x(i)), ted->y(i)); - std::cout << "training error: " << tre / trd->size() - << ", test error: " << tee / ted->size() << "\n"; - - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/lpbst.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/lpbst.cpp deleted file mode 100644 index 424e1d2..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/lpbst.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* $Id: lpbst.cpp 2664 2006-03-07 19:50:51Z ling $ */ - -#include -#include -#include -#include -#include - -int main (unsigned int argc, char* argv[]) { - if (argc < 5) { - std::cerr << "Usage: " << argv[0] << " datafile n_train n_test" - << " #_boost\n"; - return -1; - } - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": data file (" - << argv[1] << ") open error\n"; - return -2; - } - - /* load training and test data */ - lemga::pDataSet trd = lemga::load_data(fd, atoi(argv[2])); - lemga::pDataSet ted = lemga::load_data(fd, atoi(argv[3])); - std::cout << trd->size() << " training samples and " - << ted->size() << " test samples loaded\n"; - fd.close(); - - /* set base model */ - lemga::Stump st; - lemga::AdaBoost ada; - lemga::LPBoost lpb; - ada.set_base_model(st); - lpb.set_base_model(st); - - /* train the AdaBoost of stumps */ - ada.set_max_models(atoi(argv[4])); - ada.set_train_data(trd); - ada.train(); - - /* train the LPBoost of stumps */ - lpb.set_max_models(atoi(argv[4])); - lpb.set_C(10); - lpb.set_train_data(trd); - lpb.train(); - - std::cout << "Minimal margin: AdaBoost " - << ada.min_margin() / ada.margin_norm() - << ", LPBoost " - << lpb.min_margin() / lpb.margin_norm() << '\n'; - - /* test the performance */ - double ada_tre(0), ada_tee(0), lpb_tre(0), lpb_tee(0); - for (UINT i = 0; i < trd->size(); ++i) { - ada_tre += st.c_error(ada(trd->x(i)), trd->y(i)); - lpb_tre += st.c_error(lpb(trd->x(i)), trd->y(i)); - } - for (UINT i = 0; i < ted->size(); ++i) { - ada_tee += st.c_error(ada(ted->x(i)), ted->y(i)); - lpb_tee += st.c_error(lpb(ted->x(i)), ted->y(i)); - } - std::cout << "training error: AdaBoost " << 100*ada_tre/trd->size() - << "%,\tLPBoost " << 100*lpb_tre/trd->size() << "%\n"; - std::cout << " test error: AdaBoost " << 100*ada_tee/ted->size() - << "%,\tLPBoost " << 100*lpb_tee/ted->size() << "%\n"; - - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/multi.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/multi.cpp deleted file mode 100644 index 0829429..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/multi.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* $Id: multi.cpp 2891 2006-11-08 03:17:31Z ling $ */ - -#include -#include -#include -#include -#include - -int main (unsigned int argc, char* argv[]) { - if (argc < 4) { - std::cerr << "Usage: " << argv[0] << " datafile n_train n_test\n"; - return -1; - } - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": data file (" - << argv[1] << ") open error\n"; - return -2; - } - - /* load training and test data */ - lemga::pDataSet trd = lemga::load_data(fd, atoi(argv[2])); - const UINT n_in = trd->x(0).size(); - lemga::pDataSet ted = lemga::load_data(fd, atoi(argv[3]), n_in, 1); - std::cout << trd->size() << " training samples and " - << ted->size() << " test samples loaded\n"; - fd.close(); - - /* base model */ - set_seed(0); // perceptron learning may need this - lemga::Perceptron p; - p.set_parameter(0.002, 0, 500); - p.set_train_method(p.RCD_BIAS); - p.set_weight(lemga::Input(n_in+1,0)); - //p.start_with_fld(); - - lemga::AdaBoost agg; - agg.set_base_model(p); - // no AdaBoost init, to keep the initial perceptron zero weight - agg.set_max_models(50); - - /* ECOC model */ - lemga::MultiClass_ECOC mult; - mult.set_base_model(agg); - mult.set_ECOC_table(lemga::ONE_VS_ALL); - mult.set_train_data(trd); - mult.train(); - - std::ofstream fw("multi-perc.lm"); - if (!(fw << mult)) { - std::cerr << argv[0] << ": MultiClass file save error\n"; - return -3; - } - std::cout << "MultiClass saved\n"; - fw.close(); - - /* test the performance */ - double tre(0), tee(0); - for (UINT i = 0; i < trd->size(); ++i) - tre += mult.c_error(mult(trd->x(i)), trd->y(i)); - for (UINT i = 0; i < ted->size(); ++i) - tee += mult.c_error(mult(ted->x(i)), ted->y(i)); - - std::cout << "training error: " << 100*tre/trd->size() - << "%,\ttest error: " << 100*tee/ted->size() << "%\n"; - - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/nnet.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/nnet.cpp deleted file mode 100644 index bc82e2c..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/nnet.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* $Id: nnet.cpp 1907 2004-12-11 00:51:14Z ling $ */ - -#include -#include -#include -#include -#include - -int main (unsigned int argc, char* argv[]) { - if (argc < 6) { - std::cerr << "Usage: " << argv[0] << " datafile n_train n_test" - << " #_input #_hidden ... #_output\n"; - return -1; - } - - set_seed(0); - - /* constructing neural nets with layers */ - UINT n_in = atoi(argv[4]); - lemga::FeedForwardNN nn; - - std::cout << "Neural network " << n_in; - UINT l_in = n_in; - for (UINT i = 5; i < argc; ++i) { - UINT l_out = atoi(argv[i]); - std::cout << 'x' << l_out; - - lemga::NNLayer l(l_in, l_out); - l.set_weight_range(-0.2, 0.2); - nn.add_top(l); - - l_in = l_out; - } - std::cout << " constructed\n"; - UINT n_out = l_in; - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": data file (" - << argv[1] << ") open error\n"; - return -2; - } - - /* load training and test data */ - lemga::pDataSet trd = lemga::load_data(fd, atoi(argv[2]), n_in, n_out); - lemga::pDataSet ted = lemga::load_data(fd, atoi(argv[3]), n_in, n_out); - std::cout << trd->size() << " training samples and " - << ted->size() << " test samples loaded\n"; - fd.close(); - - /* train the neural network */ - nn.initialize(); - nn.set_train_data(trd); - nn.set_train_method(nn.CONJUGATE_GRADIENT); - nn.set_parameter(0.1, 1e-4, 1000); - nn.train(); - - /* save the neural network to a file */ - std::ofstream fw("nnet.lm"); - if (!(fw << nn)) { - std::cerr << argv[0] << ": neural network file save error\n"; - return -3; - } - std::cout << "network saved\n"; - fw.close(); - - /* load the network back */ - lemga::FeedForwardNN nn2; - std::ifstream fr("nnet.lm"); - if (!(fr >> nn2)) { - std::cerr << argv[0] << ": neural network file save error\n"; - return -3; - } - std::cout << "network loaded\n"; - fr.close(); - - /* test the network */ - double tre = 0, tee = 0; - for (UINT i = 0; i < trd->size(); ++i) - tre += nn2.r_error(nn2(trd->x(i)), trd->y(i)); - for (UINT i = 0; i < ted->size(); ++i) - tee += nn2.r_error(nn2(ted->x(i)), ted->y(i)); - std::cout << "training error: " << tre / trd->size() - << ", test error: " << tee / ted->size() << "\n"; - - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/showlm.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/showlm.cpp deleted file mode 100644 index 2613401..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/showlm.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include -#include -// Though we don't need other LEMGA header files here, we do -// need to link all modules. - -bool show_lm (std::istream& is) { - /* load the model */ - Object *p = Object::create(is); - if (p == 0) return false; - - std::cout << "File loaded:\n" << *p; - delete p; - return true; -} - -int main (unsigned int argc, char* argv[]) { - bool ok; - if (argc < 2) ok = show_lm(std::cin); - else { - std::ifstream fr(argv[1]); - ok = show_lm(fr); - fr.close(); - } - if (!ok) - std::cerr << argv[0] << ": model load error\n"; - return !ok; -} diff --git a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/testsvm.cpp b/src/Algorithms/orensemble/orensemble/lemga-20060516/test/testsvm.cpp deleted file mode 100644 index 14888ab..0000000 --- a/src/Algorithms/orensemble/orensemble/lemga-20060516/test/testsvm.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* $Id: testsvm.cpp 2557 2006-01-19 00:43:17Z ling $ */ - -#include -#include -#include -#include -#include - -int main (unsigned int argc, char* argv[]) { - if (argc < 4) { - std::cerr << "Usage: " << argv[0] << " datafile n_train n_test" - << "\n"; - return -1; - } - - /* open data file */ - std::ifstream fd(argv[1]); - if (!fd.is_open()) { - std::cerr << argv[0] << ": data file (" - << argv[1] << ") open error\n"; - return -2; - } - - /* load training and test data */ - lemga::pDataSet trd = lemga::load_data(fd, atoi(argv[2])); - lemga::pDataSet ted = lemga::load_data(fd, atoi(argv[3])); - std::cout << trd->size() << " training samples and " - << ted->size() << " test samples loaded\n"; - fd.close(); - -#if 1 - /* set up cross-validation */ - set_seed(0); - lemga::vFoldCrossVal cv; - cv.set_folds(5, 4); // folds = 5, repeating 4 times - - /* set up different kernels & C values */ - std::vector ker; - ker.push_back(new lemga::kernel::RBF(0.5)); - ker.push_back(new lemga::kernel::Stump); - ker.push_back(new lemga::kernel::Perceptron); - for (UINT i = 0; i < ker.size(); delete ker[i++]) { - lemga::SVM svm(*ker[i]); - for (int c = 0; c < 4; ++c) { - svm.set_C(1 << c); - cv.add_model(svm); - } - } - cv.set_train_data(trd); - cv.set_full_train(false); - cv.train(); - - // output the CV errors - std::cout.precision(4); - UINT nc = cv.size()/ker.size(); - for (UINT i = 0; i < nc; ++i) - std::cout << "\tC=" << ((lemga::SVM&) cv.model(i)).C(); - for (UINT i = 0; i < cv.size(); ++i) { - if (i % nc == 0) - std::cout << '\n' << - ((lemga::SVM&) cv.model(i)).kernel().id().substr(15,7); - std::cout << '\t' << 100*cv.error(i) << '%'; - } - - // train the best model - lemga::pLearnModel plm = cv.best_model().clone(); - lemga::SVM& svm = (lemga::SVM&) *plm; -#else // OR if no validation is wanted, simply do - lemga::kernel::RBF kernel(0.5); - lemga::SVM svm(kernel); - svm.set_C(10); -#endif - - std::cout << "\nTrain on the model with " << svm.kernel().id().substr(15) - << " and C = " << svm.C() << '\n'; - svm.set_train_data(trd); - svm.train(); - std::cout << "Total " << svm.n_support_vectors() << " support vectors." - << " Please check with svm_train() output above.\n"; - std::cout << "training error: " << svm.train_c_error()*100 - << "%, test error: " << svm.test_c_error(ted)*100 << "%\n"; - - return 0; -} diff --git a/src/Algorithms/orensemble/orensemble/libsvm-2.81/COPYRIGHT b/src/Algorithms/orensemble/orensemble/libsvm-2.81/COPYRIGHT deleted file mode 100644 index b4e1a35..0000000 --- a/src/Algorithms/orensemble/orensemble/libsvm-2.81/COPYRIGHT +++ /dev/null @@ -1,31 +0,0 @@ - -Copyright (c) 2000-2005 Chih-Chung Chang and Chih-Jen Lin -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - -3. Neither name of copyright holders nor the names of its contributors -may be used to endorse or promote products derived from this software -without specific prior written permission. - - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/Algorithms/orensemble/orensemble/libsvm-2.81/FAQ.html b/src/Algorithms/orensemble/orensemble/libsvm-2.81/FAQ.html deleted file mode 100644 index 1547984..0000000 --- a/src/Algorithms/orensemble/orensemble/libsvm-2.81/FAQ.html +++ /dev/null @@ -1,1154 +0,0 @@ - - -LIBSVM FAQ - - - -

LIBSVM FAQ

-last modified : -Thu, 20 Oct 2005 09:03:48 GMT - -
  • All Questions(52)
    • - - - - - - -
    -

    - - -Q: Some courses which have used libsvm as a tool -
    -

    -

    -[Go Top] -

    - -Q: Where can I find documents of libsvm ? -
    -

    -In the package there is a README file which -details all options, data format, and library calls. -The model selection tool and the python interface -have a separate README under the directory python. -The guide - -A practical guide to support vector classification - shows beginners how to train/test their data. -The paper LIBSVM -: a library for support vector machines discusses the implementation of -libsvm in detail. -

    -[Go Top] -

    - -Q: What are changes in previous versions? -
    -

    See the change log. - -

    You can download earlier versions -here. -

    -[Go Top] -

    - -Q: I would like to cite libsvm. Which paper should I cite ? -
    -

    -Please cite the following document: -

    -Chih-Chung Chang and Chih-Jen Lin, LIBSVM -: a library for support vector machines, 2001. -Software available at http://www.csie.ntu.edu.tw/~cjlin/libsvm -

    -The bibtex format is as follows -

    -@Manual{CC01a,
-  author =	 {Chih-Chung Chang and Chih-Jen Lin},
-  title =	 {{LIBSVM}: a library for support vector machines},
-  year =	 {2001},
-  note =	 {Software available at \url{http://www.csie.ntu.edu.tw/~cjlin/libsvm}}
-}
-
    -

    -[Go Top] -

    - -Q: I would like to use libsvm in my software. Is there any license problem? -
    -

    -The libsvm license ("the modified BSD license") -is compatible with many -free software licenses such as GPL. Hence, it is very easy to -use libsvm in your software. -It can also be used in commercial products. -

    -[Go Top] -

    - -Q: Is there a repository of additional tools based on libsvm? -
    -

    -Yes, see libsvm -tools -

    -[Go Top] -

    - -Q: On unix machines, I got "error in loading shared libraries" or "cannot open shared object file." What happened ? -
    - -

    -This usually happens if you compile the code -on one machine and run it on another which has incompatible -libraries. -Try to recompile the program on that machine or use static linking. -

    -[Go Top] -

    - -Q: I have modified the source and would like to build the graphic interface "svm-toy" on MS windows. How should I do it ? -
    - -

    -Build it as a project by choosing "Win32 Project." -On the other hand, for "svm-train" and "svm-predict" -you want to choose "Win32 Console Project." -After libsvm 2.5, you can also use the file Makefile.win. -See details in README. - - -

    -If you are not using Makefile.win and see the following -link error -

    -LIBCMTD.lib(wwincrt0.obj) : error LNK2001: unresolved external symbol
-_wWinMain@16
-
    -you may have selected a wrong project type. -

    -[Go Top] -

    - -Q: I am an MS windows user but why only one (SVM_toy) of those precompiled .exe actually runs ? -
    - -

    -You need to open a command window -and type svmtrain.exe to see all options. -Some examples are in README file. -

    -[Go Top] -

    - -Q: Why sometimes not all attributes of a data appear in the training/model files ? -
    -

    -libsvm uses the so called "sparse" format where zero -values do not need to be stored. Hence a data with attributes -

    -1 0 2 0
-
    -is represented as -
    -1:1 3:2
-
    -

    -[Go Top] -

    - -Q: What if my data are non-numerical ? -
    -

    -Currently libsvm supports only numerical data. -You may have to change non-numerical data to -numerical. For example, you can use several -binary attributes to represent a categorical -attribute. -

    -[Go Top] -

    - -Q: Why do you consider sparse format ? Will the training of dense data be much slower ? -
    -

    -This is a controversial issue. The kernel -evaluation (i.e. inner product) of sparse vectors is slower -so the total training time can be at least twice or three times -of that using the dense format. -However, we cannot support only dense format as then we CANNOT -handle extremely sparse cases. Simplicity of the code is another -concern. Right now we decide to support -the sparse format only. -

    -[Go Top] -

    - -Q: The output of training C-SVM is like the following. What do they mean? -
    -
    optimization finished, #iter = 219 -
    nu = 0.431030 -
    obj = -100.877286, rho = 0.424632 -
    nSV = 132, nBSV = 107 -
    Total nSV = 132 -

    -obj is the optimal objective value of the dual SVM problem. -rho is the bias term in the decision function -sgn(w^Tx - rho). -nSV and nBSV are number of support vectors and bounded support -vectors (i.e., alpha_i = C). nu-svm is a somewhat equivalent -form of C-SVM where C is replaced by nu. nu simply shows the -corresponding parameter. More details are in - -libsvm document. -

    -[Go Top] -

    - -Q: Can you explain more about the model file? -
    - -

    -After the parameters, each line represents a support vector. -Support vectors are listed in the order of "labels" listed earlier. -(i.e., those from the first class in the "labels" list are -grouped first, and so on.) -If k is the total number of classes, -in front of each support vector, there are -k-1 coefficients -y*alpha where alpha are dual solution of the -following two class problems: -
    -1 vs j, 2 vs j, ..., j-1 vs j, j vs j+1, j vs j+2, ..., j vs k -
    -and y=1 in first j-1 coefficients, y=-1 in the remaining -k-j coefficients. - -For example, if there are 4 classes, the file looks like: - -

    -+-+-+-+--------------------+
-|1|1|1|                    |
-|v|v|v|  SVs from class 1  |
-|2|3|4|                    |
-+-+-+-+--------------------+
-|1|2|2|                    |
-|v|v|v|  SVs from class 2  |
-|2|3|4|                    |
-+-+-+-+--------------------+
-|1|2|3|                    |
-|v|v|v|  SVs from class 3  |
-|3|3|4|                    |
-+-+-+-+--------------------+
-|1|2|3|                    |
-|v|v|v|  SVs from class 4  |
-|4|4|4|                    |
-+-+-+-+--------------------+
-
    -

    -[Go Top] -

    - -Q: Should I use float or double to store numbers in the cache ? -
    - -

    -We have float as the default as you can store more numbers -in the cache. -In general this is good enough but for few difficult -cases (e.g. C very very large) where solutions are huge -numbers, it might be possible that the numerical precision is not -enough using only float. -

    -[Go Top] -

    - -Q: How do I choose the kernel? -
    - -

    -In general we suggest you to try the RBF kernel first. -A recent result by Keerthi and Lin -( -download paper here) -shows that if RBF is used with model selection, -then there is no need to consider the linear kernel. -The kernel matrix using sigmoid may not be positive definite -and in general it's accuracy is not better than RBF. -(see the paper by Lin and Lin -( -download paper here). -Polynomial kernels are ok but if a high degree is used, -numerical difficulties tend to happen -(thinking about dth power of (<1) goes to 0 -and (>1) goes to infinity). -

    -[Go Top] -

    - -Q: Does libsvm have special treatments for linear SVM? -
    - -

    - -No, at this point libsvm solves linear/nonlinear SVMs by the -same way. -Note that there are some possible -tricks to save training/testing time if the -linear kernel is used. -Hence libsvm is NOT particularly efficient for linear SVM, -especially for -using large C on -problems whose number of data is much larger -than number of attributes. -You can -

    - -

    On the other hand, you do not really need to solve -linear SVMs. See the previous question about choosing -kernels for details. -

    -[Go Top] -

    - -Q: The number of free support vectors is large. What should I do? -
    -

    -This usually happens when the data are overfitted. -If attributes of your data are in large ranges, -try to scale them. Then the region -of appropriate parameters may be larger. -Note that there is a scale program -in libsvm. -

    -[Go Top] -

    - -Q: Should I scale training and testing data in a similar way? -
    -

    -Yes, you can do the following: -
    svm-scale -s scaling_parameters train_data > scaled_train_data -
    svm-scale -r scaling_parameters test_data > scaled_test_data -

    -[Go Top] -

    - -Q: Does it make a big difference if I scale each attribute to [0,1] instead of [-1,1]? -
    - -

    -For the linear scaling method, if the RBF kernel is -used and parameter selection is conducted, there -is no difference. Assume Mi and mi are -respectively the maximal and minimal values of the -ith attribute. Scaling to [0,1] means -

    -                x'=(x-mi)/(Mi-mi)
-
    -For [-1,1], -
    -                x''=2(x-mi)/(Mi-mi)-1.
-
    -In the RBF kernel, -
    -                x'-y'=(x-y)/(Mi-mi), x''-y''=2(x-y)/(Mi-mi).
-
    -Hence, using (C,g) on the [0,1]-scaled data is the -same as (C,g/2) on the [-1,1]-scaled data. - -

    Though the performance is the same, the computational -time may be different. For data with many zero entries, -[0,1]-scaling keeps the sparsity of input data and hence -may save the time. -

    -[Go Top] -

    - -Q: The prediction rate is low. How could I improve it? -
    -

    -Try to use the model selection tool grid.py in the python -directory find -out good parameters. To see the importance of model selection, -please -see my talk: - -A practical guide to support vector -classification - -

    -[Go Top] -

    - -Q: My data are unbalanced. Could libsvm handle such problems? -
    -

    -Yes, there is a -wi options. For example, if you use -

    - svm-train -s 0 -c 10 -w1 1 -w-1 5 data_file -

    -the penalty for class "-1" is larger. -Note that this -w option is for C-SVC only. -

    -[Go Top] -

    - -Q: What is the difference between nu-SVC and C-SVC? -
    -

    -Basically they are the same thing but with different -parameters. The range of C is from zero to infinity -but nu is always between [0,1]. A nice property -of nu is that it is related to the ratio of -support vectors and the ratio of the training -error. -

    -[Go Top] -

    - -Q: The program keeps running without showing any output. What should I do? -
    -

    -You may want to check your data. Each training/testing -data must be in one line. It cannot be separated. -In addition, you have to remove empty lines. -

    -[Go Top] -

    - -Q: The program keeps running (with output, i.e. many dots). What should I do? -
    -

    -In theory libsvm guarantees to converge if the kernel -matrix is positive semidefinite. -After version 2.4 it can also handle non-PSD -kernels such as the sigmoid (tanh). -Therefore, this means you are -handling ill-conditioned situations -(e.g. too large/small parameters) so numerical -difficulties occur. -

    -[Go Top] -

    - -Q: The training time is too long. What should I do? -
    -

    -For large problems, please specify enough cache size (i.e., --m). -Slow convergence may happen for some difficult cases (e.g. -c is large). -You can try to use a looser stopping tolerance with -e. -If that still doesn't work, you may want to contact us. We can show you some -tricks on improving the training time. -

    -[Go Top] -

    - -Q: How do I get the decision value(s)? -
    -

    -We print out decision values for regression. For classification, -we solve several binary SVMs for multi-class cases, so -you obtain values by easily calling the subroutine -svm_predict_values. Their corresponding labels -can be obtained from svm_get_labels. -Details are in -README of libsvm package. - -

    -We do not recommend the following. But if you would -like to get values for -TWO-class classification with labels +1 and -1 -(note: +1 and -1 but not things like 5 and 10) -in the easiest way, simply add -

    -		printf("%f\n", dec_values[0]*model->label[0]);
-
    -after the line -
    -		svm_predict_values(model, x, dec_values);
-
    -of the file svm.cpp. -Positive (negative) -decision values correspond to data predicted as +1 (-1). - - -

    -[Go Top] -

    - -Q: How do I get the distance between a point and the hyperplane? -
    -

    -The distance is |decision_value| / |w|. -We have |w|^2 = w^Tw = alpha^T Q alpha = 2*(dual_obj + sum alpha_i). -Thus in svm.cpp please find the place -where we print the dual objective value -and add a statement to print w^Tw. - -

    -[Go Top] -

    - -Q: For some problem sets if I use a large cache (i.e. large -m) on a linux machine, why sometimes I get "segmentation fault ?" -
    -

    - -On 32-bit machines, the maximum addressable -memory is 4GB. The Linux kernel uses 3:1 -split which means user space is 3G and -kernel space is 1G. Although there are -3G user space, the maximum dynamic allocation -memory is 2G. So, if you specify -m near 2G, -the memory will be exhausted. And svm-train -will fail when it asks more memory. -For more details, please read - -this article. -

    -There are two ways to solve this. If your -machine supports Intel's PAE (Physical Address -Extension), you can turn on the option HIGHMEM64G -in Linux kernel which uses 4G:4G split for -kernel and user space. If you don't, you can -try a software `tub' which can elimate the 2G -boundary for dymanic allocated memory. The `tub' -is available at -http://www.bitwagon.com/tub.html. - - - -

    -[Go Top] -

    - -Q: How do I disable screen output of svm-train and svm-predict ? -
    -

    -Simply update svm.cpp: -

    -#if 1
-void info(char *fmt,...)
-
    -to -
    -#if 0
-void info(char *fmt,...)
-
    -

    -[Go Top] -

    - -Q: I would like to use my own kernel but find out that there are two subroutines for kernel evaluations: k_function() and kernel_function(). Which one should I modify ? -
    -

    -The reason why we have two functions is as follows: -For the RBF kernel exp(-g |xi - xj|^2), if we calculate -xi - xj first and then the norm square, there are 3n operations. -Thus we consider exp(-g (|xi|^2 - 2dot(xi,xj) +|xj|^2)) -and by calculating all |xi|^2 in the beginning, -the number of operations is reduced to 2n. -This is for the training. For prediction we cannot -do this so a regular subroutine using that 3n operations is -needed. - -The easiest way to have your own kernel is -to put the same code in these two -subroutines by replacing any kernel. -

    -[Go Top] -

    - -Q: What method does libsvm use for multi-class SVM ? Why don't you use the "1-against-the rest" method ? -
    -

    -It is one-against-one. We chose it after doing the following -comparison: -C.-W. Hsu and C.-J. Lin. - -A comparison of methods -for multi-class support vector machines -, -IEEE Transactions on Neural Networks, 13(2002), 415-425. - -

    -"1-against-the rest" is a good method whose performance -is comparable to "1-against-1." We do the latter -simply because its training time is shorter. -

    -[Go Top] -

    - -Q: After doing cross validation, why there is no model file outputted ? -
    -

    -Cross validation is used for selecting good parameters. -After finding them, you want to re-train the whole -data without the -v option. -

    -[Go Top] -

    - -Q: I would like to try different random partition for cross validation, how could I do it ? -
    -

    -If you use GNU C library, -the default seed 1 is considered. Thus you always -get the same result of running svm-train -v. -To have different seeds, you can add the following code -in svm-train.c: -

    -#include <time.h>
-
    -and in the beginning of the subroutine do_cross_validation(), -
    -srand(time(0));
-
    -

    -[Go Top] -

    - -Q: I would like to solve L2-SVM (i.e., error term is quadratic). How should I modify the code ? -
    -

    -It is extremely easy. Taking c-svc for example, only two -places of svm.cpp have to be changed. -First, modify the following line of -solve_c_svc from -

    -	s.Solve(l, SVC_Q(*prob,*param,y), minus_ones, y,
-		alpha, Cp, Cn, param->eps, si, param->shrinking);
-
    -to -
    -	s.Solve(l, SVC_Q(*prob,*param,y), minus_ones, y,
-		alpha, INF, INF, param->eps, si, param->shrinking);
-
    -Second, in the class of SVC_Q, declare C as -a private variable: -
    -	double C;
-
    -In the constructor we assign it to param.C: -
    -        this->C = param.C;		
-
    -Than in the the subroutine get_Q, after the for loop, add -
    -        if(i >= start && i < len) 
-		data[i] += 1/C;
-
    - -For one-class svm, the modification is exactly the same. For SVR, you don't need an if statement like the above. Instead, you only need a simple assignment: -
    -	data[real_i] += 1/C;
-
    -

    -[Go Top] -

    - -Q: There seems to be a zero division ? -
    -

    -We intend -to have this zero division. -Under the IEEE floating point standard, zero -division will returns infinity. Then with the -operations later to bound it, things go back to normal numbers without any problem. - -In general no warning messages happen. On some computers, you may -need to add an option (e.g. -mieee on alpha). - -Reasons of doing so are described in libsvm document. -

    -[Go Top] -

    - -Q: How do I choose parameters for one-class svm as training data are in only one class? -
    -

    -You have pre-specified true positive rate in mind and then search for -parameters which achieve similar cross-validation accuracy. -

    -[Go Top] -

    - -Q: Why training a probability model (i.e., -b 1) takes longer time -
    -

    -To construct this probability model, we internaly conduct a -cross validation, which is more time consuming than -a regular training. -Hence, in general you do parameter selection first without --b 1. You only use -b 1 when good parameters have been -selected. In other words, you avoid using -b 1 and -v -together. -

    -[Go Top] -

    - -Q: Why using the -b option does not give me better accuracy? -
    -

    -There is absolutely no reason the probability outputs guarantee -you better accuracy. The main purpose of this option is -to provide you the probability estimates, but not to boost -prediction accuracy. From our experience, -after proper parameter selections, in general with -and without -b have similar accuracy. Occasionally there -are some differences. -It is not recommended to compare the two under -just a fixed parameter -set as more differences will be observed. -

    -[Go Top] -

    - -Q: Why the code gives NaN (not a number) results? -
    -

    -This rarely happens, but few users reported the problem. -It seems that their -computers for training libsvm have the VPN client -running. The VPN software has some bugs and causes this -problem. Please try to close or disconnect the VPN client. -

    -[Go Top] -

    - -Q: Why on windows sometimes grid.py fails? -
    -

    -The error message is probably -

    -Traceback (most recent call last):
-  File "grid.py", line 349, in ?
-    main()
-  File "grid.py", line 344, in main
-    redraw(db)
-  File "grid.py", line 132, in redraw
-    gnuplot.write("set term windows\n")
-IOError: [Errno 22] Invalid argument
-
    - -

    There are some problems about using gnuplot on windows. -So far we have not found a good solution. -Please try to close gnuplot windows and rerun. -If the problem still occurs, comment the following -two lines in grid.py by inserting "#" in the beginning: -

    -        redraw(db)
-        redraw(db,1)
-
    -Then you get accuracy only but not cross validation contours. -

    -[Go Top] -

    - -Q: How can I save images drawn by svm-toy? -
    -

    -For Microsoft windows, first press the "print screen" key on the keyboard. -Open "Microsoft Paint" -(included in Windows) -and press "ctrl-v." Then you can clip -the part of picture which you want. -For X windows, you can -use the program "xv" to grab the picture of the svm-toy window. -

    -[Go Top] -

    - -Q: I press the "load" button to load data points but why svm-toy does not draw them ? -
    -

    -The program svm-toy assumes both attributes (i.e. x-axis and y-axis -values) are in (0,1). Hence you want to scale your -data to between a small positive number and -a number less than but very close to 1. -Moreover, class labels must be 1, 2, or 3 -(not 1.0, 2.0 or anything else). -

    -[Go Top] -

    - -Q: I would like svm-toy to handle more than three classes of data, what should I do ? -
    -

    -Taking windows/svm-toy.cpp as an example, you need to -modify it and the difference -from the original file is as the following: (for five classes of -data) -

    -30,32c30
-< 	RGB(200,0,200),
-< 	RGB(0,160,0),
-< 	RGB(160,0,0)
----
-> 	RGB(200,0,200)
-39c37
-< HBRUSH brush1, brush2, brush3, brush4, brush5;
----
-> HBRUSH brush1, brush2, brush3;
-113,114d110
-< 	brush4 = CreateSolidBrush(colors[7]);
-< 	brush5 = CreateSolidBrush(colors[8]);
-155,157c151
-< 	else if(v==3) return brush3;
-< 	else if(v==4) return brush4;
-< 	else return brush5;
----
-> 	else return brush3;
-325d318
-< 	  int colornum = 5;
-327c320
-< 		svm_node *x_space = new svm_node[colornum * prob.l];
----
-> 		svm_node *x_space = new svm_node[3 * prob.l];
-333,338c326,331
-< 			x_space[colornum * i].index = 1;
-< 			x_space[colornum * i].value = q->x;
-< 			x_space[colornum * i + 1].index = 2;
-< 			x_space[colornum * i + 1].value = q->y;
-< 			x_space[colornum * i + 2].index = -1;
-< 			prob.x[i] = &x_space[colornum * i];
----
-> 			x_space[3 * i].index = 1;
-> 			x_space[3 * i].value = q->x;
-> 			x_space[3 * i + 1].index = 2;
-> 			x_space[3 * i + 1].value = q->y;
-> 			x_space[3 * i + 2].index = -1;
-> 			prob.x[i] = &x_space[3 * i];
-397c390
-< 				if(current_value > 5) current_value = 1;
----
-> 				if(current_value > 3) current_value = 1;
-
    -

    -[Go Top] -

    - -Q: What is the difference between Java version and C++ version of libsvm? -
    -

    -They are the same thing. We just rewrote the C++ code -in Java. -

    -[Go Top] -

    - -Q: Is the Java version significantly slower than the C++ version? -
    -

    -This depends on the VM you used. We have seen good -VM which leads the Java version to be quite competitive with -the C++ code. (though still slower) -

    -[Go Top] -

    - -Q: While training I get the following error message: java.lang.OutOfMemoryError. What is wrong? -
    -

    -You should try to increase the maximum Java heap size. -For example, -

    -java -Xmx256m svm_train.java ...
-
    -sets the maximum heap size to 256M. -

    -[Go Top] -

    - -Q: Why you have the main source file svm.m4 and then transform it to svm.java? -
    -

    -Unlike C, Java does not have a preprocessor built-in. -However, we need some macros (see first 3 lines of svm.m4). - - -

    -[Go Top] -

    - -Q: Using python on MS windows, it fails to load the dll file. -
    -

    -It seems the dll file is version dependent. So far we haven't -found out a good solution. Please email us if you have any -good suggestions. -

    -[Go Top] -

    - -Q: How to modify the python interface on MS windows and rebuild the dll file ? -
    -

    - -To modify the interface, follow the instructions given in - -http://www.swig.org/Doc1.1/HTML/Python.html#n2 - -

    - -If you just want to build DLL for a different python version, -after libsvm 2.5, you can easily use the file Makefile.win. -See details in README. - -Alternatively, you can use Visual C++: -

      -
    1. Create a Win32 DLL project and set (Build->Active Configuration) to "Release." -
    2. Add svm.cpp, svmc_wrap.c, python2x.lib to your project. -
    3. Add the directories containing Python.h and svm.h to -the Additional include directories. (in Project -Settings->C/C++->Preprocessor) -
    4. add __WIN32__ to Preprocessor definitions -
    5. Make sure that in the "General" category of -Project->Settings->Settings For "Win32 Release", Output -directories should be "Release" -
    6. Build the DLL. -
    - - - -

    -[Go Top] -

    - -Q: Except the python-C++ interface provided, could I use Jython to call libsvm ? -
    -

    Yes, here are some examples: - -

    -$ export CLASSPATH=$CLASSPATH:~/libsvm-2.4/java/libsvm.jar
-$ ./jython
-Jython 2.1a3 on java1.3.0 (JIT: jitc)
-Type "copyright", "credits" or "license" for more information.
->>> from libsvm import *
->>> dir()
-['__doc__', '__name__', 'svm', 'svm_model', 'svm_node', 'svm_parameter',
-'svm_problem']
->>> x1 = [svm_node(index=1,value=1)]
->>> x2 = [svm_node(index=1,value=-1)]
->>> param = svm_parameter(svm_type=0,kernel_type=2,gamma=1,cache_size=40,eps=0.001,C=1,nr_weight=0,shrinking=1)
->>> prob = svm_problem(l=2,y=[1,-1],x=[x1,x2])
->>> model = svm.svm_train(prob,param)
-*
-optimization finished, #iter = 1
-nu = 1.0
-obj = -1.018315639346838, rho = 0.0
-nSV = 2, nBSV = 2
-Total nSV = 2
->>> svm.svm_predict(model,x1)
-1.0
->>> svm.svm_predict(model,x2)
--1.0
->>> svm.svm_save_model("test.model",model)
-
-
    - -

    -[Go Top] -

    - -Q: How could I install the python interface on Mac OS? -
    -

    According to S V N Vishwanathan in Australian National University, instead of -LDFLAGS = -shared -in the Makefile, you need -

    -LDFLAGS = -bundle -flat_namespace -undefined suppress
-
    -

    -[Go Top] -

    -

    -LIBSVM home page -

    - - diff --git a/src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile b/src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile deleted file mode 100644 index 452a810..0000000 --- a/src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -CXXC = g++ -CFLAGS = -Wall -O3 - -all: svm-train svm-predict svm-scale - -svm-predict: svm-predict.c svm.o - $(CXXC) $(CFLAGS) svm-predict.c svm.o -o svm-predict -lm -svm-train: svm-train.c svm.o - $(CXXC) $(CFLAGS) svm-train.c svm.o -o svm-train -lm -svm-scale: svm-scale.c - $(CXXC) $(CFLAGS) svm-scale.c -o svm-scale -svm.o: svm.cpp svm.h - $(CXXC) $(CFLAGS) -c svm.cpp -clean: - rm -f *~ svm.o svm-train svm-predict svm-scale diff --git a/src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile.win b/src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile.win deleted file mode 100644 index 74733af..0000000 --- a/src/Algorithms/orensemble/orensemble/libsvm-2.81/Makefile.win +++ /dev/null @@ -1,39 +0,0 @@ -#You must ensure nmake.exe, cl.exe, link.exe are in system path. -#VCVARS32.bat -#Under dosbox prompt -#nmake -f Makefile.win - -PYTHON_INC = c:\python24\include -PYTHON_LIB = c:\python24\libs\python24.lib -########################################## -CXXC = cl.exe -CFLAGS = -nologo -O2 -GX -I. -D __WIN32__ -TARGET = windows - -all: $(TARGET)\svmtrain.exe $(TARGET)\svmpredict.exe $(TARGET)\svmscale.exe $(TARGET)\svmtoy.exe - -python: $(TARGET)\python\svmc.dll - -$(TARGET)\svmpredict.exe: svm.h svm-predict.c svm.obj - $(CXXC) $(CFLAGS) svm-predict.c svm.obj -o $(TARGET)\svmpredict.exe - -$(TARGET)\svmtrain.exe: svm.h svm-train.c svm.obj - $(CXXC) $(CFLAGS) svm-train.c svm.obj -o $(TARGET)\svmtrain.exe - -$(TARGET)\svmscale.exe: svm.h svm-scale.c - $(CXXC) $(CFLAGS) svm-scale.c -o $(TARGET)\svmscale.exe - -$(TARGET)\svmtoy.exe: svm.h svm.obj svm-toy\windows\svm-toy.cpp - $(CXXC) $(CFLAGS) svm-toy\windows\svm-toy.cpp svm.obj user32.lib gdi32.lib comdlg32.lib -o $(TARGET)\svmtoy.exe - -svm.obj: svm.cpp svm.h - $(CXXC) $(CFLAGS) -c svm.cpp - -$(TARGET)\python\svmc.dll: python\svmc_wrap.c svm.obj $(PYTHON_LIB) - $(CXXC) $(CFLAGS) -I$(PYTHON_INC) -ML -LD $** -o $(TARGET)\python\svmc.dll -# $** means all dependencies - -clean: - -erase /Q *.obj $(TARGET)\. $(TARGET)\python\. - - diff --git a/src/Algorithms/orensemble/orensemble/libsvm-2.81/README b/src/Algorithms/orensemble/orensemble/libsvm-2.81/README deleted file mode 100644 index 7b28411..0000000 --- a/src/Algorithms/orensemble/orensemble/libsvm-2.81/README +++ /dev/null @@ -1,524 +0,0 @@ -Libsvm is a simple, easy-to-use, and efficient software for SVM -classification and regression. It can solve C-SVM classification, -nu-SVM classification, one-class-SVM, epsilon-SVM regression, and -nu-SVM regression. It also provides an automatic model selection -tool for C-SVM classification. This document explains the use of -libsvm. - -Libsvm is available at -http://www.csie.ntu.edu.tw/~cjlin/libsvm -Please read the COPYRIGHT file before using libsvm. - -Quick Start -=========== - -If you are new to SVM and if the data is not large, please go to -tools directory and use easy.py after installation. It does -everything automatic -- from data scaling to parameter selection. - -Usage: easy.py training_file [testing_file] - -More information about parameter selction can be found in -tools/README. - -Installation -============ - -On Unix systems, type `make' to build the `svm-train' and `svm-predict' -programs. Run them without arguments to show the usages of them. - -On other systems, consult `Makefile' to build them (e.g., see -'Building Windows binaries' in this file) or use the pre-built -binaries (Windows binaries are in the directory `windows'). - -The format of training and testing data file is: - -