-
Notifications
You must be signed in to change notification settings - Fork 150
/
aws_caffe_flow_miniGoogleNet.sh
88 lines (66 loc) · 4.53 KB
/
aws_caffe_flow_miniGoogleNet.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
#!/bin/sh
CAFFE_ROOT=$HOME/caffe_tools/BVLC1v0-Caffe
CAFFE_TOOLS_DIR=$CAFFE_ROOT/distribute
WORK_DIR=$HOME/ML/cifar10/caffe #working dir
MOD_NUM=3 # model number
NUMIT=40000 # number of iterations
NET=miniGoogleNet
# ################################################################################################################
# SCRIPTS 1 2 3 (DATABASE AND MEAN VALUES)
: '
# go to TensorFlow environment, load the database from Keras and write it as JPEG images
source activate tensorflow_p27
export PYTHONPATH=/usr/local/lib/python2.7/dist-packages/:/home/ubuntu/anaconda3/envs/caffe_p27/lib/python2.7/site-packages/:$PYTHONPATH #needed for opencv
echo "DATABASE: training and validation in LMDB, test in JPG and MEAN values"
python $WORK_DIR/code/1_write_cifar10_images.py #--pathname /home/danieleb/ML/cifar10/input/cifar10_jpg
export PYTHONPATH=/home/ubuntu/src/cntk/bindings/python
source deactivate tensorflow_p27
'
# go to Caffe environment
source activate caffe_p27
#export PYTHONPATH=/usr/local/lib/python2.7/dist-packages/:$PYTHONPATH
#export LD_LIBRARY_PATH=/home/ubuntu/src/caffe_python_2/build/install/lib64/:$LD_LIBRARY_PATH
: '
#create LMDB databases -training (50K), validation (9K), test (1K) images - and compute mean values
python $WORK_DIR/code/2a_create_lmdb.py
# DO NOT RUN BELOW COMMAND AS mean.binaryproto is already available! Is is here only for reference
##python $WORK_DIR/code/2b_compute_mean.py
#check goodness of LMDB databases (just for debug: you can skip it)
python $WORK_DIR/code/3_read_lmdb.py
'
# ################################################################################################################
# SCRIPT 4 (SOLVER AND TRAINING AND LEARNING CURVE)
echo "TRAINING. Remember that: <Epoch_index = floor((iteration_index * batch_size) / (# data_samples))>"
#$CAFFE_TOOLS_DIR/bin/caffe.bin train --solver $WORK_DIR/models/$NET/m$MOD_NUM/solver_$MOD_NUM\_$NET.prototxt \
# 2>&1 | tee $WORK_DIR/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log
cd $HOME/ML
python $WORK_DIR/code/4_training.py -s cifar10/caffe/models/$NET/m$MOD_NUM/solver_$MOD_NUM\_$NET.prototxt -l cifar10/caffe/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log
# # example of trainining the CNN from a certain snapshot
#echo "RETRAINING from previous snapshot"
# #$CAFFE_TOOLS_DIR/bin/caffe.bin train --solver $WORK_DIR/models/$NET/m$MOD_NUM/solver_$MOD_NUM\_$NET.prototxt \
# --snapshot $WORK_DIR/models/$NET/m3/snapshot_3\$NET__iter_20000.solverstate \
# 2>&1 | tee $WORK_DIR/models/$NET/m$MOD_NUM/retrain_logfile_$MOD_NUM\_$NET.log
#cp -f $WORK_DIR/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log $WORK_DIR/models/$NET/m$MOD_NUM/orig_logfile_$MOD_NUM\_$NET.log
#cp -f $WORK_DIR/models/$NET/m$MOD_NUM/retrain_logfile_$MOD_NUM\_$NET.log $WORK_DIR/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log
# print image of CNN architecture
echo "PRINT CNN BLOCK DIAGRAM"
python $CAFFE_TOOLS_DIR/python/draw_net.py $WORK_DIR/models/$NET/m$MOD_NUM/train_val_$MOD_NUM\_$NET.prototxt $WORK_DIR/models/$NET/m$MOD_NUM/bd_$MOD_NUM\_$NET.png
# ################################################################################################################
# SCRIPT 5: plot the learning curve
echo "PLOT LEARNING CURVERS"
python $WORK_DIR/code/5_plot_learning_curve.py $WORK_DIR/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log $WORK_DIR/models/$NET/m$MOD_NUM/plt_train_val_$MOD_NUM\_$NET.png
## 0 Test Accuracy vs Iters
## 1 Test Accuracy vs Seconds
## 2 Test Loss vs Iters
## 3 Test Loss vs Seconds
## 4 Train lr vs. Iters
## 5 Train lr vs. Seconds
## 6 Train Loss vs Iters
## 7 Train Loss vs Seconds
#python $WORK_DIR/code/plot_training_log.py 6 $WORK_DIR/models/$NET/m$MOD_NUM/plt_trainLoss_$MOD_NUM\_$NET.png $WORK_DIR/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log
#python $WORK_DIR/code/plot_training_log.py 2 $WORK_DIR/models/$NET/m$MOD_NUM/plt_testLoss_$MOD_NUM\_$NET.png $WORK_DIR/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log
#python $WORK_DIR/code/plot_training_log.py 0 $WORK_DIR/models/$NET/m$MOD_NUM/plt_testAccuracy_$MOD_NUM\_$NET.png $WORK_DIR/models/$NET/m$MOD_NUM/logfile_$MOD_NUM\_$NET.log
# ################################################################################################################
# SCRIPT 6 (PREDICTION)
echo "COMPUTE PREDICTIONS"
python $WORK_DIR/code/6_make_predictions.py -d cifar10/caffe/models/$NET/m$MOD_NUM/deploy_$MOD_NUM\_$NET.prototxt -w cifar10/caffe/models/$NET/m$MOD_NUM/snapshot_$MOD_NUM\_$NET\__iter_$NUMIT.caffemodel 2>&1 | tee cifar10/caffe/models/$NET/m$MOD_NUM/predictions_$MOD_NUM\_$NET.txt