Skip to content
This repository has been archived by the owner on Nov 16, 2019. It is now read-only.

Commit

Permalink
Merge pull request #174 from yahoo/jun_cos_layer
Browse files Browse the repository at this point in the history 
enhance cos layer
  • Loading branch information
@anfeng
anfeng authored Oct 26, 2016
2 parents 96eab59 + 49da8f7 commit 081407c
Show file tree
Hide file tree
Showing 6 changed files with 133 additions and 75 deletions.
1 change: 1 addition & 0 deletions caffe-distri/include/extra_layers.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class CoSDataLayer : public Layer<Dtype> {
vector<int> channels_;
vector<int> height_;
vector<int> width_;
vector<int> sample_num_axes_;
vector<Dtype*> data_;
};

Expand Down
28 changes: 22 additions & 6 deletions caffe-distri/src/main/cpp/cos_data_layer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,29 @@ void CoSDataLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
height_.resize(top_size_);
width_.resize(top_size_);
data_.resize(top_size_);
sample_num_axes_.resize(top_size_);
for (int i = 0; i < top_size_; i++) {
channels_[i] = cos_data_param.top(i).channels();
height_[i] = cos_data_param.top(i).height();
width_[i] = cos_data_param.top(i).width();
top[i]->Reshape(batch_size_, channels_[i], height_[i], width_[i]);
sample_num_axes_[i] = cos_data_param.top(i).sample_num_axes();
CHECK_LE(sample_num_axes_[i], 3) << "sample number of axes "
<< "should be less or equal to 3";
channels_[i] = cos_data_param.top(i).out_channels() == 0 ?
cos_data_param.top(i).channels() : cos_data_param.top(i).out_channels();
height_[i] = cos_data_param.top(i).out_height() == 0 ?
cos_data_param.top(i).height() : cos_data_param.top(i).out_height();
width_[i] = cos_data_param.top(i).out_width() == 0 ?
cos_data_param.top(i).width() : cos_data_param.top(i).out_width();
bool transpose = cos_data_param.top(i).transpose();
if (transpose) {
CHECK_EQ(sample_num_axes_[i], 1)
<< "each sample must be a 1d-vector if transpose is used";
int shape[] = {channels_[i], batch_size_};
top[i]->Reshape(vector<int>(shape, shape + 2));
} else {
int shape[] = {batch_size_, channels_[i], height_[i], width_[i]};
// + 1 due to batch_size axis
top[i]->Reshape(vector<int> (shape, shape + sample_num_axes_[i] + 1));
}
LOG(INFO) << "CoSDataLayer Top #" << i << " " << top[i]->shape_string();
}
}

Expand All @@ -44,7 +62,6 @@ void CoSDataLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
for (int i = 0; i < top_size_; i++) {
CHECK(data_[i]) <<
"CoSDataLayer needs to be initialized by calling Reset";
top[i]->Reshape(batch_size_, channels_[i], height_[i], width_[i]);
top[i]->set_cpu_data(data_[i]);
}
}
Expand All @@ -55,7 +72,6 @@ void CoSDataLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
for (int i = 0; i < top_size_; i++) {
CHECK(data_[i]) <<
"CoSDataLayer needs to be initialized by calling Reset";
top[i]->Reshape(batch_size_, channels_[i], height_[i], width_[i]);
top[i]->set_gpu_data(data_[i]);
}
}
Expand Down
3 changes: 2 additions & 1 deletion caffe-grid/src/main/scala/com/yahoo/ml/caffe/CaffeProcessor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,8 @@ private[caffe] class CaffeProcessor[T1, T2](val sources: Array[DataSource[T1, T2
}

case CoSDataParameter.DataType.RAW_IMAGE |
CoSDataParameter.DataType.ENCODED_IMAGE => {
CoSDataParameter.DataType.ENCODED_IMAGE |
CoSDataParameter.DataType.ENCODED_IMAGE_WITH_DIM=> {
if (transformers(i) != null) {
transformers(i).transform(dataArray(i).asInstanceOf[MatVector], data(i))
} else {
Expand Down
168 changes: 107 additions & 61 deletions caffe-grid/src/main/scala/com/yahoo/ml/caffe/DataFrameSource.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import caffe.Caffe._
import com.yahoo.ml.jcaffe._
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.spark.sql.{Row, DataFrame, SQLContext}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.mutable.WrappedArray
Expand Down Expand Up @@ -52,15 +52,8 @@ class DataFrameSource(conf: Config, layerId: Int, isTrain: Boolean)
numTops = cosDataParam.getTopCount()
tops = new Array[Top](numTops)
for (i <- 0 until numTops) {
val top = cosDataParam.getTop(i)
val dataType: CoSDataParameter.DataType = top.getType()
val name: String = top.getName()
val channels: Int = top.getChannels()
val width: Int = top.getWidth()
val height: Int = top.getHeight()
val transformParam: TransformationParameter =
if (top.hasTransformParam()) top.getTransformParam() else null
tops(i) = new Top(dataType, name, channels, width, height, transformParam)
val topParam = cosDataParam.getTop(i)
tops(i) = new Top(topParam)
}

batchSize_ = cosDataParam.getBatchSize()
Expand Down Expand Up @@ -114,47 +107,68 @@ class DataFrameSource(conf: Config, layerId: Int, isTrain: Boolean)
}

/* helper function to set up a float blob */
def setFloatBlob(offset: Int, data: Array[Float], blob: FloatBlob): Unit = {
if (data.length + offset > blob.count()) {
def setFloatBlob(offset: Int, stride: Int, data: Array[Float],
blob: FloatBlob): Unit = {
val dataLen: Int = (data.length - 1) * stride + offset + 1
if (dataLen > blob.count()) {
throw new IllegalArgumentException("blob size is "
+ blob.count() + ", but total data length is "
+ (data.length + offset) + ".")
+ dataLen + ".")
}
val blobCPU = blob.cpu_data()
for (i <- 0 until data.length) {
val index = offset + i
val index = offset + i * stride
blobCPU.set(index, data(i))
}
}

/* help function to set up a matvector */
def setMatVector(offset: Int, topidx: Int, data: Array[Byte],
mats: MatVector, encoded: Boolean): Unit = {
def setMatVector(offset: Int, topidx: Int, data: Array[Byte], rawHeight: Int,
rawWidth: Int, mats: MatVector, encoded: Boolean): Unit = {
var mat: Mat = null
var oldmat: Mat = null
val imageHeight = tops(topidx).height_
val imageWidth = tops(topidx).width_
val imageChannels = tops(topidx).channels_
val rawDim: Boolean = (rawHeight > 0) && (rawWidth > 0)

if (encoded) {
// encoded image
mat = new Mat(data)
imageChannels match {
case 1 => mat.decode(Mat.CV_LOAD_IMAGE_GRAYSCALE)
case 3 => mat.decode(Mat.CV_LOAD_IMAGE_COLOR)
case _ => mat.decode(Mat.CV_LOAD_IMAGE_UNCHANGED)
}

if (mat.width() == 0) {
if (rawDim && ((mat.height() != rawHeight) ||
(mat.width() != rawWidth))) {
// raw dimension provided but mat does not match them
log.warn("Skip image at top#" + topidx + ", index#" + offset)
mat = null
}
} else {
mat = new Mat(imageChannels, imageHeight, imageWidth, data)
if (mat.width() != imageWidth || mat.height() != imageHeight) {
log.warn("Skip image at top#" + topidx + ", index#" + offset)
mat = null
// raw image
if (rawDim) {
// raw image provided
mat = new Mat(imageChannels, rawHeight, rawWidth, data)
if (mat.width() != rawWidth || mat.height() != rawHeight) {
log.warn("Skip image at top#" + topidx + ", index#" + offset)
mat = null
}
} else {
// raw image not provided, use dim from proto instead
mat = new Mat(imageChannels, imageHeight, imageWidth, data)
if (mat.width() != imageWidth || mat.height() != imageHeight) {
log.warn("Skip image at top#" + topidx + ", index#" + offset)
mat = null
}
}
}
if (mat != null) {
if (conf.resize && ((mat.height() != imageHeight) ||
(mat.width() != imageWidth))) {
mat.resize(imageHeight, imageWidth)
}
oldmat = mats.put(offset, mat)
if (oldmat != null)
oldmat.deallocate()
Expand All @@ -165,23 +179,26 @@ class DataFrameSource(conf: Config, layerId: Int, isTrain: Boolean)
def dummyDataHolder(): Array[Any] = {
val holder = new Array[Any](numTops)
for (i <- 0 until numTops) {
val dataType = getTopDataType(i)
val channels = getTopChannel(i)
val height = getTopHeight(i)
val width = getTopWidth(i)
val dataType = tops(i).dataType_
val sampleShape = tops(i).sampleShape_
val transpose = tops(i).transpose_
// if transpose is on, blobShape has exactly 2 axes.
val blobShape = if (transpose) Array(sampleShape(0), batchSize_)
else batchSize_ +: sampleShape
holder(i) = dataType match {
case CoSDataParameter.DataType.STRING |
CoSDataParameter.DataType.INT |
CoSDataParameter.DataType.FLOAT |
CoSDataParameter.DataType.INT_ARRAY |
CoSDataParameter.DataType.FLOAT_ARRAY => {
val blob = new FloatBlob()
blob.reshape(batchSize, channels, height, width)
blob.reshape(blobShape)
blob
}
case CoSDataParameter.DataType.RAW_IMAGE |
CoSDataParameter.DataType.ENCODED_IMAGE =>
new MatVector(batchSize)
CoSDataParameter.DataType.ENCODED_IMAGE |
CoSDataParameter.DataType.ENCODED_IMAGE_WITH_DIM =>
new MatVector(batchSize_)
}
}
holder
Expand All @@ -190,11 +207,13 @@ class DataFrameSource(conf: Config, layerId: Int, isTrain: Boolean)
def dummyDataBlobs(): Array[FloatBlob] = {
val blob = new Array[FloatBlob](numTops)
for (i <- 0 until numTops) {
val channels = getTopChannel(i)
val height = getTopHeight(i)
val width = getTopWidth(i)
val sampleShape = tops(i).outSampleShape_
val transpose = tops(i).transpose_
// if transpose is on, blobShape has exactly 2 axes.
val blobShape = if (transpose) Array(sampleShape(0), batchSize_)
else batchSize_ +: sampleShape
blob(i) = new FloatBlob()
blob(i).reshape(batchSize_, channels, height, width)
blob(i).reshape(blobShape)
}
blob
}
Expand All @@ -220,47 +239,58 @@ class DataFrameSource(conf: Config, layerId: Int, isTrain: Boolean)
sample._2(i).asInstanceOf[String].toFloat)
val blob: FloatBlob = batchData(i).asInstanceOf[FloatBlob]
val offset: Int = count
setFloatBlob(offset, data, blob)
setFloatBlob(offset, 1, data, blob)
}
case CoSDataParameter.DataType.INT => {
val data: Array[Float] = Array(
sample._2(i).asInstanceOf[Int].toFloat)
val blob: FloatBlob = batchData(i).asInstanceOf[FloatBlob]
val offset: Int = count
setFloatBlob(offset, data, blob)
setFloatBlob(offset, 1, data, blob)
}
case CoSDataParameter.DataType.FLOAT => {
val data: Array[Float] = Array(sample._2(i).asInstanceOf[Float])
val blob: FloatBlob = batchData(i).asInstanceOf[FloatBlob]
val offset: Int = count
setFloatBlob(offset, data, blob)
setFloatBlob(offset, 1, data, blob)
}
case CoSDataParameter.DataType.INT_ARRAY => {
val data: WrappedArray[Int] = sample._2(
i).asInstanceOf[WrappedArray[Int]]
val floatData: Array[Float] = data.toArray.map(x => x.toFloat)
val blob: FloatBlob = batchData(i).asInstanceOf[FloatBlob]
val offset = count * tops(i).getSize()
setFloatBlob(offset, floatData, blob)
val offset = if (tops(i).transpose_) count else count * tops(i).size_
val stride = if (tops(i).transpose_) batchSize_ else 1
setFloatBlob(offset, stride, floatData, blob)
}
case CoSDataParameter.DataType.FLOAT_ARRAY => {
val data: Array[Float] = sample._2(
i).asInstanceOf[WrappedArray[Float]].toArray
val blob: FloatBlob = batchData(i).asInstanceOf[FloatBlob]
val offset = count * tops(i).getSize()
setFloatBlob(offset, data, blob)
val offset = if (tops(i).transpose_) count else count * tops(i).size_
val stride = if (tops(i).transpose_) batchSize_ else 1
setFloatBlob(offset, stride, data, blob)
}
case CoSDataParameter.DataType.ENCODED_IMAGE => {
val data: Array[Byte] = sample._2(i).asInstanceOf[Array[Byte]]
val mats: MatVector = batchData(i).asInstanceOf[MatVector]
val offset: Int = count
setMatVector(offset, i, data, mats, true)
setMatVector(offset, i, data, -1, -1, mats, true)
}
case CoSDataParameter.DataType.RAW_IMAGE => {
val data: Array[Byte] = sample._2(i).asInstanceOf[Array[Byte]]
val mats: MatVector = batchData(i).asInstanceOf[MatVector]
val offset: Int = count
setMatVector(offset, i, data, mats, false)
setMatVector(offset, i, data, -1, -1, mats, false)
}
case CoSDataParameter.DataType.ENCODED_IMAGE_WITH_DIM => {
val row: Row = sample._2(i).asInstanceOf[Row]
val data: Array[Byte] = row.getAs[WrappedArray[Byte]]("image").toArray
val height: Int = row.getAs[Int]("height")
val width: Int = row.getAs[Int]("width")
val mats: MatVector = batchData(i).asInstanceOf[MatVector]
val offset: Int = count
setMatVector(offset, i, data, height, width, mats, true)
}
case _ => throw new Exception("Unsupported data type for CoS Data Layer")
}
Expand All @@ -280,28 +310,44 @@ class DataFrameSource(conf: Config, layerId: Int, isTrain: Boolean)

override def getTopTransformParam(index: Int): TransformationParameter
= tops(index).transformParam_

override def getTopHeight(index: Int): Int = tops(index).height_

override def getTopWidth(index: Int): Int = tops(index).width_

override def getTopChannel(index: Int): Int = tops(index).channels_
}


class Top(dataType:CoSDataParameter.DataType, name:String, channels:Int,
width:Int, height:Int, transformParam: TransformationParameter) {
val dataType_ :CoSDataParameter.DataType = dataType
val name_ :String = name
val channels_ :Int = channels
val width_ :Int = width
val height_ :Int = height
val transformParam_ :TransformationParameter = transformParam

def getSize() : Int = {
return width_ * height_ * channels_
class Top(topParam: CoSDataParameter.TopBlob) {
val dataType_ :CoSDataParameter.DataType = topParam.getType()
val name_ :String = topParam.getName()
// pre-transformer (if exists) blob dimensions
val channels_ :Int = topParam.getChannels()
val width_ :Int = topParam.getWidth()
val height_ :Int = topParam.getHeight()
val sampleNumAxes_ :Int = topParam.getSampleNumAxes()
// post-transformer (if exists) blob dimensions
val outChannels_ :Int = if (topParam.getOutChannels() == 0) channels_
else topParam.getOutChannels()
val outHeight_ :Int = if (topParam.getOutHeight() == 0) height_
else topParam.getOutHeight()
val outWidth_ :Int = if (topParam.getOutWidth() == 0) width_
else topParam.getOutWidth()
val transformParam_ :TransformationParameter =
if (topParam.hasTransformParam()) topParam.getTransformParam() else null
val shape_ :Array[Int] = Array(channels_, height_, width_)
val outShape_ :Array[Int] = Array(outChannels_, outHeight_, outWidth_)
// pre-transformer (if exists) blob shape
val sampleShape_ :Array[Int] = shape_.slice(0,sampleNumAxes_)
// post-transformer (if exists) blob shape
val outSampleShape_ :Array[Int] = outShape_.slice(0,sampleNumAxes_)
// pre-transformer (if exists) blob shape
val size_ :Int = sampleShape_.foldLeft(1)(_*_)
// post-transformer (if exists) blob shape
val outSize_ :Int = outSampleShape_.foldLeft(1)(_*_)
// flag to transpose the blob
val transpose_ :Boolean = topParam.getTranspose()
if (transpose_) {
assert(sampleNumAxes_ == 1)
}


override def toString = s"Top(type=$dataType_, name=$name_, " +
s"channels=$channels_, width=$width_, height=$height_)"
s"channels=$channels_, height=$height_, width=$width_, " +
s"outChannels=$outChannels_, outHeight=$outHeight_, outWidth=$outWidth_, " +
s"sampleNumAxes=$sampleNumAxes_, size=$size_, outSize=$outSize_)"
}
6 changes: 0 additions & 6 deletions caffe-grid/src/main/scala/com/yahoo/ml/caffe/DataSource.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,12 +125,6 @@ abstract class DataSource[T1, T2](val conf: Config, val layerId : Int, val isTra
def getTopDataType(index: Int): CoSDataParameter.DataType = null

def getTopTransformParam(index: Int): TransformationParameter = null

def getTopHeight(index: Int): Int = 0

def getTopWidth(index: Int): Int = 0

def getTopChannel(index: Int): Int = 0
}

object DataSource extends Serializable {
Expand Down
2 changes: 1 addition & 1 deletion caffe-public
Submodule caffe-public updated 2 files
+1 −2 src/caffe/net.cpp
+19 −6 src/caffe/proto/caffe.proto

0 comments on commit 081407c

Please sign in to comment.