Skip to content

Commit

Permalink
Add Bias and Activations (#15)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cancan101
cancan101 authored Mar 13, 2017
1 parent bcf2e70 commit 9994aea
Show file tree
Hide file tree
Showing 3 changed files with 143 additions and 11 deletions.
90 changes: 88 additions & 2 deletions tensorflow/contrib/ios_examples/bnns/test_python.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,27 @@ def test_same_padding(self):
padding = 'SAME'
self._run_test(padding, strides)

def _run_test(self, padding, strides):
images = np.arange(180.).reshape((3, 5, 3, 4)).astype(np.float32)
def test_linear_activation(self):
strides = [1,1,2,1]
padding = 'VALID'
self._run_test(padding, strides, activation_function='Linear')

def _run_test(self, padding, strides, activation_function='Linear'):
images = (np.arange(180.)-90.).reshape((3, 5, 3, 4)).astype(np.float32)
images_t = images.transpose(0, 2, 3, 1)
filters = np.zeros((3, 3, 5, 2), np.float32)
filters[:, :, 0, 0] = 1
filters[:, :, 0, 1] = 2
with self.test_session():
# TODO: Figure out if there is someway for bias to be optional:
# http://stackoverflow.com/questions/42754965/marking-input-as-optional-in-tensorflow
bnns_output = self.bnns_module.conv2dbnns(
images,
filters,
bias=[],
strides=strides,
padding=padding,
activation_function=activation_function,
data_format='NCHW',
).eval()
with self.test_session():
Expand All @@ -62,6 +71,83 @@ def _run_test(self, padding, strides):
)


class TestConv2dBnnsBias(TestBnns):
def test_valid_padding(self):
strides = [1,1,1,1]
padding = 'VALID'
self._run_test(padding, strides)

def _run_test(self, padding, strides):
images = (np.arange(180.)-90.).reshape((3, 5, 3, 4)).astype(np.float32)
images_t = images.transpose(0, 2, 3, 1)
filters = np.zeros((3, 3, 5, 2), np.float32)
filters[:, :, 0, 0] = 1
filters[:, :, 0, 1] = 2

bias = np.arange(1., 3.)

with self.test_session():
bnns_output = self.bnns_module.conv2dbnns(
images,
filters,
bias=bias,
strides=strides,
padding=padding,
data_format='NCHW',
).eval()
with self.test_session():
value = nn_ops.conv2d(
images_t,
filters,
strides=shape_nchw_to_nhwc(strides),
padding=padding,
)
gold_output = nn_ops.bias_add(
value,
bias,
).eval()
np.testing.assert_allclose(
bnns_output,
gold_output.transpose(0, 3, 1, 2),
)


class TestConv2dBnnsReLU(TestBnns):
def test_valid_padding(self):
strides = [1,1,1,1]
padding = 'VALID'
self._run_test(padding, strides)

def _run_test(self, padding, strides):
images = (np.arange(180.)-90.).reshape((3, 5, 3, 4)).astype(np.float32)
images_t = images.transpose(0, 2, 3, 1)
filters = np.zeros((3, 3, 5, 2), np.float32)
filters[:, :, 0, 0] = 1
filters[:, :, 0, 1] = 2

with self.test_session():
bnns_output = self.bnns_module.conv2dbnns(
images,
filters,
bias=[],
strides=strides,
padding=padding,
activation_function='ReLU',
data_format='NCHW',
).eval()
with self.test_session():
value = nn_ops.conv2d(
images_t,
filters,
strides=shape_nchw_to_nhwc(strides),
padding=padding,
)
gold_output = nn_ops.relu(value).eval()
np.testing.assert_allclose(
bnns_output,
gold_output.transpose(0, 3, 1, 2),
)

class TestMaxPoolBnns(TestBnns):
def test_valid_padding(self):
ksize = [1,1,3,3]
Expand Down
60 changes: 53 additions & 7 deletions tensorflow/core/user_ops/bnns_conv.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,41 @@ using namespace tensorflow;

typedef Eigen::ThreadPoolDevice CPUDevice;

bool BNNSActivationFunctionFromString(const string& activation_function_str, BNNSActivationFunction* bnnActivationFunction) {
if (activation_function_str == "Linear") {
*bnnActivationFunction = BNNSActivationFunctionIdentity;
return true;
} else if (activation_function_str == "ReLU") {
*bnnActivationFunction = BNNSActivationFunctionRectifiedLinear;
return true;
}
return false;
}


// Inspired by: https://github.com/tensorflow/tensorflow/blob/8746f8ac9e9ef652611180e0bf64466af2707b20/tensorflow/core/ops/nn_ops.cc#L503-L553
REGISTER_OP("Conv2DBNNS")
.Input("input: T")
.Input("filter: T")
.Input("bias: T")
.Output("output: T")
.Attr("T: {float}")
.Attr("strides: list(int)")
.Attr("strides: list(int) >= 4")
.Attr(GetPaddingAttrString())
.Attr(GetConvnetDataFormatAttrString())
.Attr("activation_function: {'Linear', 'ReLU'} = 'Linear'")
.SetShapeFn(shape_inference::Conv2DShape);

// Inspired by: https://github.com/tensorflow/tensorflow/blob/8746f8ac9e9ef652611180e0bf64466af2707b20/tensorflow/core/kernels/conv_ops.cc#L243-L391
// TODO: Fix usage of float rather than T
// Would be cool to handle half and ints
template <typename Device, typename T>
class Conv2DBNNSOp : public BinaryOp<T> {
class Conv2DBNNSOp : public OpKernel {
public:
explicit Conv2DBNNSOp(OpKernelConstruction* context) : BinaryOp<T>(context) {
explicit Conv2DBNNSOp(OpKernelConstruction* context) : OpKernel(context) {
const DataType dt = DataTypeToEnum<T>::v();
OP_REQUIRES_OK(context, context->MatchSignature({dt, dt, dt}, {dt}));

OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
string data_format;
OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
Expand All @@ -53,6 +70,10 @@ class Conv2DBNNSOp : public BinaryOp<T> {
errors::InvalidArgument("Current implementation does not yet support "
"strides in the batch and depth dimensions."));
OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
string activation_function;
OP_REQUIRES_OK(context, context->GetAttr("activation_function", &activation_function));
OP_REQUIRES(context, BNNSActivationFunctionFromString(activation_function, &bnnActivationFunction_),
errors::InvalidArgument("Invalid activation function"));
}

void Compute(OpKernelContext* context) override {
Expand Down Expand Up @@ -127,6 +148,21 @@ class Conv2DBNNSOp : public BinaryOp<T> {
GetWindowedOutputSize(input_cols, filter_cols, stride_cols,
padding_, &out_cols, &pad_cols));

const Tensor& bias = context->input(2);

OP_REQUIRES(context, TensorShapeUtils::IsVector(bias.shape()),
errors::InvalidArgument("Biases must be 1D: ",
bias.shape().DebugString()));

bool isBias = bias.shape().dim_size(0) > 0;
OP_REQUIRES(
context,
!isBias || bias.shape().dim_size(0) == out_depth,
errors::InvalidArgument(
"Must provide as many biases as the last dimension "
"of the input tensor: ",
bias.shape().DebugString(), " vs. ", input.shape().DebugString()));

// This should have this format, the actual shape is fixed to match format
TensorShape out_shape =
ShapeFromFormat(data_format_, batch, out_rows, out_cols, out_depth);
Expand Down Expand Up @@ -204,12 +240,21 @@ class Conv2DBNNSOp : public BinaryOp<T> {
context->eigen_device<Device>(), To32Bit(filter.tensor<T, 4>()),
To32Bit(transformed_filter.tensor<T, 4>()));

const float* weights = transformed_filter.flat<float>().data();
const T* weights = transformed_filter.flat<T>().data();

// Attach weight buffer to layer parameters
layer_params.weights.data = weights;
layer_params.weights.data_type = BNNSDataTypeFloat32;

// Attach bias buffer to layer parameters
if (isBias) {
const T* biasVector = bias.flat<T>().data();
layer_params.bias.data = biasVector;
layer_params.bias.data_type = BNNSDataTypeFloat32;
}

layer_params.activation.function = bnnActivationFunction_;

// Common filter parameters
BNNSFilterParameters filter_params;
bzero(&filter_params, sizeof(filter_params));
Expand All @@ -222,8 +267,8 @@ class Conv2DBNNSOp : public BinaryOp<T> {
OP_REQUIRES(context, filter_bnns != nullptr,
errors::Unknown("BNNSFilterCreateConvolutionLayer failed"));

const float* i_stack = input.flat<float>().data();
float* o_stack = output->flat<float>().data();
const T* i_stack = input.flat<T>().data();
T* o_stack = output->flat<T>().data();

// Apply filter to input stack. Result is written in output stack.
int status = BNNSFilterApplyBatch(filter_bnns, batch, i_stack, input_cols * input_rows * in_depth, o_stack, out_cols * out_rows * out_depth);
Expand All @@ -241,14 +286,15 @@ class Conv2DBNNSOp : public BinaryOp<T> {
std::vector<int32> strides_;
Padding padding_;
TensorFormat data_format_;
BNNSActivationFunction bnnActivationFunction_;

TF_DISALLOW_COPY_AND_ASSIGN(Conv2DBNNSOp);

};

#define REGISTER_CPU(T) \
REGISTER_KERNEL_BUILDER( \
Name("Conv2DBNNS").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
Name("Conv2DBNNS").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
Conv2DBNNSOp<CPUDevice, T>);

TF_CALL_float(REGISTER_CPU);
4 changes: 2 additions & 2 deletions tensorflow/core/user_ops/bnns_maxpool.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,8 @@ class MaxPoolingBNNSOp : public OpKernel {
OP_REQUIRES(context, filter_bnns != nullptr,
errors::Unknown("BNNSFilterCreatePoolingLayer failed"));

const float* i_stack = tensor_in.flat<float>().data();
float* o_stack = output->flat<float>().data();
const T* i_stack = tensor_in.flat<T>().data();
T* o_stack = output->flat<T>().data();

// Apply filter to input stack. Result is written in output stack.
int status = BNNSFilterApplyBatch(
Expand Down

0 comments on commit 9994aea

Please sign in to comment.