feat/native: Add support for softmax w/ test and benches.

benches/softmax.rs

@@ -0,0 +1,86 @@
+#![feature(test)]
+#![feature(clone_from_slice)]
+
+extern crate test;
+extern crate collenchyma as co;
+extern crate collenchyma_nn as co_nn;
+extern crate rand;
+
+use test::Bencher;
+use co::backend::{Backend, BackendConfig};
+use co::frameworks::Native;
+use co::framework::IFramework;
+use co::tensor::SharedTensor;
+use co_nn::*;
+
+use rand::{thread_rng, Rng};
+
+fn backend() -> Backend<Native> {
+ let framework = Native::new();
+ let hardwares = framework.hardwares();
+ let backend_config = BackendConfig::new(framework, hardwares);
+ Backend::new(backend_config).unwrap()
+}
+
+fn arguments<T: IFramework + Clone>(backend: &Backend<T>, size: usize) -> (SharedTensor<f32>, SharedTensor<f32>) {
+ let mut rng = thread_rng();
+ let slice_x = rng.gen_iter::<f32>().take(size).collect::<Vec<f32>>();
+
+ let mut x = SharedTensor::<f32>::new(backend.device(), &size).unwrap();
+ let out = SharedTensor::<f32>::new(backend.device(), &size).unwrap();
+ x.get_mut(backend.device()).unwrap().as_mut_native().unwrap().as_mut_slice().clone_from_slice(&slice_x);
+ (x, out)
+}
+
+fn arguments_grad<T: IFramework + Clone>(backend: &Backend<T>, size: usize) -> (SharedTensor<f32>, SharedTensor<f32>, SharedTensor<f32>) {
+ let mut rng = thread_rng();
+ let slice_x = rng.gen_iter::<f32>().take(size).collect::<Vec<f32>>();
+
+ let mut x = SharedTensor::<f32>::new(backend.device(), &size).unwrap();
+ let mut dx = SharedTensor::<f32>::new(backend.device(), &size).unwrap();
+ let dout = SharedTensor::<f32>::new(backend.device(), &size).unwrap();
+ x.get_mut(backend.device()).unwrap().as_mut_native().unwrap().as_mut_slice().clone_from_slice(&slice_x);
+ dx.get_mut(backend.device()).unwrap().as_mut_native().unwrap().as_mut_slice().clone_from_slice(&slice_x);
+ (x, dx, dout)
+}
+
+#[inline(never)]
+fn bench_profile<F: FnMut() -> ()>(
+ b: &mut Bencher,
+ mut bench_func: F,
+ times: usize
+) {
+ b.iter(|| { for _ in 0..times { bench_func(); } });
+}
+
+#[bench]
+fn bench_1000_softmax_100_native(b: &mut Bencher) {
+ let backend = backend();
+ let (mut x, mut out) = arguments(&backend, 100);
+ let mut func = || { let _ = backend.softmax_plain(&mut x, &mut out); };
+ { func(); bench_profile(b, func, 1000); }
+}
+
+#[bench]
+fn bench_10_softmax_10000_native(b: &mut Bencher) {
+ let backend = backend();
+ let (mut x, mut out) = arguments(&backend, 10000);
+ let mut func = || { let _ = backend.softmax_plain(&mut x, &mut out); };
+ { func(); bench_profile(b, func, 10); }
+}
+
+#[bench]
+fn bench_1000_softmax_grad_100_native(b: &mut Bencher) {
+ let backend = backend();
+ let (mut x, mut dx, mut dout) = arguments_grad(&backend, 100);
+ let mut func = || { let _ = backend.softmax_grad_plain(&mut x, &mut dx, &mut dout); };
+ { func(); bench_profile(b, func, 1000); }
+}
+
+#[bench]
+fn bench_10_softmax_grad_10000_native(b: &mut Bencher) {
+ let backend = backend();
+ let (mut x, mut dx, mut dout) = arguments_grad(&backend, 10000);
+ let mut func = || { let _ = backend.softmax_grad_plain(&mut x, &mut dx, &mut dout); };
+ { func(); bench_profile(b, func, 10); }
+}

src/frameworks/native/helper.rs

@@ -130,8 +130,8 @@ macro_rules! impl_ops_sigmoid_for {
  result_diff: &mut ::co::tensor::SharedTensor<$t>
  ) -> Result<(), ::co::error::Error> {
  match x.add_device(self.device()) { _ => try!(x.sync(self.device())) }
- match x_diff.add_device(self.device()) { _ => try!(x.sync(self.device())) }
- match result.add_device(self.device()) { _ => try!(x.sync(self.device())) }
+ match x_diff.add_device(self.device()) { _ => try!(x_diff.sync(self.device())) }
+ match result.add_device(self.device()) { _ => try!(result.sync(self.device())) }
  match result_diff.add_device(self.device()) { _ => () }
  self.sigmoid_grad_plain(x, x_diff, result, result_diff)
  }
@@ -193,8 +193,8 @@ macro_rules! impl_ops_relu_for {
  result_diff: &mut ::co::tensor::SharedTensor<$t>
  ) -> Result<(), ::co::error::Error> {
  match x.add_device(self.device()) { _ => try!(x.sync(self.device())) }
- match x_diff.add_device(self.device()) { _ => try!(x.sync(self.device())) }
- match result.add_device(self.device()) { _ => try!(x.sync(self.device())) }
+ match x_diff.add_device(self.device()) { _ => try!(x_diff.sync(self.device())) }
+ match result.add_device(self.device()) { _ => try!(result.sync(self.device())) }
  self.relu_grad_plain(x, x_diff, result, result_diff)
  }
 
@@ -256,8 +256,8 @@ macro_rules! impl_ops_tanh_for {
  result_diff: &mut ::co::tensor::SharedTensor<$t>
  ) -> Result<(), ::co::error::Error> {
  match x.add_device(self.device()) { _ => try!(x.sync(self.device())) }
- match x_diff.add_device(self.device()) { _ => try!(x.sync(self.device())) }
- match result.add_device(self.device()) { _ => try!(x.sync(self.device())) }
+ match x_diff.add_device(self.device()) { _ => try!(x_diff.sync(self.device())) }
+ match result.add_device(self.device()) { _ => try!(result.sync(self.device())) }
  self.tanh_grad_plain(x, x_diff, result, result_diff)
  }
 
@@ -354,34 +354,64 @@ macro_rules! impl_ops_softmax_for {
  x: &mut ::co::tensor::SharedTensor<$t>,
  result: &mut ::co::tensor::SharedTensor<$t>
  ) -> Result<(), ::co::error::Error> {
- unimplemented!();
- Ok(())
+ match x.add_device(self.device()) { _ => try!(x.sync(self.device())) }
+ match result.add_device(self.device()) { _ => () }
+ self.softmax_plain(x, result)
  }
  fn softmax_plain(
  &self,
  x: &::co::tensor::SharedTensor<$t>,
  result: &mut ::co::tensor::SharedTensor<$t>
  ) -> Result<(), ::co::error::Error> {
- unimplemented!();
- Ok(())
+ if let Some(input) = x.get(self.device()).unwrap().as_native() {
+ let mut exps = Vec::with_capacity(x.capacity());
+ let mut sum : $t = 0 as $t;
+ for exp in input.as_slice::<$t>().iter().map(|t|t.exp()) {
+ exps.push(exp);
+ sum += exp;
+ }
+ let res = exps.iter().map(|t| t / sum);
+ ::frameworks::native::helper::write_to_memory(result.get_mut(self.device()).unwrap(), res);
+ return Ok(());
+ }
+ Err(Error::Plugin(
+ PluginError::Operation("Unable to execute Native softmax Forward.")))
  }
  fn softmax_grad(
  &self,
  x: &mut ::co::tensor::SharedTensor<$t>,
  x_diff: &mut ::co::tensor::SharedTensor<$t>,
  result_diff: &mut ::co::tensor::SharedTensor<$t>
  ) -> Result<(), ::co::error::Error> {
- unimplemented!();
- Ok(())
+ match x.add_device(self.device()) { _ => try!(x.sync(self.device())) }
+ match x_diff.add_device(self.device()) { _ => try!(x_diff.sync(self.device())) }
+ match result_diff.add_device(self.device()) { _ => () }
+ self.softmax_grad_plain(x, x_diff, result_diff)
  }
  fn softmax_grad_plain(
  &self,
  x: &::co::tensor::SharedTensor<$t>,
  x_diff: &::co::tensor::SharedTensor<$t>,
  result_diff: &mut ::co::tensor::SharedTensor<$t>
  ) -> Result<(), ::co::error::Error> {
- unimplemented!();
- Ok(())
+ if let Some(sig_data) = x.get(self.device()).unwrap().as_native() {
+ if let Some(sig_dx) = x_diff.get(self.device()).unwrap().as_native() {
+ let mut dot : $t = 0 as $t;
+ let sig_data_slice = sig_data.as_slice::<$t>();
+ let sig_dx_slice = sig_dx.as_slice::<$t>();
+ for (t, dt) in sig_data_slice.iter().zip(sig_dx_slice.iter()) {
+ dot += t * dt;
+ }
+ let res = sig_data_slice.iter()
+ .zip(sig_dx_slice.iter())
+ .map(|(t, dt)| t * (dt - dot));
+ ::frameworks::native::helper::write_to_memory(result_diff.get_mut(self.device()).unwrap(), res);
+ return Ok(());
+ }
+ }
+ Err(Error::Plugin(
+ PluginError::Operation("Unable to execute Native softmax Backward.")))
+
  }
  }
  );

src/frameworks/native/mod.rs

@@ -31,7 +31,7 @@ impl_ops_sigmoid_for!(f32, Backend<Native>);
 impl_ops_relu_for!(f32, Backend<Native>);
 impl_ops_tanh_for!(f32, Backend<Native>);
 // impl_ops_convolution_for!(f32, Backend<Native>);
-// impl_ops_softmax_for!(f32, Backend<Native>);
+ impl_ops_softmax_for!(f32, Backend<Native>);
 // impl_ops_lrn_for!(f32, Backend<Native>);
 // impl_ops_pooling_for!(f32, Backend<Native>);
 
@@ -48,6 +48,6 @@ impl_ops_sigmoid_for!(f64, Backend<Native>);
 impl_ops_relu_for!(f64, Backend<Native>);
 impl_ops_tanh_for!(f64, Backend<Native>);
 // impl_ops_convolution_for!(f64, Backend<Native>);
-// impl_ops_softmax_for!(f64, Backend<Native>);
+ impl_ops_softmax_for!(f64, Backend<Native>);
 // impl_ops_lrn_for!(f64, Backend<Native>);
 // impl_ops_pooling_for!(f64, Backend<Native>);