Support Stereo Rendering in Pano

Summary: Add StereoBasicTextureMaterial for supporting basic stereo rendering. By setting stereo format in source.stereo, you can show a stereo 360 photo or play a stereo 360 video.

Reviewed By: mikearmstrong001

Differential Revision: D4993764

fbshipit-source-id: d551ef9

Libraries/Pano/Pano.js

@@ -47,10 +47,16 @@ const Pano = React.createClass({
      * or
      * `[{uri: 'http..'}, {uri: 'http..'}, {uri: 'http..'},
      *   {uri: 'http..'}, {uri: 'http..'}, {uri: 'http..'}]` for a cubemap
+     *
+     * stereo(optional): the stereo format of a panorama: '2D' | 'TOP_BOTTOM_3D' |
+     * 'BOTTOM_TOP_3D' | 'LEFT_RIGHT_3D' | 'RIGHT_LEFT_3D'
+     *
+     * If stereo is not a supported stereo format, it'll by default use '2D'
      */
     source: PropTypes.oneOfType([
       PropTypes.shape({
         uri: PropTypes.string,
+        stereo: PropTypes.string,
       }),
       PropTypes.arrayOf(
         PropTypes.shape({

Libraries/VideoPano/VideoPano.js

@@ -53,9 +53,13 @@ const VideoPano = React.createClass({
      *
      * layout(optional): the layout format of 360 video: 'SPHERE' | 'CUBEMAP_32'
      *
+     * stereo(optional): the stereo format of 360 video: '2D' | 'TOP_BOTTOM_3D' |
+     * 'BOTTOM_TOP_3D' | 'LEFT_RIGHT_3D' | 'RIGHT_LEFT_3D'
+     *
      * metaData(optional): the video meta data, used for customized video player
      *
      * If layout is not a supported layout format, it'll by default use 'SPHERE'(equirectangular)
+     * If stereo is not a supported stereo format, it'll by default use '2D'
      * Source can be an array of sources with different formats, and VideoPano will
      * choose one of the formats that current browser supports. If format is not specified,
      * it can be chose whatever format is supported.
@@ -65,13 +69,15 @@ const VideoPano = React.createClass({
         uri: PropTypes.string,
         format: PropTypes.string,
         layout: PropTypes.string,
+        stereo: PropTypes.string,
         metaData: PropTypes.any,
       }),
       PropTypes.arrayOf(
         PropTypes.shape({
           uri: PropTypes.string,
           format: PropTypes.string,
           layout: PropTypes.string,
+          stereo: PropTypes.string,
           metaData: PropTypes.any,
         })
       ),

OVRUI/src/Control/VREffect.js

@@ -22,8 +22,10 @@ const DEFAULT_RIGHT_BOUNDS = [0.5, 0.0, 0.5, 1.0];
 // Pre-allocate objects
 const leftCamera = new THREE.PerspectiveCamera();
 leftCamera.layers.enable(1);
+leftCamera.viewID = 0;
 const rightCamera = new THREE.PerspectiveCamera();
 rightCamera.layers.enable(2);
+rightCamera.viewID = 1;
 const leftTranslation = new THREE.Vector3();
 const rightTranslation = new THREE.Vector3();
 

OVRUI/src/Materials/StereoBasicTextureMaterial.js

@@ -0,0 +1,79 @@
+/**
+ * Basic Material support rendering mono/stereo textures
+ * @constructor
+ * @param {Object} parameters - parameters for THREE.Material
+ */
+
+import THREE from '../ThreeShim';
+import StereoTextureUniforms from './StereoTextureUniforms';
+import StereoShaderLib from './StereoShaderLib';
+
+const DEFAULT_UNIFORM_COLOR = new THREE.Color();
+const DEFAULT_OFFSET_REPEATS = [new THREE.Vector4(0, 0, 1, 1)];
+
+export default class StereoBasicTextureMaterial extends THREE.ShaderMaterial {
+  constructor(parameters) {
+    const uniforms = THREE.UniformsUtils.merge([
+      new StereoTextureUniforms(),
+      {
+        color: {value: DEFAULT_UNIFORM_COLOR, type: 'f'}, // The color of material
+        opacity: {value: 1.0, type: 'f'}, // The opacity of material
+        map: {value: null, type: 't'}, // The color map of material
+        envMap: {value: null, type: 't'}, // The environment map of material
+        flipEnvMap: {value: 1.0, type: 'f'}, // Three.js use this to differentiate CubeTexture and WebGLRenderTargetCube
+        reflectivity: {value: 1.0, type: 'f'}, // How much the environment map affect the surface
+        refractionRatio: {value: 0.0, type: 'f'}, // The ratio of refraction for environment map
+      },
+    ]);
+
+    super({
+      uniforms: uniforms,
+      vertexShader: StereoShaderLib.stereo_basic_vert,
+      fragmentShader: StereoShaderLib.stereo_basic_frag,
+    });
+
+    this.isStereoBasicTextureMaterial = true;
+    // The offset repeats of each eye, this makes each eye use different uv to get stereo rendering output
+    this.stereoOffsetRepeats = DEFAULT_OFFSET_REPEATS;
+    this.setValues(parameters);
+  }
+
+  copy(source) {
+    super.copy(source);
+    this.stereoOffsetRepeats = source.stereoOffsetRepeats.slice();
+    return this;
+  }
+
+  set color(value) {
+    this.uniforms.color.value = new THREE.Color(value);
+  }
+
+  get color() {
+    return this.uniforms.color.value;
+  }
+
+  set opacity(value) {
+    this.uniforms && (this.uniforms.opacity.value = value);
+  }
+
+  get opacity() {
+    return this.uniforms.opacity.value;
+  }
+
+  set map(value) {
+    this.uniforms.map.value = value;
+  }
+
+  get map() {
+    return this.uniforms.map.value;
+  }
+
+  set envMap(value) {
+    this.uniforms.envMap.value = value;
+    this.uniforms.flipEnvMap.value = !(value && value.isCubeTexture) ? 1 : -1;
+  }
+
+  get envMap() {
+    return this.uniforms.envMap.value;
+  }
+}

OVRUI/src/Materials/StereoShaderLib.js

@@ -0,0 +1,105 @@
+/**
+ * Shader library for rendering stereo textures
+ *
+ * Part of this code is source from meshbasic_vert.glsl and meshbasic_frag.glsl
+ * of Three.js
+ * https://github.com/mrdoob/three.js/
+ */
+
+const StereoShaderLib = {
+  stereo_basic_vert: `
+      uniform vec4 stereoOffsetRepeat;
+      uniform vec3 color;
+      #ifdef USE_ENVMAP
+        varying vec3 vWorldPosition;
+        varying vec3 vNormal;
+      #endif
+      varying lowp vec3 vColor;
+      #ifdef USE_MAP
+        varying highp vec2 vUv;
+      #endif
+      void main()
+      {
+          #ifdef USE_MAP
+            vUv = uv * stereoOffsetRepeat.zw + stereoOffsetRepeat.xy;
+          #endif
+          vColor = color;
+
+          #ifdef USE_ENVMAP
+            vec4 worldPosition = modelMatrix * vec4( position, 1.0 );
+            vWorldPosition = worldPosition.xyz;
+            vNormal = normalMatrix * normal;
+          #endif
+
+          gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
+      }
+  `,
+
+  stereo_basic_frag: `
+      #define RECIPROCAL_PI2 0.15915494
+      uniform vec4 stereoOffsetRepeat;
+      uniform float opacity;
+      uniform sampler2D map;
+      #ifdef USE_ENVMAP
+        varying vec3 vWorldPosition;
+        varying vec3 vNormal;
+        #ifdef ENVMAP_TYPE_CUBE
+          uniform samplerCube envMap;
+        #else
+          uniform sampler2D envMap;
+        #endif
+        uniform float reflectivity;
+        uniform float flipEnvMap;
+        uniform float refractionRatio;
+      #endif
+      #ifdef USE_MAP
+        varying highp vec2 vUv;
+      #endif
+      varying lowp vec3 vColor;
+      void main()
+      {
+        vec4 diffuseColor = vec4( 1.0, 1.0, 1.0, opacity );
+
+        #ifdef DOUBLE_SIDED
+          float flipNormal = ( float( gl_FrontFacing ) * 2.0 - 1.0 );
+        #else
+          float flipNormal = 1.0;
+        #endif
+
+        #ifdef USE_MAP
+          vec4 texColor = texture2D( map, vUv );
+          diffuseColor *= texColor;
+        #endif
+
+        #ifdef USE_ENVMAP
+          vec3 cameraToVertex = normalize( vWorldPosition - cameraPosition );
+          vec3 worldNormal = normalize( ( vec4( vNormal, 0.0 ) * viewMatrix ).xyz );
+          vec3 reflectVec = refract( cameraToVertex, worldNormal, refractionRatio );
+
+          #ifdef ENVMAP_TYPE_CUBE
+            vec4 envColor = textureCube( envMap, flipNormal * vec3( flipEnvMap * reflectVec.x, reflectVec.yz ) );
+          #elif defined( ENVMAP_TYPE_EQUIREC )
+            vec2 sampleUV;
+            sampleUV.y = saturate( flipNormal * reflectVec.y * 0.5 + 0.5 );
+            sampleUV.x = atan( flipNormal * reflectVec.z, flipNormal * reflectVec.x ) * RECIPROCAL_PI2 + 0.5;
+            vec2 stereoSampleUV = sampleUV * stereoOffsetRepeat.zw + stereoOffsetRepeat.xy;
+            vec4 envColor = texture2D( envMap, stereoSampleUV );
+          #elif defined( ENVMAP_TYPE_SPHERE )
+            vec3 reflectView = flipNormal * normalize( ( viewMatrix * vec4( reflectVec, 0.0 ) ).xyz
+              + vec3( 0.0, 0.0, 1.0 ) );
+            vec2 sampleUV = reflectView.xy * 0.5 + 0.5;
+            vec2 stereoSampleUV = sampleUV * stereoOffsetRepeat.zw + stereoOffsetRepeat.xy;
+            vec4 envColor = texture2D( envMap, stereoSampleUV );
+          #else
+            vec4 envColor = vec4( 0.0 );
+          #endif
+          diffuseColor.rgb = mix( diffuseColor.rgb, diffuseColor.rgb * envColor.rgb, reflectivity );
+        #endif
+
+        diffuseColor.rgb *= vColor;
+        gl_FragColor = diffuseColor;
+      }
+  `,
+};
+
+export default StereoShaderLib;

OVRUI/src/Materials/StereoTextureUniforms.js

@@ -0,0 +1,26 @@
+/**
+ * Dynamic uniforms for rendering stereo texture.
+ *
+ * Callback function onUpdateCallback will be called in every render loop before
+ * setting the value of the uniform to WebGL. To enable stereo texture, you
+ * should set viewID=1 for the right eye camera. If you set two stereoOffsetRepeats
+ * to the material, the right eye camera will use the second stereoOffsetRepeat
+ */
+export default class StereoTextureUniforms {
+  constructor() {
+    /** The right eye camera will use stereoOffsetRepeats[1] if it's defined. */
+    this.stereoOffsetRepeat = {
+      dynamic: true,
+      type: 'f',
+      value: null,
+      onUpdateCallback: function(object, camera) {
+        // if it's right eye camera and has second offsetRepeats, use the second offsetRepeats
+        if (camera.viewID === 1 && object.material.stereoOffsetRepeats[1]) {
+          this.value = object.material.stereoOffsetRepeats[1];
+        } else {
+          this.value = object.material.stereoOffsetRepeats[0];
+        }
+      },
+    };
+  }
+}

OVRUI/src/OVRUI.js

@@ -33,6 +33,7 @@ import {
   TOP,
   SDFFONT_MARKER_COLOR,
 } from './SDFFont/SDFFont';
+import StereoBasicTextureMaterial from './Materials/StereoBasicTextureMaterial';
 import GuiSys from './UIView/GuiSys';
 import {GuiSysEventType, GuiSysEvent, UIViewEventType, UIViewEvent} from './UIView/GuiSysEvent';
 import UIView from './UIView/UIView';
@@ -66,6 +67,9 @@ export {RIGHT_LINE};
 export {TOP};
 export {SDFFONT_MARKER_COLOR};
 
+// Stereo Material
+export {StereoBasicTextureMaterial};
+
 // UIView
 export {GuiSys};
 export {GuiSysEventType};

ReactVR/js/Utils/StereoOffsetRepeats.js

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/**
+ * StereoOffsetRepeats: Constants of offsetRepeats for different stereo format
+ */
+
+import * as THREE from 'three';
+
+const StereoOffsetRepeats = {
+  // 2D texture
+  '2D': [new THREE.Vector4(0, 0, 1, 1)],
+  // 3D texture splited into top and bottom, the top-half for left eye and the bottom-half for right eye
+  TOP_BOTTOM_3D: [new THREE.Vector4(0, 0.5, 1, 0.5), new THREE.Vector4(0, 0, 1, 0.5)],
+  // 3D texture splited into top and bottom, the bottom-half for left eye and the top-half for right eye
+  BOTTOM_TOP_3D: [new THREE.Vector4(0, 0, 1, 0.5), new THREE.Vector4(0, 0.5, 1, 0.5)],
+  // 3D texture splited into left and right, the left-half for left eye and the right-half for right eye
+  LEFT_RIGHT_3D: [new THREE.Vector4(0, 0, 0.5, 1), new THREE.Vector4(0.5, 0, 0.5, 1)],
+  // 3D texture splited into left and right, the right-half for left eye and the left-half for right eye
+  RIGHT_LEFT_3D: [new THREE.Vector4(0.5, 0, 0.5, 1), new THREE.Vector4(0, 0, 0.5, 1)],
+};
+
+export default StereoOffsetRepeats;

ReactVR/js/Views/Pano.js

@@ -18,6 +18,7 @@
 
 import RCTBaseView from './BaseView';
 import merge from '../Utils/merge';
+import StereoOffsetRepeats from '../Utils/StereoOffsetRepeats';
 import {HPanoBufferGeometry} from '../Utils/HPano';
 import {CubePanoBufferGeometry} from '../Utils/CubePano';
 import {RCTBindedResource} from '../Utils/RCTBindedResource';
@@ -69,8 +70,7 @@ export default class RCTPano extends RCTBaseView {
 
     this._sphereGeometry = new THREE.SphereGeometry(1000, 50, 50);
     this._cubeGeometry = new CubePanoBufferGeometry(2000, 3, 2, 1.01);
-    this._material = new THREE.MeshBasicMaterial({
-      wireframe: false,
+    this._material = new OVRUI.StereoBasicTextureMaterial({
       color: 'white',
       side: THREE.DoubleSide,
     });
@@ -180,6 +180,13 @@ export default class RCTPano extends RCTBaseView {
           this._material.map = flatTexture;
           this._material.envMap = cubeTexture;
         }
+        const stereoFormat = value && value.stereo ? value.stereo : '2D';
+        this._material.stereoOffsetRepeats = StereoOffsetRepeats[stereoFormat];
+        if (!this._material.stereoOffsetRepeats) {
+          console.warn(`Pano: stereo format '${stereoFormat}' not supported.`);
+          // fallback to 2D
+          this._material.stereoOffsetRepeats = StereoOffsetRepeats['2D'];
+        }
         this._material.needsUpdate = true;
 
         // call onLoad in React