How to apply texture on filtered point cloud data?

[oscarzhaoyb]

Required Info
Camera Model	{D435}
Firmware Version	05.08.15.00
Operating System & Version	{Ubuntu 16}
Kernel Version (Linux Only)	4.13.0-45
Platform	PC
SDK Version	{ 2.11.1 }
Language	{C++ }
Segment	{others }

Issue Description

I am trying to view the textured filtered point cloud data using the example code of post_processing code. The point cloud data does not have the colored texture on it. Could you please let me know why.

[oscarzhaoyb]

I am also trying to write my own c++ code to filter the data and save the point cloud data to ply. It does not allow my to do that. I don't know why. Could you please point out where I did was wrong? The code is stuck at "filtered_points = filtered_pc.calculate(filtered);" The following is my code.

#include <librealsense2/rs.hpp> // Include RealSense Cross Platform API
#include "../example.hpp"          // Include short list of convenience functions for rendering

#include <algorithm>            // std::min, std::max
#include <string>
#include <iostream>

#include <time.h>
#include <sys/time.h>
#include <stdlib.h>
#include <stdio.h>


using namespace std;
// Helper functions
void register_glfw_callbacks(window& app, glfw_state& app_state);

int main(int argc, char * argv[]) try
{
    // Create a simple OpenGL window for rendering:
    window app(1280, 720, "RealSense Pointcloud Example");
    // Construct an object to manage view state
    glfw_state app_state;
    // register callbacks to allow manipulation of the pointcloud
    register_glfw_callbacks(app, app_state);
    
     

    // Declare pointcloud object, for calculating pointclouds and texture mappings
    rs2::pointcloud pc;
    rs2::pointcloud filtered_pc;
    // We want the points object to be persistent so we can display the last cloud when a frame drops
    rs2::points points;
    rs2::points filtered_points;

    // Declare RealSense pipeline, encapsulating the actual device and sensors
    //rs2::pipeline pipe = std::make_shared<rs::pipeline>();
    auto pipe = std::make_shared<rs2::pipeline>();

    rs2::config cfg;
    string str0 = "a.bag";
    cfg.enable_device_from_file(str0.c_str());  
    pipe->start(cfg); //File will be opened at this point 
    
    
    // Start streaming with default recommended configuration
    rs2::decimation_filter dec_filter;  // Decimation - reduces depth frame density
    rs2::spatial_filter spat_filter;    // Spatial    - edge-preserving spatial smoothing
    rs2::temporal_filter temp_filter;   // Temporal   - reduces temporal noise

                                        // Declare disparity trkeansform from depth to disparity and vice versa
    const std::string disparity_filter_name = "Disparity";
    rs2::disparity_transform depth_to_disparity(true);
    rs2::disparity_transform disparity_to_depth(true);

    rs2::process_interface *filter0 = &dec_filter;
    rs2::process_interface *filter1 = &depth_to_disparity;
    rs2::process_interface *filter2 = &spat_filter;
    rs2::process_interface *filter3 = &temp_filter;
    rs2::process_interface *filter4 = &disparity_to_depth;

    rs2_option r1 = RS2_OPTION_FILTER_MAGNITUDE;
    rs2_option r2 = RS2_OPTION_FILTER_SMOOTH_ALPHA;
    rs2_option r3 = RS2_OPTION_FILTER_SMOOTH_DELTA;

    int val_dec_m = 1;
    int val_spat_m = 2;
    int val_spat_d = 20;
    float val_spat_a = 0.5;
    float val_temp_a = 0.4;
    int val_temp_d = 20;
    filter0->set_option(r1,val_dec_m); // 1:1:8
    filter2->set_option(r1,val_spat_m); // 1:1:5
    filter2->set_option(r2, val_spat_a); //0.25-1
    filter2->set_option(r3,val_spat_d); // 1:1:50
    filter3->set_option(r2,val_temp_a); // 0:1
    filter3->set_option(r3,val_temp_d); //1-100
    
    bool dec_filter_app = true;
    bool spat_filter_app = true;
    bool temp_filter_app = true;
    bool depth2dispty_app = true;
    bool dispty2depth_app = depth2dispty_app;
    
    rs2::device device = pipe->get_active_profile().get_device();

    string filename;
    string filename_filtered;
    int n = 0;

    struct timeval start;
    int frame_cnt = 0;
    string filelogname = "/var/voters/206/realsensedata/stream.log";

    FILE* pfFilelog = fopen(filelogname.c_str(),"w");
    string filefilteredlogname = "/var/voters/206/realsensedata/filteredstream.log";
    FILE* pfFilefilteredlog = fopen(filefilteredlogname.c_str(),"w");
    cout << "loop starts" << endl; 
    string rootpath = "/var/voters/206/realsensedata/";
    while (app) // Application still alive?
    {
	string filename0 = "pointcloud"+to_string(n)+".ply";  
	string filename1 = "pointcloud_filtered"+to_string(n)+".ply";
	filename = rootpath+filename0; 
	filename_filtered = rootpath+filename1;
	// Wait for the next set of frames from the camera
	cout << "get the time now"<< endl;	
	gettimeofday(&start, NULL);        
	cout << "time obtained : " << start.tv_sec << " sec, " << start.tv_usec << " usec" <<endl;
	rs2::frameset frames = pipe->wait_for_frames();
	cout << "Get the frame and proceed." << endl;

		cout << "Start to get the depth frame" << endl;
		rs2::frame depth = frames.get_depth_frame();
		
		if (!depth){
			continue;
		}
		rs2::frame filtered = depth;

		// Generate the pointcloud and texture mappings
		points = pc.calculate(depth);

		auto color = frames.get_color_frame();

		// Tell pointcloud object to map to this color frame
		pc.map_to(color);
		
		cout << "start to apply filter" << endl;
		if (dec_filter_app){
			filtered = dec_filter.process(filtered);
		}
		if (depth2dispty_app){
			filtered = depth_to_disparity.process(filtered);
		}
		if (spat_filter_app){
			filtered = spat_filter.process(filtered);
		}
		if (temp_filter_app){
			filtered = temp_filter.process(filtered);
		}
		if (dispty2depth_app){
			filtered = disparity_to_depth.process(filtered);
		}
		rs2::colorizer color_map;
		cout << "Filtered depth" << endl;
		filtered_points = filtered_pc.calculate(filtered);
		cout << "Filtered point cloud data" << endl;
		rs2::frame colorized_depth = color_map(filtered);
		cout << "Apply texture" << endl; 
		filtered_pc.map_to(colorized_depth);
		cout << "Saving ply file" << endl;
		filtered_points.export_to_ply(filename_filtered, color);
		cout << "filtered point cloud file saved" << endl;
		//filtered_pc.map_to(color);
		
		points.export_to_ply(filename, color);
		cout << "point cloud file saved." << endl;
		frame_cnt++;

		// Upload the color frame to OpenGL
		app_state.tex.upload(color);

		// Draw the pointcloud
		draw_pointcloud(app.width(), app.height(), app_state, points);
	
 	n++;
    }

    fclose(pfFilelog);
    return EXIT_SUCCESS;
}
catch (const rs2::error & e)
{
    std::cerr << "RealSense error calling " << e.get_failed_function() << "(" << e.get_failed_args() << "):\n    " << e.what() << std::endl;
    return EXIT_FAILURE;
}
catch (const std::exception & e)
{
    std::cerr << e.what() << std::endl;
    return EXIT_FAILURE;
}

[RealSense-Customer-Engineering]

[Realsense Customer Engineering Team Comment]
@oscarzhaoyb
First, you have to change the parameter of disparity_transform instance to (false) if you want it to transform from disparity back to depth as below :

rs2::disparity_transform disparity_to_depth(false);

Then you should design which texture to map to depth pointcloud then save them into PLY. For example, below you map to colorized_depth, but export the texture with color. You should export with the colorized_depth to make the texture align with it.

        filtered_pc.map_to(colorized_depth); cout << "Saving ply file" << endl; filtered_points.export_to_ply(filename_filtered, color);

[RealSense-Customer-Engineering]

[Realsense Customer Engineering Team Comment]
@oscarzhaoyb
issue fixed?