Question about dynamic scatter

[ozzCb]

I'm trying to use dynamic voxel in mmdet3d, but get wrong result when testing. Here is my test code

from mmdet3d.ops import Voxelization, DynamicScatter
# from mmdet3d.ops.voxel.scatter_points import dynamic_point_to_voxel_forward
import torch
from torch.nn import functional as F
# from pcdet.ops.dynamic_voxel.dynamic_voxel_utils import VoxelGenerator, scatterMean


class Tmp:
    def __init__(self, _voxel_size, _point_cloud_range):
        self.voxe_size = _voxel_size
        self.point_cloud_range = _point_cloud_range
        self.vx = _voxel_size[0]
        self.vy = _voxel_size[1]
        self.vz = _voxel_size[2]
        self.x_offset = self.vx / 2 + _point_cloud_range[0]
        self.y_offset = self.vy / 2 + _point_cloud_range[1]
        self.z_offset = self.vz / 2 + _point_cloud_range[2]

        self.voxelizer = Voxelization(_voxel_size, _point_cloud_range, -1, -1)  # use dynamic voxel
        self.cluster_scatter = DynamicScatter(
            _voxel_size, _point_cloud_range, average_points=True)

    # copied from mmdet3d
    def map_voxel_center_to_point(self, pts_coors, voxel_mean, voxel_coors):
        """Map the centers of voxels to its corresponding points.

        Args:
            pts_coors (torch.Tensor): The coordinates of each points, shape
                (M, 3), where M is the number of points.
            voxel_mean (torch.Tensor): The mean or aggreagated features of a
                voxel, shape (N, C), where N is the number of voxels.
            voxel_coors (torch.Tensor): The coordinates of each voxel.

        Returns:
            torch.Tensor: Corresponding voxel centers of each points, shape
                (M, C), where M is the numver of points.
        """
        # Step 1: scatter voxel into canvas
        # Calculate necessary things for canvas creation
        canvas_y = int(
            (self.point_cloud_range[4] - self.point_cloud_range[1]) / self.vy)
        canvas_x = int(
            (self.point_cloud_range[3] - self.point_cloud_range[0]) / self.vx)
        canvas_channel = voxel_mean.size(1)
        batch_size = pts_coors[-1, 0] + 1
        canvas_len = canvas_y * canvas_x * batch_size
        # Create the canvas for this sample
        canvas = voxel_mean.new_zeros(canvas_channel, canvas_len)
        # Only include non-empty pillars
        indices = (
            voxel_coors[:, 0] * canvas_y * canvas_x +
            voxel_coors[:, 2] * canvas_x + voxel_coors[:, 3])
        # Scatter the blob back to the canvas
        canvas[:, indices.long()] = voxel_mean.t()

        # Step 2: get voxel mean for each point
        voxel_index = (
            pts_coors[:, 0] * canvas_y * canvas_x +
            pts_coors[:, 2] * canvas_x + pts_coors[:, 3])
        center_per_point = canvas[:, voxel_index.long()].t()
        return center_per_point

    def test_voxelization(self, points):
        # voxelization
        # coors = self.voxelizer(points[:, :3].contiguous())
        coors = self.voxelizer(points)
        coors = F.pad(coors, (1, 0), mode='constant', value=0)
        print(f"coors: \n{coors}")

        # scatter in PFN
        voxel_mean, mean_coors = self.cluster_scatter(points, coors)
        points_mean = self.map_voxel_center_to_point(
            coors, voxel_mean, mean_coors)
        print(f'output:\n{points_mean}')


if __name__ == "__main__":
    voxel_size = [0.25, 0.25, 8.0]
    point_cloud_range = [-64, -64, -5.0, 64, 64, 3.0]

    pts_1 = torch.tensor([
        [-11.7974, -13.8524, 0.7906, 0.1],
        [-11.9223, -13.8233, 1.1659, 0.2],
    ]).float().cuda()
    # pts_2 = torch.randn([12, 4]).float().cuda()  # * 10
    # test_points = torch.cat([pts_1, pts_2], dim=0)
    test_points = pts_1
    print(f"input shape: {test_points.shape}")
    print(f"input:\n {test_points}")

    test_module = Tmp(_voxel_size=voxel_size, _point_cloud_range=point_cloud_range)
    test_module.test_voxelization(test_points)

result I got:

input shape: torch.Size([2, 4])
input:
 tensor([[-11.7974, -13.8524,   0.7906,   0.1000],
        [-11.9223, -13.8233,   1.1659,   0.2000]], device='cuda:0')
coors: 
tensor([[  0,   0, 200, 208],
        [  0,   0, 200, 208]], device='cuda:0', dtype=torch.int32)
output:
tensor([[0., 0., 0., 0.],
        [0., 0., 0., 0.]], device='cuda:0')

where output is not supposed to be 0.

can u tell me what wrong in my test code or there is something wrong with dynamicScatter?

[zhanggefan]

Hi, sorry for the late reply.

It is a bug. Really sorry for the trouble.

mmdetection3d/mmdet3d/ops/voxel/src/scatter_points_cuda.cu

Lines 202 to 210 in 075f444

	auto coors_clean = coors.masked_fill(coors.lt(0).any(-1, true), -1);
	std::tie(out_coors, coors_map, reduce_count) =
	at::unique_dim(coors_clean, 0, true, true, true);
	// the first element of out_coors is always (-1,-1,-1) and should be removed
	out_coors = out_coors.slice(0, 1);
	reduce_count = reduce_count.slice(0, 1).to(torch::kInt32);
	coors_map = coors_map.to(torch::kInt32) - 1;

The root cause is that the assumption here in the comment is too strong:

// the first element of out_coors is always (-1,-1,-1) and should be removed

It is correct for most cases where the input has at least one invalid (out of bound) point. But it is wrong when all input points are within the bound.

I'll launch a PR for this.