Open3D (C++ API)  0.15.1
ContinuousConv.h
Go to the documentation of this file.
1// ----------------------------------------------------------------------------
2// - Open3D: www.open3d.org -
3// ----------------------------------------------------------------------------
4// The MIT License (MIT)
5//
6// Copyright (c) 2018-2021 www.open3d.org
7//
8// Permission is hereby granted, free of charge, to any person obtaining a copy
9// of this software and associated documentation files (the "Software"), to deal
10// in the Software without restriction, including without limitation the rights
11// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12// copies of the Software, and to permit persons to whom the Software is
13// furnished to do so, subject to the following conditions:
14//
15// The above copyright notice and this permission notice shall be included in
16// all copies or substantial portions of the Software.
17//
18// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24// IN THE SOFTWARE.
25// ----------------------------------------------------------------------------
26
27#pragma once
28
29#include <tbb/parallel_for.h>
30
32
33namespace open3d {
34namespace ml {
35namespace impl {
36
39template <class TFeat,
40 class TOut,
41 class TReal,
42 class TIndex,
43 InterpolationMode INTERPOLATION,
44 CoordinateMapping MAPPING,
45 bool ALIGN_CORNERS,
46 bool INDIVIDUAL_EXTENT,
47 bool ISOTROPIC_EXTENT,
48 bool POINT_IMPORTANCE>
49void _CConvComputeFeaturesCPU(TOut* out_features,
50 const std::vector<int>& filter_dims,
51 const TFeat* filter,
52 size_t num_out,
53 const TReal* out_positions,
54 size_t num_inp,
55 const TReal* inp_positions,
56 const TFeat* inp_features,
57 const TFeat* inp_importance,
58 size_t neighbors_index_size,
59 const TIndex* neighbors_index,
60 const TFeat* neighbors_importance,
61 const int64_t* neighbors_row_splits,
62 const TReal* extents,
63 const TReal* offsets,
64 bool normalize) {
65 const bool NEIGHBORS_IMPORTANCE = neighbors_importance != nullptr;
66 const int VECSIZE = 32;
67 typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
68 typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
69 InterpolationVec_t interpolation;
70
71 const int in_channels = filter_dims[filter_dims.size() - 2];
72 const int out_channels = filter_dims[filter_dims.size() - 1];
73
74 int spatial_filter_size = 1;
75 for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
76 Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
77 filter_dims[0]);
78
79 memset(out_features, 0, sizeof(TOut) * num_out * out_channels);
80
81 tbb::parallel_for(
82 tbb::blocked_range<size_t>(0, num_out, 32),
83 [&](const tbb::blocked_range<size_t>& r) {
84 int range_length = r.end() - r.begin();
85
86 Eigen::Matrix<TOut, Eigen::Dynamic, 1> normalizers(range_length,
87 1);
88 normalizers.setZero();
89
90 Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
91 in_channels * spatial_filter_size, range_length);
92 B.setZero();
93
94 typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
95 Matrix infeat(VECSIZE, in_channels);
96
97 Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
98 offsets[2]);
99
100 Eigen::Array<TReal, VECSIZE, 3> inv_extents;
101 if (INDIVIDUAL_EXTENT == false) {
102 if (ISOTROPIC_EXTENT) {
103 inv_extents = 1 / extents[0];
104 } else {
105 inv_extents.col(0) = 1 / extents[0];
106 inv_extents.col(1) = 1 / extents[1];
107 inv_extents.col(2) = 1 / extents[2];
108 }
109 }
110
111 for (size_t out_idx = r.begin(); out_idx != r.end();
112 ++out_idx) {
113 const int out_col = out_idx - r.begin();
114 const size_t neighbor_start = neighbors_row_splits[out_idx];
115 const size_t neighbor_end =
116 neighbors_row_splits[out_idx + 1];
117
118 if (INDIVIDUAL_EXTENT) {
119 if (ISOTROPIC_EXTENT) {
120 inv_extents = 1 / extents[out_idx];
121 } else {
122 inv_extents.col(0) = 1 / extents[3 * out_idx + 0];
123 inv_extents.col(1) = 1 / extents[3 * out_idx + 1];
124 inv_extents.col(2) = 1 / extents[3 * out_idx + 2];
125 }
126 }
127
128 typename InterpolationVec_t::Weight_t interp_weights;
129 typename InterpolationVec_t::Idx_t interp_indices;
130
131 int vec_valid_count = 0;
132 Vec_t x, y, z;
133
134 // set to zero to avoid problems with vectors with less than
135 // VECSIZE valid entries
136 x.setZero();
137 y.setZero();
138 z.setZero();
139 for (size_t n = neighbor_start; n < neighbor_end; ++n) {
140 const size_t inp_idx = neighbors_index[n];
141 const int i = vec_valid_count;
142 x(i) = inp_positions[inp_idx * 3 + 0] -
143 out_positions[out_idx * 3 + 0];
144 y(i) = inp_positions[inp_idx * 3 + 1] -
145 out_positions[out_idx * 3 + 1];
146 z(i) = inp_positions[inp_idx * 3 + 2] -
147 out_positions[out_idx * 3 + 2];
148
149 const TFeat n_importance =
150 (NEIGHBORS_IMPORTANCE ? neighbors_importance[n]
151 : TFeat(1));
152 normalizers(out_col) += TOut(n_importance);
153
154 for (int ic = 0; ic < in_channels; ++ic)
155 infeat(i, ic) =
156 inp_features[inp_idx * in_channels + ic];
157
158 TFeat importance(1.0);
159 if (POINT_IMPORTANCE)
160 importance = inp_importance[inp_idx];
161 if (NEIGHBORS_IMPORTANCE) importance *= n_importance;
162
163 if (POINT_IMPORTANCE || NEIGHBORS_IMPORTANCE) {
164 for (int ic = 0; ic < in_channels; ++ic)
165 infeat(i, ic) *= importance;
166 }
167
168 ++vec_valid_count;
169 if (vec_valid_count == VECSIZE) {
170 ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
171 x, y, z, filter_size_xyz, inv_extents,
172 offsets_);
173 interpolation.Interpolate(
174 interp_weights, interp_indices, x, y, z,
175 filter_size_xyz, in_channels);
176 for (int k = 0; k < VECSIZE; ++k)
177 for (int j = 0; j < InterpolationVec_t::Size();
178 ++j) {
179 for (int ic = 0; ic < in_channels; ++ic)
180 B(interp_indices(j, k) + ic, out_col) +=
181 TFeat(interp_weights(j, k)) *
182 infeat(k, ic);
183 }
184 vec_valid_count = 0;
185 }
186 }
187 if (vec_valid_count) {
188 ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
189 x, y, z, filter_size_xyz, inv_extents,
190 offsets_);
191 interpolation.Interpolate(interp_weights,
192 interp_indices, x, y, z,
193 filter_size_xyz, in_channels);
194 for (int k = 0; k < vec_valid_count; ++k)
195 for (int j = 0; j < InterpolationVec_t::Size();
196 ++j) {
197 for (int ic = 0; ic < in_channels; ++ic)
198 B(interp_indices(j, k) + ic, out_col) +=
199 TFeat(interp_weights(j, k)) *
200 infeat(k, ic);
201 }
202 }
203
204 } // out_idx
205
206 Eigen::Map<const Eigen::Matrix<TFeat, Eigen::Dynamic,
207 Eigen::Dynamic>>
208 A(filter, out_channels,
209 spatial_filter_size * in_channels);
210 Eigen::Map<Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic>>
211 C(out_features + (r.begin() * out_channels),
212 out_channels, range_length);
213
214 C = (A * B).template cast<TOut>();
215 if (normalize) {
216 for (int i = 0; i < range_length; ++i) {
217 if (normalizers(i) != TOut(0))
218 C.col(i) /= normalizers(i);
219 }
220 }
221 });
222}
223
298template <class TFeat, class TOut, class TReal, class TIndex>
299void CConvComputeFeaturesCPU(TOut* out_features,
300 const std::vector<int>& filter_dims,
301 const TFeat* filter,
302 size_t num_out,
303 const TReal* out_positions,
304 size_t num_inp,
305 const TReal* inp_positions,
306 const TFeat* inp_features,
307 const TFeat* inp_importance,
308 size_t neighbors_index_size,
309 const TIndex* neighbors_index,
310 const TFeat* neighbors_importance,
311 const int64_t* neighbors_row_splits,
312 const TReal* extents,
313 const TReal* offsets,
314 InterpolationMode interpolation,
315 CoordinateMapping coordinate_mapping,
316 bool align_corners,
317 bool individual_extent,
318 bool isotropic_extent,
319 bool normalize) {
320 // Dispatch all template parameter combinations
321 bool has_importance = inp_importance;
322
323#define FN_PARAMETERS \
324 out_features, filter_dims, filter, num_out, out_positions, num_inp, \
325 inp_positions, inp_features, inp_importance, neighbors_index_size, \
326 neighbors_index, neighbors_importance, neighbors_row_splits, \
327 extents, offsets, normalize
328
329#define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
330 INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, HAS_IMPORTANCE) \
331 if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
332 ALIGN_CORNERS == align_corners && \
333 INDIVIDUAL_EXTENT == individual_extent && \
334 ISOTROPIC_EXTENT == isotropic_extent && \
335 HAS_IMPORTANCE == has_importance) \
336 _CConvComputeFeaturesCPU<TFeat, TOut, TReal, TIndex, INTERPOLATION, \
337 MAPPING, ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
338 ISOTROPIC_EXTENT, HAS_IMPORTANCE>( \
339 FN_PARAMETERS);
340
341#define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
342 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
343 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
344 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
345 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
346 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
347 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
348 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
349 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
350 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
351 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
352 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
353 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
354 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
355 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
356 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
357 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
358
359#define CALL_TEMPLATE3(INTERPOLATION) \
360 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
361 CALL_TEMPLATE2(INTERPOLATION, \
362 CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
363 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
364
365#define CALL_TEMPLATE4 \
366 CALL_TEMPLATE3(InterpolationMode::LINEAR) \
367 CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
368 CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
369
371
372#undef CALL_TEMPLATE
373#undef CALL_TEMPLATE2
374#undef CALL_TEMPLATE3
375#undef CALL_TEMPLATE4
376
377#undef FN_PARAMETERS
378}
379
380} // namespace impl
381} // namespace ml
382} // namespace open3d
#define CALL_TEMPLATE4
#define VECSIZE
InterpolationMode
Definition: ContinuousConvTypes.h:37
void _CConvComputeFeaturesCPU(TOut *out_features, const std::vector< int > &filter_dims, const TFeat *filter, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, bool normalize)
Definition: ContinuousConv.h:49
void CConvComputeFeaturesCPU(TOut *out_features, const std::vector< int > &filter_dims, const TFeat *filter, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConv.h:299
CoordinateMapping
Definition: ContinuousConvTypes.h:45
Definition: PinholeCameraIntrinsic.cpp:35
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:204