Move proposal generation headers to oss.
Summary: TSIA - it used to cause build errors.
Reviewed By: pietern
Differential Revision: D6652354
fbshipit-source-id: fd291f662e3793b6d11a7e02e1acc741c027a1fd
diff --git a/caffe2/operators/generate_proposals_op.h b/caffe2/operators/generate_proposals_op.h
new file mode 100644
index 0000000..14b7452
--- /dev/null
+++ b/caffe2/operators/generate_proposals_op.h
@@ -0,0 +1,118 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#ifndef CAFFE2_OPERATORS_GENERATE_PROPOSALS_OP_H_
+#define CAFFE2_OPERATORS_GENERATE_PROPOSALS_OP_H_
+
+#include "caffe2/core/context.h"
+#include "caffe2/core/operator.h"
+#include "caffe2/utils/eigen_utils.h"
+#include "caffe2/utils/math.h"
+
+namespace caffe2 {
+
+namespace utils {
+
+// A sub tensor view
+template <class T>
+class ConstTensorView {
+ public:
+ ConstTensorView(const T* data, const std::vector<int>& dims)
+ : data_(data), dims_(dims) {}
+
+ int ndim() const {
+ return dims_.size();
+ }
+ const std::vector<int>& dims() const {
+ return dims_;
+ }
+ int dim(int i) const {
+ DCHECK_LE(i, dims_.size());
+ return dims_[i];
+ }
+ const T* data() const {
+ return data_;
+ }
+ size_t size() const {
+ return std::accumulate(
+ dims_.begin(), dims_.end(), 1, std::multiplies<size_t>());
+ }
+
+ private:
+ const T* data_ = nullptr;
+ std::vector<int> dims_;
+};
+
+// Generate a list of bounding box shapes for each pixel based on predefined
+// bounding box shapes 'anchors'.
+// anchors: predefined anchors, size(A, 4)
+// Return: all_anchors_vec: (H * W, A * 4)
+// Need to reshape to (H * W * A, 4) to match the format in python
+ERMatXf ComputeAllAnchors(
+ const TensorCPU& anchors,
+ int height,
+ int width,
+ float feat_stride);
+
+} // namespace utils
+
+// C++ implementation of GenerateProposalsOp
+// Generate bounding box proposals for Faster RCNN. The propoasls are generated
+// for a list of images based on image score 'score', bounding box
+// regression result 'deltas' as well as predefined bounding box shapes
+// 'anchors'. Greedy non-maximum suppression is applied to generate the
+// final bounding boxes.
+// Reference: detection.caffe2/lib/ops/generate_proposals.py
+template <class Context>
+class GenerateProposalsOp final : public Operator<Context> {
+ public:
+ USE_OPERATOR_CONTEXT_FUNCTIONS;
+ GenerateProposalsOp(const OperatorDef& operator_def, Workspace* ws)
+ : Operator<Context>(operator_def, ws),
+ spatial_scale_(
+ OperatorBase::GetSingleArgument<float>("spatial_scale", 1.0 / 16)),
+ feat_stride_(1.0 / spatial_scale_),
+ rpn_pre_nms_topN_(
+ OperatorBase::GetSingleArgument<int>("pre_nms_topN", 6000)),
+ rpn_post_nms_topN_(
+ OperatorBase::GetSingleArgument<int>("post_nms_topN", 300)),
+ rpn_nms_thresh_(
+ OperatorBase::GetSingleArgument<float>("nms_thresh", 0.7f)),
+ rpn_min_size_(OperatorBase::GetSingleArgument<float>("min_size", 16)) {}
+
+ ~GenerateProposalsOp() {}
+
+ bool RunOnDevice() override;
+
+ // Generate bounding box proposals for a given image
+ // im_info: [height, width, im_scale]
+ // all_anchors: (H * W * A, 4)
+ // bbox_deltas_tensor: (4 * A, H, W)
+ // scores_tensor: (A, H, W)
+ // out_boxes: (n, 5)
+ // out_probs: n
+ void ProposalsForOneImage(
+ const Eigen::Array3f& im_info,
+ const Eigen::Map<const ERMatXf>& all_anchors,
+ const utils::ConstTensorView<float>& bbox_deltas_tensor,
+ const utils::ConstTensorView<float>& scores_tensor,
+ ERArrXXf* out_boxes,
+ EArrXf* out_probs) const;
+
+ protected:
+ // spatial_scale_ must be declared before feat_stride_
+ float spatial_scale_{1.0};
+ float feat_stride_{1.0};
+
+ // RPN_PRE_NMS_TOP_N
+ int rpn_pre_nms_topN_{6000};
+ // RPN_POST_NMS_TOP_N
+ int rpn_post_nms_topN_{300};
+ // RPN_NMS_THRESH
+ float rpn_nms_thresh_{0.7};
+ // RPN_MIN_SIZE
+ float rpn_min_size_{16};
+};
+
+} // namespace caffe2
+
+#endif // CAFFE2_OPERATORS_GENERATE_PROPOSALS_OP_H_
diff --git a/caffe2/operators/generate_proposals_op_util_boxes.h b/caffe2/operators/generate_proposals_op_util_boxes.h
new file mode 100644
index 0000000..4e67598
--- /dev/null
+++ b/caffe2/operators/generate_proposals_op_util_boxes.h
@@ -0,0 +1,135 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#ifndef CAFFE2_OPERATORS_UTILS_BOXES_H_
+#define CAFFE2_OPERATORS_UTILS_BOXES_H_
+
+#include "caffe2/utils/eigen_utils.h"
+#include "caffe2/utils/math.h"
+
+// Bounding box utils for generate_proposals_op
+// Reference: detection.caffe2/lib/utils/boxes.py
+
+namespace caffe2 {
+namespace utils {
+
+// Default value for minimum bounding box width and height after bounding box
+// transformation (bbox_transform()) in log-space
+const float BBOX_XFORM_CLIP_DEFAULT = log(1000.0 / 16.0);
+
+// Forward transform that maps proposal boxes to ground-truth boxes using
+// bounding-box regression deltas.
+// boxes: pixel coordinates of the bounding boxes
+// size (M, 4), format [x1; y1; x2; y2], x2 >= x1, y2 >= y1
+// deltas: bounding box translations and scales
+// size (M, 4), format [dx; dy; dw; dh]
+// dx, dy: scale-invariant translation of the center of the bounding box
+// dw, dh: log-space sclaing of the width and height of the bounding box
+// weights: weights [wx, wy, ww, wh] for the deltas
+// bbox_xform_clip: minimum bounding box width and height in log-space after
+// transofmration
+// return: pixel coordinates of the bounding boxes
+// size (M, 4), format [x1; y1; x2; y2]
+// see "Rich feature hierarchies for accurate object detection and semantic
+// segmentation" Appendix C for more details
+template <class Derived1, class Derived2>
+EArrXXt<typename Derived1::Scalar> bbox_transform(
+ const Eigen::ArrayBase<Derived1>& boxes,
+ const Eigen::ArrayBase<Derived2>& deltas,
+ const std::vector<typename Derived2::Scalar>& weights =
+ std::vector<typename Derived2::Scalar>{1.0, 1.0, 1.0, 1.0},
+ const float bbox_xform_clip = BBOX_XFORM_CLIP_DEFAULT) {
+ using T = typename Derived1::Scalar;
+ using EArrXX = EArrXXt<T>;
+ using EArrX = EArrXt<T>;
+
+ if (boxes.rows() == 0) {
+ return EArrXX::Zero(T(0), deltas.cols());
+ }
+
+ CAFFE_ENFORCE_EQ(boxes.rows(), deltas.rows());
+ CAFFE_ENFORCE_EQ(boxes.cols(), 4);
+ CAFFE_ENFORCE_EQ(deltas.cols(), 4);
+
+ EArrX widths = boxes.col(2) - boxes.col(0) + T(1.0);
+ EArrX heights = boxes.col(3) - boxes.col(1) + T(1.0);
+ auto ctr_x = boxes.col(0) + T(0.5) * widths;
+ auto ctr_y = boxes.col(1) + T(0.5) * heights;
+
+ auto dx = deltas.col(0).template cast<T>() / weights[0];
+ auto dy = deltas.col(1).template cast<T>() / weights[1];
+ auto dw =
+ (deltas.col(2).template cast<T>() / weights[2]).cwiseMin(bbox_xform_clip);
+ auto dh =
+ (deltas.col(3).template cast<T>() / weights[3]).cwiseMin(bbox_xform_clip);
+
+ EArrX pred_ctr_x = dx * widths + ctr_x;
+ EArrX pred_ctr_y = dy * heights + ctr_y;
+ EArrX pred_w = dw.exp() * widths;
+ EArrX pred_h = dh.exp() * heights;
+
+ EArrXX pred_boxes = EArrXX::Zero(deltas.rows(), deltas.cols());
+ // x1
+ pred_boxes.col(0) = pred_ctr_x - T(0.5) * pred_w;
+ // y1
+ pred_boxes.col(1) = pred_ctr_y - T(0.5) * pred_h;
+ // x2
+ pred_boxes.col(2) = pred_ctr_x + T(0.5) * pred_w;
+ // y2
+ pred_boxes.col(3) = pred_ctr_y + T(0.5) * pred_h;
+
+ return pred_boxes;
+}
+
+// Clip boxes to image boundaries
+// boxes: pixel coordinates of bounding box, size (M * 4)
+template <class Derived>
+EArrXXt<typename Derived::Scalar>
+clip_boxes(const Eigen::ArrayBase<Derived>& boxes, int height, int width) {
+ CAFFE_ENFORCE_EQ(boxes.cols(), 4);
+
+ EArrXXt<typename Derived::Scalar> ret(boxes.rows(), boxes.cols());
+
+ // x1 >= 0 && x1 < width
+ ret.col(0) = boxes.col(0).cwiseMin(width - 1).cwiseMax(0);
+ // y1 >= 0 && y1 < height
+ ret.col(1) = boxes.col(1).cwiseMin(height - 1).cwiseMax(0);
+ // x2 >= 0 && x2 < width
+ ret.col(2) = boxes.col(2).cwiseMin(width - 1).cwiseMax(0);
+ // y2 >= 0 && y2 < height
+ ret.col(3) = boxes.col(3).cwiseMin(height - 1).cwiseMax(0);
+
+ return ret;
+}
+
+// Only keep boxes with both sides >= min_size and center within the image.
+// boxes: pixel coordinates of bounding box, size (M * 4)
+// im_info: [height, width, img_scale]
+// return: row indices for 'boxes'
+template <class Derived>
+std::vector<int> filter_boxes(
+ const Eigen::ArrayBase<Derived>& boxes,
+ double min_size,
+ const Eigen::Array3f& im_info) {
+ CAFFE_ENFORCE_EQ(boxes.cols(), 4);
+
+ // Scale min_size to match image scale
+ min_size *= im_info[2];
+
+ using T = typename Derived::Scalar;
+ using EArrX = EArrXt<T>;
+
+ EArrX ws = boxes.col(2) - boxes.col(0) + T(1);
+ EArrX hs = boxes.col(3) - boxes.col(1) + T(1);
+ EArrX x_ctr = boxes.col(0) + ws / T(2);
+ EArrX y_ctr = boxes.col(1) + hs / T(2);
+
+ EArrXb keep = (ws >= min_size) && (hs >= min_size) &&
+ (x_ctr < T(im_info[1])) && (y_ctr < T(im_info[0]));
+
+ return GetArrayIndices(keep);
+}
+
+} // namespace utils
+} // namespace caffe2
+
+#endif // CAFFE2_OPERATORS_UTILS_BOXES_H_
diff --git a/caffe2/utils/eigen_utils.h b/caffe2/utils/eigen_utils.h
new file mode 100644
index 0000000..b00e355
--- /dev/null
+++ b/caffe2/utils/eigen_utils.h
@@ -0,0 +1,130 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#ifndef CAFFE2_OPERATORS_UTILS_EIGEN_H_
+#define CAFFE2_OPERATORS_UTILS_EIGEN_H_
+
+#include "Eigen/Core"
+#include "Eigen/Dense"
+#include "caffe2/core/logging.h"
+
+namespace caffe2 {
+
+// 1-d array
+template <typename T>
+using EArrXt = Eigen::Array<T, Eigen::Dynamic, 1>;
+using EArrXf = Eigen::ArrayXf;
+using EArrXd = Eigen::ArrayXd;
+using EArrXi = Eigen::ArrayXi;
+using EArrXb = EArrXt<bool>;
+
+// 2-d array, column major
+template <typename T>
+using EArrXXt = Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>;
+using EArrXXf = Eigen::ArrayXXf;
+
+// 2-d array, row major
+template <typename T>
+using ERArrXXt =
+ Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
+using ERArrXXf = ERArrXXt<float>;
+
+// 1-d vector
+template <typename T>
+using EVecXt = Eigen::Matrix<T, Eigen::Dynamic, 1>;
+using EVecXd = Eigen::VectorXd;
+using EVecXf = Eigen::VectorXf;
+
+// 1-d row vector
+using ERVecXd = Eigen::RowVectorXd;
+using ERVecXf = Eigen::RowVectorXf;
+
+// 2-d matrix, column major
+template <typename T>
+using EMatXt = Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>;
+using EMatXd = Eigen::MatrixXd;
+using EMatXf = Eigen::MatrixXf;
+
+// 2-d matrix, row major
+template <typename T>
+using ERMatXt =
+ Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
+using ERMatXd = ERMatXt<double>;
+using ERMatXf = ERMatXt<float>;
+
+namespace utils {
+
+template <typename T>
+Eigen::Map<const EArrXt<T>> AsEArrXt(const std::vector<T>& arr) {
+ return {arr.data(), static_cast<int>(arr.size())};
+}
+template <typename T>
+Eigen::Map<EArrXt<T>> AsEArrXt(std::vector<T>& arr) {
+ return {arr.data(), static_cast<int>(arr.size())};
+}
+
+// return a sub array of 'array' based on indices 'indices'
+template <class Derived, class Derived1, class Derived2>
+void GetSubArray(
+ const Eigen::ArrayBase<Derived>& array,
+ const Eigen::ArrayBase<Derived1>& indices,
+ Eigen::ArrayBase<Derived2>* out_array) {
+ CAFFE_ENFORCE_EQ(array.cols(), 1);
+ // using T = typename Derived::Scalar;
+
+ out_array->derived().resize(indices.size());
+ for (int i = 0; i < indices.size(); i++) {
+ DCHECK_LT(indices[i], array.size());
+ (*out_array)[i] = array[indices[i]];
+ }
+}
+
+// return a sub array of 'array' based on indices 'indices'
+template <class Derived, class Derived1>
+EArrXt<typename Derived::Scalar> GetSubArray(
+ const Eigen::ArrayBase<Derived>& array,
+ const Eigen::ArrayBase<Derived1>& indices) {
+ using T = typename Derived::Scalar;
+ EArrXt<T> ret(indices.size());
+ GetSubArray(array, indices, &ret);
+ return ret;
+}
+
+// return a sub array of 'array' based on indices 'indices'
+template <class Derived>
+EArrXt<typename Derived::Scalar> GetSubArray(
+ const Eigen::ArrayBase<Derived>& array,
+ const std::vector<int>& indices) {
+ return GetSubArray(array, AsEArrXt(indices));
+}
+
+// return 2d sub array of 'array' based on row indices 'row_indices'
+template <class Derived, class Derived1, class Derived2>
+void GetSubArrayRows(
+ const Eigen::ArrayBase<Derived>& array2d,
+ const Eigen::ArrayBase<Derived1>& row_indices,
+ Eigen::ArrayBase<Derived2>* out_array) {
+ out_array->derived().resize(row_indices.size(), array2d.cols());
+
+ for (int i = 0; i < row_indices.size(); i++) {
+ DCHECK_LT(row_indices[i], array2d.size());
+ out_array->row(i) =
+ array2d.row(row_indices[i]).template cast<typename Derived2::Scalar>();
+ }
+}
+
+// return indices of 1d array for elements evaluated to true
+template <class Derived>
+std::vector<int> GetArrayIndices(const Eigen::ArrayBase<Derived>& array) {
+ std::vector<int> ret;
+ for (int i = 0; i < array.size(); i++) {
+ if (array[i]) {
+ ret.push_back(i);
+ }
+ }
+ return ret;
+}
+
+} // namespace utils
+} // namespace caffe2
+
+#endif
