Cyclical learning rate multiplier: use fabs(base_lr) (#25628)
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/25628
We figured that base_lr is negative in learning_rate_functors.h. So using fabs(base_lr) for cyclical learning rate multiplier
computation.
Test Plan: Canary: f135306794
Reviewed By: chenshouyuan
Differential Revision: D17167635
fbshipit-source-id: e7fb55835f9fc07712edd63e81f1cf355e05b9f4
diff --git a/caffe2/sgd/learning_rate_functors.h b/caffe2/sgd/learning_rate_functors.h
index aec2435..bcdb990 100644
--- a/caffe2/sgd/learning_rate_functors.h
+++ b/caffe2/sgd/learning_rate_functors.h
@@ -261,7 +261,8 @@
T operator()(const int64_t iter) const override {
int cycle = static_cast<int>((iter / (2 * stepsize_)) + 1);
T x = abs(static_cast<T>(iter) / stepsize_ - 2 * cycle + 1);
- return (1 + (T(max_lr_) / T(base_lr_) - 1) * std::max(T(0.0), (1 - x)));
+ return (
+ 1 + (T(max_lr_) / T(fabs(base_lr_)) - 1) * std::max(T(0.0), (1 - x)));
}
T base_lr_;
T max_lr_;
