Adding Peter's Swish Op ULP analysis. (#42573)
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/42573
* Generate the ULP png files for different ranges.
Test Plan: test_op_ulp_error.py
Reviewed By: hyuen
Differential Revision: D22938572
fbshipit-source-id: 6374bef6d44c38e1141030d44029dee99112cd18
diff --git a/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py b/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py
index 36577f7..f9615c5 100644
--- a/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py
+++ b/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py
@@ -3,22 +3,17 @@
from __future__ import print_function
from __future__ import unicode_literals
-import ctypes
import numpy as np
-import os
import caffe2.python.fakelowp.init_shared_libs # noqa
-
-from hypothesis import given, settings
+from hypothesis import given
from hypothesis import strategies as st
-
-
from caffe2.proto import caffe2_pb2
-from caffe2.python import dyndep
from caffe2.python import core
from caffe2.python import workspace
from caffe2.python.onnx.onnxifi import onnxifi_caffe2_net
from caffe2.python.fakelowp.test_utils import print_test_debug_info
+from caffe2.python.oss.fakelowp.test_utils import compute_ulp_error
import caffe2.python.serialized_test.serialized_test_util as serial
core.GlobalInit(["caffe2", "--caffe2_log_level=-3", "--glow_global_fp16=1"])
@@ -119,12 +114,9 @@
class UnaryOpTest(serial.SerializedTestCase):
- def _test_unary_op(self, opname, value, rtol=1e-5, atol=1e-8):
+ def _test_unary_op(self, opname, X, rtol=1e-5, atol=1e-8):
workspace.ResetWorkspace()
- n = 1
- m = 10001
- X = np.linspace(-value, value, num=m, dtype=np.float32)
pred_net = caffe2_pb2.NetDef()
pred_net.name = "pred"
pred_net.external_input.append("X")
@@ -147,7 +139,7 @@
)
print("REF NET = {}".format(ref_net))
- shape_hints = {"X": (n, m)}
+ shape_hints = {"X": X.shape}
pred_net_onnxified = onnxifi_caffe2_net(pred_net,
shape_hints,
debug=True,
@@ -167,8 +159,6 @@
workspace.RunNet(ref_net.name)
Y_c2 = workspace.FetchBlob('Y')
-
-
if not np.allclose(Y_c2, Y_glow, rtol=atol, atol=atol):
diff = np.abs(Y_c2 - Y_glow)
np.save('/tmp/' + opname + 'diff', diff)
@@ -181,19 +171,42 @@
})
assert(0)
+ return Y_glow
+
+ def _test_op_w_ulp_error(self, opname, regions, atol=0, err_threshold=2):
+ ulp_err = 0
+ for x0, x1 in regions:
+ X = np.linspace(x0, x1, num=1025, dtype=np.float16).astype(np.float32)
+ Y_glow = self._test_unary_op(opname, X, atol=atol)
+ region_err = compute_ulp_error(opname, X, Y_glow)
+ ulp_err = max(np.max(np.abs(region_err)), ulp_err)
+ if (ulp_err > err_threshold):
+ print(r'{} Op detected ulp_err={}'.format(opname, ulp_err))
+ assert(0)
+
# These tests doesn't need to run multiple times given that it is a
# linear sweep and it is deterministic.
# Once hypothesis.testing version is updated, we can re-enable
# testing with different hypothesis examples.
def test_sigmoid(self):
- self._test_unary_op("Sigmoid", value=20)
+ opname = "Sigmoid"
+ regions = [[-8., -4.], [-4., -2.], [-2., -1.], [-1., -.5], [-.5, -.25],
+ [-.25, .25], [.25, .5], [.5, 1.], [1., 2.], [2., 4.],
+ [4., 8.]]
+ self._test_op_w_ulp_error(opname, regions, atol=0, err_threshold=2.5)
# These tests doesn't need to run multiple times given that it is a
# linear sweep and it is deterministic.
# Once hypothesis.testing version is updated, we can re-enable
# testing with different hypothesis examples.
def test_tanh(self):
- self._test_unary_op("Tanh", value=20)
+ opname = "Tanh"
+ regions = [[2.**(-9), 2.**(-8)], [2.**(-8), 2.**(-7)],
+ [2.**(-7), 2.**(-6)], [2.**(-6), 2.**(-5)],
+ [2.**(-5), 2.**(-4)], [2.**(-4), 2.**(-3)],
+ [2.**(-3), 2.**(-2)], [2.**(-2), 2.**(-1)],
+ [2.**(-1), 1.], [1., 2.], [2., 4.], [4., 8.]]
+ self._test_op_w_ulp_error(opname, regions, atol=0, err_threshold=2)
# These tests doesn't need to run multiple times given that it is a
# linear sweep and it is deterministic.
@@ -201,7 +214,10 @@
# testing with different hypothesis examples.
# TODO: move atol to 1e-8 once we get a non-lowered swish implementation
def test_swish(self):
- self._test_unary_op("Swish", value=20, atol=0.008)
+ opname = "Swish"
+ regions = [[-20.5, -11.], [-11., -8.], [-8., -1.], [-1., -0.1],
+ [-1. / 8., 1. / 8.], [1. / 8, 5.], [5., 8.]]
+ self._test_op_w_ulp_error(opname, regions, atol=0.008, err_threshold=384)
# These tests doesn't need to run multiple times given that it is a
# linear sweep and it is deterministic.
@@ -328,6 +344,6 @@
if not np.allclose(Y_c2, Y_glow):
diff = np.abs((Y_glow - Y_c2) / (Y_c2 + kEpsilon))
print_test_debug_info("Relu", {
- "seed":seed, "X": X,
+ "seed": seed, "X": X,
"Y_glow": Y_glow, "Y_c2": Y_c2, "diff": diff})
assert(0)
diff --git a/caffe2/python/fakelowp/test_utils.py b/caffe2/python/fakelowp/test_utils.py
index 275289b..75e4422 100644
--- a/caffe2/python/fakelowp/test_utils.py
+++ b/caffe2/python/fakelowp/test_utils.py
@@ -6,7 +6,6 @@
import sys
import numpy as np
-
def print_test_debug_info(testname, items_dict):
filename = "debug_operator_onnxifi_" + testname + ".txt"
np.set_printoptions(threshold=sys.maxsize)
@@ -16,7 +15,6 @@
f.write("{}\n".format(key))
f.write("{}\n".format(value))
-
def print_net(net):
for i in net.external_input:
print("Input: {}".format(i))
@@ -28,3 +26,40 @@
print(" input: {}".format(x))
for y in op.output:
print(" output: {}".format(y))
+
+def _sigmoid(x):
+ return 1. / (1. + np.exp(np.float64(-x)))
+
+def _tanh(x):
+ return np.tanh(np.float64(x))
+
+def _swish(x):
+ return np.float64(x) * _sigmoid(x)
+
+def _gelu_by_sigmoid(x):
+ return np.float64(x) / (1. + np.exp(np.float64(x) * 1.702))
+
+
+def _acc_func(opname, x):
+ if opname == "Swish":
+ return _swish(x)
+ elif opname == "Sigmoid":
+ return _sigmoid(x)
+ elif opname == "Tanh":
+ return _tanh(x)
+ elif opname == "Gelu":
+ return _gelu_by_sigmoid(x)
+ else:
+ return x
+
+def _get_ulp16(x):
+ abs_x = np.abs(x)
+ mask = (abs_x > 2.**(-14))
+ abs_x = mask * abs_x + (1 - mask) * 2.**(-14)
+ k = np.floor(np.log2(abs_x))
+ return 2.**(k - 10)
+
+def compute_ulp_error(opname, xvec, y_nnpi):
+ y_acc = _acc_func(opname, np.float64(xvec))
+ scale = 1. / _get_ulp16(y_acc)
+ return (y_nnpi - y_acc) * scale
