commit 696f2291b2f1d4905b7e30290bc4c0daba55ca6c
author Justin Klaassen <justinklaassen@google.com> Fri May 30 18:13:04 2014 -0700
committer Justin Klaassen <justinklaassen@google.com> Sat May 31 11:23:34 2014 -0700
tree a55db32c96962951402cf28830a0d2f6d2c1b737
parent 4b3af0578b1a44038856bc56244aea8aaeac22d1

Fix floating point errors

Bug: 15332726
Change-Id: I6e07f3281192760d3e5f001ba658686ddd42495b

src/com/android/calculator2/CalculatorExpressionEvaluator.java

diff --git a/src/com/android/calculator2/CalculatorExpressionEvaluator.java b/src/com/android/calculator2/CalculatorExpressionEvaluator.java
index 0f41423..992735c 100644
--- a/src/com/android/calculator2/CalculatorExpressionEvaluator.java
+++ b/src/com/android/calculator2/CalculatorExpressionEvaluator.java
@@ -23,7 +23,8 @@
 import org.javia.arity.Util;
 
 public class CalculatorExpressionEvaluator {
-    private static final int MAX_DIGITS = 14;
+    private static final int MAX_DIGITS = 12;
+    private static final int ROUNDING_DIGITS = 1;
 
     private final Symbols mSymbols;
     private final CalculatorExpressionTokenizer mTokenizer;
@@ -65,8 +66,11 @@
             if (Double.isNaN(result)) {
                 callback.onEvaluate(expr, null, mErrorNaN);
             } else {
+                // The arity library uses floating point arithmetic when evaluating the expression
+                // leading to precision errors in the result. The method doubleToString hides these
+                // errors; rounding the result by dropping N digits of precision.
                 callback.onEvaluate(expr, mTokenizer.getLocalizedExpression(
-                        Util.doubleToString(result, MAX_DIGITS, 0)), null);
+                        Util.doubleToString(result, MAX_DIGITS, ROUNDING_DIGITS)), null);
             }
         } catch (SyntaxException e) {
             callback.onEvaluate(expr, null, mErrorSyntax);