Add number formatting test and fix rounding

jscl/src/main/java/org/solovyev/common/NumberFormatter.java

@@ -1,13 +1,17 @@
 package org.solovyev.common;
 
-import midpcalc.Real;
-
-import javax.annotation.Nonnull;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 
+import javax.annotation.Nonnull;
+
+import midpcalc.Real;
+
 import static java.lang.Math.pow;
-import static midpcalc.Real.NumberFormat.*;
+import static midpcalc.Real.NumberFormat.FSE_ENG;
+import static midpcalc.Real.NumberFormat.FSE_FIX;
+import static midpcalc.Real.NumberFormat.FSE_NONE;
+import static midpcalc.Real.NumberFormat.FSE_SCI;
 
 public class NumberFormatter {
 
@@ -40,6 +44,10 @@ public class NumberFormatter {
     }
 
     public void setPrecision(int precision) {
+        if (precision == NO_ROUNDING) {
+            this.precision = NO_ROUNDING;
+            return;
+        }
         this.precision = Math.max(MIN_PRECISION, Math.min(precision, MAX_PRECISION));
     }
 
@@ -63,9 +71,10 @@ public class NumberFormatter {
         double absValue = Math.abs(value);
         final boolean simpleFormat = useSimpleFormat(radix, absValue);
 
-        final int effectivePrecision = precision == NO_ROUNDING ? MAX_PRECISION : precision;
+        int precision = getPrecision();
         if (simpleFormat) {
-            final int newScale = (int) (effectivePrecision * Math.max(1, radix / 10f));
+            precision += 1;
+            final int newScale = Math.max(1, (int) (precision * Math.max(1, radix / 10f)) - 1);
             value = BigDecimal.valueOf(value).setScale(newScale, BigDecimal.ROUND_HALF_UP).doubleValue();
             absValue = Math.abs(value);
         }
@@ -79,7 +88,7 @@ public class NumberFormatter {
             numberFormat.fse = format;
         }
         numberFormat.thousand = groupingSeparator;
-        numberFormat.precision = effectivePrecision;
+        numberFormat.precision = precision;
         numberFormat.base = radix;
         numberFormat.maxwidth = simpleFormat ? 100 : 30;
 
@@ -89,6 +98,10 @@ public class NumberFormatter {
         return prepare(value);
     }
 
+    private int getPrecision() {
+        return precision == NO_ROUNDING ? MAX_PRECISION : precision;
+    }
+
     @Nonnull
     public CharSequence format(@Nonnull BigInteger value, int radix) {
         checkRadix(radix);

jscl/src/test/java/jscl/JsclMathEngineTest.java

@@ -6,8 +6,6 @@ import org.solovyev.common.NumberFormatter;
 
 import midpcalc.Real;
 
-import midpcalc.Real;
-
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -46,18 +44,18 @@ public class JsclMathEngineTest {
 
             assertEquals("111 1111 0011 0110", me.format(32566d, NumeralBase.bin));
             assertEquals("100.0100110011", me.format(4.3d, NumeralBase.bin));
-            assertEquals("1 0001 0101 0011.010101011", me.format(4435.33423d, NumeralBase.bin));
+            assertEquals("1 0001 0101 0011.01010101101", me.format(4435.33423d, NumeralBase.bin));
-            assertEquals("1100.0101010101", me.format(12.3333d, NumeralBase.bin));
+            assertEquals("1100.01010101011", me.format(12.3333d, NumeralBase.bin));
             assertEquals("1 0011 1101 1110 0100 0011 0101 0101.00011111", me.format(333333333.1212213321d, NumeralBase.bin));
 
-            assertEquals("0.EEEEEEEEEF", me.format(14d / 15d, NumeralBase.hex));
+            assertEquals("0.EEEEEEEEEEF", me.format(14d / 15d, NumeralBase.hex));
             assertEquals("7F 36", me.format(32566d, NumeralBase.hex));
             assertEquals("24", me.format(36d, NumeralBase.hex));
             assertEquals("8", me.format(8d, NumeralBase.hex));
             assertEquals("1 3D", me.format(317d, NumeralBase.hex));
             assertEquals("13 DE 43 55.1F085BEF", me.format(333333333.1212213321d, NumeralBase.hex));
             assertEquals("D 25 0F 77 0A.6F7319", me.format(56456345354.43534534523459999d, NumeralBase.hex));
-            assertEquals("3 E7.4CCCCCCCCD", me.format(999.3d, NumeralBase.hex));
+            assertEquals("3 E7.4CCCCCCCCCD", me.format(999.3d, NumeralBase.hex));
 
             me.setPrecision(NumberFormatter.MAX_PRECISION);
             assertEquals("6.CCDA6A054226DB6E-19", me.format(0.00000000000000000000009d, NumeralBase.hex));

jscl/src/test/java/jscl/NumeralBaseTest.java

@@ -1,11 +1,12 @@
 package jscl;
 
+import org.junit.Assert;
+import org.junit.Test;
+
 import jscl.math.function.Constant;
 import jscl.math.function.ExtendedConstant;
 import jscl.math.function.IConstant;
 import jscl.text.ParseException;
-import org.junit.Assert;
-import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 
@@ -77,7 +78,7 @@ public class NumeralBaseTest {
             assertEquals("11111110", me.evaluate("111001+11000101"));
             assertEquals("1101100100101111", me.evaluate("11011001001011110/10"));
             assertEquals("1001000011001010", me.evaluate("11011001001011110/11"));
-            assertEquals("0.101010101010101", me.evaluate("10/11"));
+            assertEquals("0.1010101010101011", me.evaluate("10/11"));
 
             me.setNumeralBase(NumeralBase.hex);
             assertEquals("637B", me.evaluate("56CE+CAD"));

jscl/src/test/java/jscl/math/ExpressionTest.java

@@ -1,5 +1,19 @@
 package jscl.math;
 
+import org.junit.Test;
+import org.solovyev.common.NumberFormatter;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Set;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 import jscl.AngleUnit;
 import jscl.JsclMathEngine;
 import jscl.MathEngine;
@@ -9,20 +23,10 @@ import jscl.math.function.ExtendedConstant;
 import jscl.math.function.IConstant;
 import jscl.text.ParseException;
 import midpcalc.Real;
-import org.junit.Test;
-import org.solovyev.common.NumberFormatter;
 
-import javax.annotation.Nonnull;
+import static org.junit.Assert.assertEquals;
-import javax.annotation.Nullable;
+import static org.junit.Assert.assertTrue;
-import java.io.BufferedReader;
+import static org.junit.Assert.fail;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Set;
-
-import static org.junit.Assert.*;
 
 public class ExpressionTest {
 
@@ -438,10 +442,10 @@ public class ExpressionTest {
         try {
             me.setNumeralBase(NumeralBase.hex);
 
-            assertEquals("0.EEEEEEEEEEEEEC9", me.evaluate("0x:E/0x:F"));
+            assertEquals("0.EEEEEEEEEEEEEC88", me.evaluate("0x:E/0x:F"));
             assertEquals("E/F", me.simplify("0x:E/0x:F"));
 
-            assertEquals("0.EEEEEEEEEEEEEC9", me.evaluate("E/F"));
+            assertEquals("0.EEEEEEEEEEEEEC88", me.evaluate("E/F"));
             assertEquals("E/F", me.simplify("E/F"));
 
         } finally {

jscl/src/test/java/org/solovyev/common/NumberFormatterTest.java

@@ -8,6 +8,7 @@ import java.math.BigInteger;
 import static java.lang.Math.pow;
 import static java.math.BigInteger.TEN;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.solovyev.common.NumberFormatter.DEFAULT_MAGNITUDE;
 import static org.solovyev.common.NumberFormatter.NO_ROUNDING;
 
@@ -175,4 +176,30 @@ public class NumberFormatterTest {
         assertEquals("5E19", numberFormatter.format(50000000000000000000d));
         assertEquals("5E40", numberFormatter.format(50000000000000000000000000000000000000000d));
     }
+
+    @Test
+    public void testMaximumPrecision() throws Exception {
+        numberFormatter.useSimpleFormat();
+        numberFormatter.setPrecision(10);
+
+        for (int i = 0; i < 1000; i++) {
+            for (int j = 2; j < 1000; j += j - 1) {
+                for (int k = 2; k < 1000; k += k - 1) {
+                    final double first = makeDouble(i, j);
+                    final double second = makeDouble(i, 1000 - k);
+                    checkMaximumPrecision(first + "-" + second, numberFormatter.format(first - second));
+                    checkMaximumPrecision(second + "-" + first, numberFormatter.format(second - first));
+                    checkMaximumPrecision(second + "+" + first, numberFormatter.format(first + second));
+                }
+            }
+        }
+    }
+
+    private void checkMaximumPrecision(String expression, CharSequence value) {
+        assertTrue(expression + "=" + value, value.length() <= 8);
+    }
+
+    private static double makeDouble(int integerPart, int fractionalPart) {
+        return Double.parseDouble(integerPart + "." + fractionalPart);
+    }
 }