From c1f013b707e70d7c89ac231335080b7116a3d2b1 Mon Sep 17 00:00:00 2001 From: serso Date: Tue, 12 Apr 2016 16:22:27 +0200 Subject: [PATCH] Simple format should be used for small numbers if rounding is enabled In order to avoid getting cos(90)=-13.3123123E-17 --- .../org/solovyev/common/NumberFormatter.java | 20 ++++++++++--------- .../solovyev/common/NumberFormatterTest.java | 14 ++++++++----- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/jscl/src/main/java/org/solovyev/common/NumberFormatter.java b/jscl/src/main/java/org/solovyev/common/NumberFormatter.java index 857b9f9e..5eb44bfa 100644 --- a/jscl/src/main/java/org/solovyev/common/NumberFormatter.java +++ b/jscl/src/main/java/org/solovyev/common/NumberFormatter.java @@ -1,12 +1,16 @@ package org.solovyev.common; +import java.math.BigDecimal; + +import javax.annotation.Nonnull; + import midpcalc.Real; -import javax.annotation.Nonnull; -import java.math.BigDecimal; - import static java.lang.Math.pow; -import static midpcalc.Real.NumberFormat.*; +import static midpcalc.Real.NumberFormat.FSE_ENG; +import static midpcalc.Real.NumberFormat.FSE_FIX; +import static midpcalc.Real.NumberFormat.FSE_NONE; +import static midpcalc.Real.NumberFormat.FSE_SCI; public class NumberFormatter { @@ -88,12 +92,10 @@ public class NumberFormatter { if (radix != 10) { return true; } - if (absValue < pow(10, -MAX_PRECISION)) { - // should never use simple format for small numbers - return false; - } if (format == FSE_NONE) { - return true; + // simple format should be used only if rounding is on or if number is big enough + final boolean round = precision != NO_ROUNDING; + return round || absValue >= pow(10, -MAX_PRECISION); } if (pow(10, -simpleFormatMagnitude) <= absValue && absValue < pow(10, simpleFormatMagnitude)) { return true; diff --git a/jscl/src/test/java/org/solovyev/common/NumberFormatterTest.java b/jscl/src/test/java/org/solovyev/common/NumberFormatterTest.java index c00a02be..45ce11f1 100644 --- a/jscl/src/test/java/org/solovyev/common/NumberFormatterTest.java +++ b/jscl/src/test/java/org/solovyev/common/NumberFormatterTest.java @@ -69,6 +69,10 @@ public class NumberFormatterTest { assertEquals("3.333333333333333E-19", numberFormatter.format(pow(10, -18) / 3)); assertEquals("1234567890000000000", numberFormatter.format(123456789 * pow(10, 10))); assertEquals("0.0000000000000001", numberFormatter.format(pow(10, -16))); + assertEquals("1E-17", numberFormatter.format(pow(10, -17))); + assertEquals("1E-18", numberFormatter.format(pow(10, -18))); + assertEquals("1.5E-18", numberFormatter.format(1.5 * pow(10, -18))); + assertEquals("1E-100", numberFormatter.format(pow(10, -100))); testSimpleFormat(); } @@ -81,9 +85,13 @@ public class NumberFormatterTest { assertEquals("1", numberFormatter.format(1d)); assertEquals("0", numberFormatter.format(pow(10, -6))); assertEquals("0.33333", numberFormatter.format(1d / 3)); - assertEquals("3.33333E-19", numberFormatter.format(pow(10, -18) / 3)); + assertEquals("0", numberFormatter.format(pow(10, -18) / 3)); assertEquals("1234567890000000000", numberFormatter.format(123456789 * pow(10, 10))); assertEquals("0", numberFormatter.format(pow(10, -16))); + assertEquals("0", numberFormatter.format(pow(10, -17))); + assertEquals("0", numberFormatter.format(pow(10, -18))); + assertEquals("0", numberFormatter.format(1.5 * pow(10, -18))); + assertEquals("0", numberFormatter.format(pow(10, -100))); testSimpleFormat(); } @@ -98,10 +106,6 @@ public class NumberFormatterTest { assertEquals("1E100", numberFormatter.format(pow(10, 100))); assertEquals("0.01", numberFormatter.format(pow(10, -2))); - assertEquals("1E-17", numberFormatter.format(pow(10, -17))); - assertEquals("1E-18", numberFormatter.format(pow(10, -18))); - assertEquals("1.5E-18", numberFormatter.format(1.5 * pow(10, -18))); - assertEquals("1E-100", numberFormatter.format(pow(10, -100))); assertEquals("5000000000000000000", numberFormatter.format(5000000000000000000d)); assertEquals("5000000000000000000", numberFormatter.format(5000000000000000001d));