TnSb/android-calculatorpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simple format should be used for small numbers if rounding is enabled

Browse Source 
In order to avoid getting cos(90)=-13.3123123E-17
This commit is contained in:
serso 2016-04-12 16:22:27 +02:00
parent 19cb4d18d5
commit c1f013b707
2 changed files with 20 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
jscl/src/main/java/org/solovyev/common/NumberFormatter.java
View File

@ -1,12 +1,16 @@
package org.solovyev.common;
import java.math.BigDecimal;
import javax.annotation.Nonnull;
import midpcalc.Real;
import javax.annotation.Nonnull;
import java.math.BigDecimal;
import static java.lang.Math.pow;
import static midpcalc.Real.NumberFormat.*;
import static midpcalc.Real.NumberFormat.FSE_ENG;
import static midpcalc.Real.NumberFormat.FSE_FIX;
import static midpcalc.Real.NumberFormat.FSE_NONE;
import static midpcalc.Real.NumberFormat.FSE_SCI;
public class NumberFormatter {
@ -88,12 +92,10 @@ public class NumberFormatter {
if (radix != 10) {
return true;
}
if (absValue < pow(10, -MAX_PRECISION)) {
// should never use simple format for small numbers
return false;
}
if (format == FSE_NONE) {
return true;
// simple format should be used only if rounding is on or if number is big enough
final boolean round = precision != NO_ROUNDING;
return round || absValue >= pow(10, -MAX_PRECISION);
}
if (pow(10, -simpleFormatMagnitude) <= absValue && absValue < pow(10, simpleFormatMagnitude)) {
return true;

14
jscl/src/test/java/org/solovyev/common/NumberFormatterTest.java
View File

@ -69,6 +69,10 @@ public class NumberFormatterTest {
assertEquals("3.333333333333333E-19", numberFormatter.format(pow(10, -18) / 3));
assertEquals("1234567890000000000", numberFormatter.format(123456789 * pow(10, 10)));
assertEquals("0.0000000000000001", numberFormatter.format(pow(10, -16)));
assertEquals("1E-17", numberFormatter.format(pow(10, -17)));
assertEquals("1E-18", numberFormatter.format(pow(10, -18)));
assertEquals("1.5E-18", numberFormatter.format(1.5 * pow(10, -18)));
assertEquals("1E-100", numberFormatter.format(pow(10, -100)));
testSimpleFormat();
}
@ -81,9 +85,13 @@ public class NumberFormatterTest {
assertEquals("1", numberFormatter.format(1d));
assertEquals("0", numberFormatter.format(pow(10, -6)));
assertEquals("0.33333", numberFormatter.format(1d / 3));
assertEquals("3.33333E-19", numberFormatter.format(pow(10, -18) / 3));
assertEquals("0", numberFormatter.format(pow(10, -18) / 3));
assertEquals("1234567890000000000", numberFormatter.format(123456789 * pow(10, 10)));
assertEquals("0", numberFormatter.format(pow(10, -16)));
assertEquals("0", numberFormatter.format(pow(10, -17)));
assertEquals("0", numberFormatter.format(pow(10, -18)));
assertEquals("0", numberFormatter.format(1.5 * pow(10, -18)));
assertEquals("0", numberFormatter.format(pow(10, -100)));
testSimpleFormat();
}
@ -98,10 +106,6 @@ public class NumberFormatterTest {
assertEquals("1E100", numberFormatter.format(pow(10, 100)));
assertEquals("0.01", numberFormatter.format(pow(10, -2)));
assertEquals("1E-17", numberFormatter.format(pow(10, -17)));
assertEquals("1E-18", numberFormatter.format(pow(10, -18)));
assertEquals("1.5E-18", numberFormatter.format(1.5 * pow(10, -18)));
assertEquals("1E-100", numberFormatter.format(pow(10, -100)));
assertEquals("5000000000000000000", numberFormatter.format(5000000000000000000d));
assertEquals("5000000000000000000", numberFormatter.format(5000000000000000001d));