TnSb/android-calculatorpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simple format should be used for small numbers if rounding is enabled

Browse Source 
In order to avoid getting cos(90)=-13.3123123E-17
This commit is contained in:
serso
2016-04-12 16:22:27 +02:00
parent 19cb4d18d5
commit c1f013b707
2 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
jscl/src/main/java/org/solovyev/common/NumberFormatter.java
View File

@@ -1,12 +1,16 @@
package org.solovyev.common;
import java.math.BigDecimal;
import javax.annotation.Nonnull;
import midpcalc.Real;
import javax.annotation.Nonnull;
import java.math.BigDecimal;
import static java.lang.Math.pow;
import static midpcalc.Real.NumberFormat.*;
import static midpcalc.Real.NumberFormat.FSE_ENG;
import static midpcalc.Real.NumberFormat.FSE_FIX;
import static midpcalc.Real.NumberFormat.FSE_NONE;
import static midpcalc.Real.NumberFormat.FSE_SCI;
public class NumberFormatter {
@@ -88,12 +92,10 @@ public class NumberFormatter {
if (radix != 10) {
return true;
}
if (absValue < pow(10, -MAX_PRECISION)) {
// should never use simple format for small numbers
return false;
}
if (format == FSE_NONE) {
return true;
// simple format should be used only if rounding is on or if number is big enough
final boolean round = precision != NO_ROUNDING;
return round || absValue >= pow(10, -MAX_PRECISION);
}
if (pow(10, -simpleFormatMagnitude) <= absValue && absValue < pow(10, simpleFormatMagnitude)) {
return true;