diff --git a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java index a995655c9..680482781 100644 --- a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java +++ b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java @@ -98,10 +98,12 @@ public short getK() { * Update this TDigest with the given value. * NaN and infinity are ignored. * @param value to update the TDigest with + * @throws ArithmeticException if the total weight would exceed Long.MAX_VALUE */ public void update(final double value) { if (!Double.isFinite(value)) { return; } - if (numBuffered_ == (centroidsCapacity_ * BUFFER_MULTIPLIER)) { compress(); } + Math.addExact(getTotalWeight(), 1); + if (numBuffered_ >= (centroidsCapacity_ * BUFFER_MULTIPLIER)) { compress(); } bufferValues_[numBuffered_] = value; numBuffered_++; minValue_ = Math.min(minValue_, value); @@ -111,9 +113,11 @@ public void update(final double value) { /** * Merge the given TDigest into this one * @param other TDigest to merge + * @throws ArithmeticException if the combined total weight would exceed Long.MAX_VALUE */ public void merge(final TDigestDouble other) { if (other.isEmpty()) { return; } + Math.addExact(getTotalWeight(), other.getTotalWeight()); final int num = numCentroids_ + numBuffered_ + other.numCentroids_ + other.numBuffered_; final double[] values = new double[num]; final long[] weights = new long[num]; @@ -256,6 +260,7 @@ public double getQuantile(final double rank) { } final double lastWeight = centroidWeights_[numCentroids_ - 1]; if ((lastWeight > 1) && ((centroidsWeight_ - weight) <= (lastWeight / 2.0))) { + if (lastWeight == 2) { return maxValue_; } return maxValue_ - (((centroidsWeight_ - weight - 1.0) / ((lastWeight / 2.0) - 1.0)) * (maxValue_ - centroidMeans_[numCentroids_ - 1])); } @@ -395,6 +400,7 @@ public static TDigestDouble heapify(final MemorySegment seg) { * @return an instance of TDigest */ public static TDigestDouble heapify(final MemorySegment seg, final boolean isFloat) { + checkSerializedSize(seg, 8); final PositionalSegment posSeg = PositionalSegment.wrap(seg); final byte preambleLongs = posSeg.getByte(); final byte serialVersion = posSeg.getByte(); @@ -411,6 +417,10 @@ public static TDigestDouble heapify(final MemorySegment seg, final boolean isFlo final byte flagsByte = posSeg.getByte(); final boolean isEmpty = (flagsByte & (1 << Flags.IS_EMPTY.ordinal())) > 0; final boolean isSingleValue = (flagsByte & (1 << Flags.IS_SINGLE_VALUE.ordinal())) > 0; + final int knownFlags = (1 << Flags.values().length) - 1; + if (((flagsByte & ~knownFlags) != 0) || (isEmpty && isSingleValue)) { + throw new SketchesArgumentException("Invalid TDigest flags: " + flagsByte); + } final byte expectedPreambleLongs = isEmpty || isSingleValue ? PREAMBLE_LONGS_EMPTY_OR_SINGLE : PREAMBLE_LONGS_MULTIPLE; if (preambleLongs != expectedPreambleLongs) { throw new SketchesArgumentException("Preamble longs mismatch: expected " + expectedPreambleLongs + ", actual " + preambleLongs); @@ -419,6 +429,7 @@ public static TDigestDouble heapify(final MemorySegment seg, final boolean isFlo if (isEmpty) { return new TDigestDouble(k); } final boolean reverseMerge = (flagsByte & (1 << Flags.REVERSE_MERGE.ordinal())) > 0; if (isSingleValue) { + checkSerializedSize(seg, 8 + (isFloat ? Float.BYTES : Double.BYTES)); final double value; if (isFloat) { value = posSeg.getFloat(); @@ -428,8 +439,12 @@ public static TDigestDouble heapify(final MemorySegment seg, final boolean isFlo checkDeserializedValue(value, "value"); return new TDigestDouble(reverseMerge, k, value, value, new double[] {value}, new long[] {1}, 1, null); } + final int valueBytes = isFloat ? Float.BYTES : Double.BYTES; + checkSerializedSize(seg, 16 + (2L * valueBytes)); final int numCentroids = posSeg.getInt(); final int numBuffered = posSeg.getInt(); + checkDeserializedCounts(k, numCentroids, numBuffered); + checkSerializedSize(seg, 16 + (2L * valueBytes) + (2L * valueBytes * numCentroids) + ((long) valueBytes * numBuffered)); final double min; final double max; if (isFloat) { @@ -439,22 +454,22 @@ public static TDigestDouble heapify(final MemorySegment seg, final boolean isFlo min = posSeg.getDouble(); max = posSeg.getDouble(); } - checkDeserializedValue(min, "min"); - checkDeserializedValue(max, "max"); + checkDeserializedExtrema(min, max); final double[] means = new double[numCentroids]; final long[] weights = new long[numCentroids]; long totalWeight = 0; for (int i = 0; i < numCentroids; i++) { means[i] = isFloat ? posSeg.getFloat() : posSeg.getDouble(); weights[i] = isFloat ? posSeg.getInt() : posSeg.getLong(); - checkDeserializedValue(means[i], "centroid mean"); + checkDeserializedRange(means[i], i == 0 ? min : means[i - 1], max, "centroid mean"); checkDeserializedWeight(weights[i]); - totalWeight += weights[i]; + totalWeight = addDeserializedWeight(totalWeight, weights[i]); } + addDeserializedWeight(totalWeight, numBuffered); final double[] buffered = new double[numBuffered]; for (int i = 0; i < numBuffered; i++) { buffered[i] = isFloat ? posSeg.getFloat() : posSeg.getDouble(); - checkDeserializedValue(buffered[i], "buffered value"); + checkDeserializedRange(buffered[i], min, max, "buffered value"); } return new TDigestDouble(reverseMerge, k, min, max, means, weights, totalWeight, buffered); } @@ -468,25 +483,27 @@ private static TDigestDouble heapifyCompat(final MemorySegment seg) { throw new SketchesArgumentException("unexpected compatibility type " + type); } if (type == COMPAT_DOUBLE) { // compatibility with asBytes() + checkSerializedSize(seg, 32); final double min = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; final double max = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; final short k = (short) seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; final int numCentroids = seg.get(JAVA_INT_UNALIGNED_BIG_ENDIAN, offset); offset += Integer.BYTES; - checkDeserializedValue(min, "min"); - checkDeserializedValue(max, "max"); + checkDeserializedCounts(k, numCentroids, 0); + checkSerializedSize(seg, offset + (16L * numCentroids)); + checkDeserializedExtrema(min, max); final double[] means = new double[numCentroids]; final long[] weights = new long[numCentroids]; long totalWeight = 0; for (int i = 0; i < numCentroids; i++) { - weights[i] = (long) seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; + weights[i] = readCompatibilityWeight(seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset)); offset += Double.BYTES; means[i] = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; - checkDeserializedValue(means[i], "centroid mean"); - checkDeserializedWeight(weights[i]); - totalWeight += weights[i]; + checkDeserializedRange(means[i], i == 0 ? min : means[i - 1], max, "centroid mean"); + totalWeight = addDeserializedWeight(totalWeight, weights[i]); } return new TDigestDouble(false, k, min, max, means, weights, totalWeight, null); } // COMPAT_FLOAT: compatibility with asSmallBytes(), reference implementation uses doubles for min and max + checkSerializedSize(seg, 30); final double min = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; final double max = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset);offset += Double.BYTES; final short k = (short) seg.get(JAVA_FLOAT_UNALIGNED_BIG_ENDIAN, offset); offset += Float.BYTES; @@ -494,17 +511,19 @@ private static TDigestDouble heapifyCompat(final MemorySegment seg) { // they can be derived from k in the constructor seg.get(JAVA_INT_UNALIGNED_BIG_ENDIAN, offset); offset += Integer.BYTES; final int numCentroids = seg.get(JAVA_SHORT_UNALIGNED_BIG_ENDIAN, offset); offset += Short.BYTES; - checkDeserializedValue(min, "min"); - checkDeserializedValue(max, "max"); + checkDeserializedCounts(k, numCentroids, 0); + checkSerializedSize(seg, offset + (8L * numCentroids)); + checkDeserializedExtrema(min, max); final double[] means = new double[numCentroids]; final long[] weights = new long[numCentroids]; long totalWeight = 0; for (int i = 0; i < numCentroids; i++) { - weights[i] = (long) seg.get(JAVA_FLOAT_UNALIGNED_BIG_ENDIAN, offset); offset += Float.BYTES; + weights[i] = readCompatibilityWeight(seg.get(JAVA_FLOAT_UNALIGNED_BIG_ENDIAN, offset)); offset += Float.BYTES; means[i] = seg.get(JAVA_FLOAT_UNALIGNED_BIG_ENDIAN, offset); offset += Float.BYTES; - checkDeserializedValue(means[i], "centroid mean"); - checkDeserializedWeight(weights[i]); - totalWeight += weights[i]; + // Float means can round just outside the double extrema in asSmallBytes(). + checkDeserializedRange(means[i], (float) (i == 0 ? min : means[i - 1]), (float) max, "centroid mean"); + means[i] = Math.max(min, Math.min(max, means[i])); + totalWeight = addDeserializedWeight(totalWeight, weights[i]); } return new TDigestDouble(false, k, min, max, means, weights, totalWeight, null); } @@ -521,6 +540,50 @@ private static void checkDeserializedWeight(final long weight) { } } + private static long readCompatibilityWeight(final double weight) { + // Validate before narrowing: Java truncates fractions and saturates overflowing casts. + if (!Double.isFinite(weight) || (weight < 1) || (weight >= 0x1p63) || (weight != Math.rint(weight))) { + throw new SketchesArgumentException("Deserialized centroid weight must be a positive long, actual: " + weight); + } + return (long) weight; + } + + private static long addDeserializedWeight(final long total, final long weight) { + if (weight > (Long.MAX_VALUE - total)) { + throw new SketchesArgumentException("Deserialized total weight exceeds Long.MAX_VALUE"); + } + return total + weight; + } + + private static void checkDeserializedExtrema(final double min, final double max) { + checkDeserializedValue(min, "min"); + checkDeserializedValue(max, "max"); + if (min > max) { throw new SketchesArgumentException("Deserialized min must not exceed max"); } + } + + private static void checkDeserializedRange(final double value, final double lower, final double upper, + final String description) { + checkDeserializedValue(value, description); + if ((value < lower) || (value > upper)) { + throw new SketchesArgumentException("Deserialized " + description + " must be within [" + lower + ", " + upper + "]"); + } + } + + private static void checkDeserializedCounts(final short k, final int numCentroids, final int numBuffered) { + checkK(k); + if ((numCentroids < 0) || (numBuffered < 0) || (numCentroids > (Integer.MAX_VALUE - numBuffered)) + || ((numCentroids + numBuffered) == 0)) { + throw new SketchesArgumentException("Invalid TDigest counts: centroids=" + numCentroids + ", buffered=" + numBuffered); + } + } + + private static void checkSerializedSize(final MemorySegment seg, final long requiredBytes) { + if (seg.byteSize() < requiredBytes) { + throw new SketchesArgumentException("Insufficient TDigest data: expected at least " + requiredBytes + + " bytes, actual: " + seg.byteSize()); + } + } + /** * Human-readable summary of this TDigest as a string * @return summary of this TDigest @@ -542,8 +605,8 @@ public String toString(final boolean printCentroids) { .append(" Compression: ").append(k_).append(LS) .append(" Centroids: ").append(numCentroids_).append(LS) .append(" Buffered: ").append(numBuffered_).append(LS) - .append(" Centroids Capacity: ").append(centroidsCapacity_).append(LS) - .append(" Buffer Capacity: ").append(centroidsCapacity_ * BUFFER_MULTIPLIER).append(LS) + .append(" Centroids Capacity: ").append(centroidMeans_.length).append(LS) + .append(" Buffer Capacity: ").append(bufferValues_.length).append(LS) .append("Centroids Weight: ").append(centroidsWeight_).append(LS) .append(" Total Weight: ").append(getTotalWeight()).append(LS) .append(" Reverse Merge: ").append(reverseMerge_).append(LS); @@ -574,12 +637,15 @@ private TDigestDouble(final boolean reverseMerge, final short k, final double mi k_ = k; minValue_ = min; maxValue_ = max; - if (k < 10) { throw new SketchesArgumentException("k must be at least 10"); } + checkK(k); final int fudge = k < 30 ? 30 : 10; - centroidsCapacity_ = (k_ * 2) + fudge; - centroidMeans_ = new double[centroidsCapacity_]; - centroidWeights_ = new long[centroidsCapacity_]; - bufferValues_ = new double[centroidsCapacity_ * BUFFER_MULTIPLIER]; + centroidsCapacity_ = (k * 2) + fudge; + // Compression thresholds are local sizing choices, not serialization limits. + final int centroidSlots = Math.max(centroidsCapacity_, means == null ? 0 : means.length); + final int bufferSlots = Math.max(centroidsCapacity_ * BUFFER_MULTIPLIER, buffer == null ? 0 : buffer.length); + centroidMeans_ = new double[centroidSlots]; + centroidWeights_ = new long[centroidSlots]; + bufferValues_ = new double[bufferSlots]; numCentroids_ = 0; numBuffered_ = 0; centroidsWeight_ = weight; @@ -594,6 +660,10 @@ private TDigestDouble(final boolean reverseMerge, final short k, final double mi } } + private static void checkK(final short k) { + if (k < 10) { throw new SketchesArgumentException("k must be at least 10"); } + } + // assumes that there is enough room in the input arrays to add centroids from this TDigest private void merge(final double[] values, final long[] weights, final long weight, int num) { System.arraycopy(centroidMeans_, 0, values, num, numCentroids_); @@ -670,25 +740,26 @@ static double z(final double compression, final double n) { } /* - * The weights are normalized before multiplying so that each term is bounded by the magnitude - * of its input, otherwise the products can overflow to infinity for values of large magnitude. + * Opposite-sign inputs need normalized weights to avoid overflowing their difference. + * Same-sign inputs need interpolation to avoid overflowing the sum of rounded products. */ private static double weightedAverage(final double x1, final double w1, final double x2, final double w2) { - final double weight = w1 + w2; - return (x1 * (w1 / weight)) + (x2 * (w2 / weight)); + final double ratio = w2 / (w1 + w2); + if (Math.copySign(1, x1) != Math.copySign(1, x2)) { + return (x1 * (1 - ratio)) + (x2 * ratio); + } + return Math.fma(x2 - x1, ratio, x1); } /* * Computes the mean of a centroid after merging in the given value with weight w, * where weight is the total weight of the centroid including w. - * The intermediate (value - mean) or its product with w can overflow to infinity - * even when both inputs are finite (e.g. means near opposite ends of the double range), - * which eventually turns the stored mean into NaN. In that case fall back to - * the overflow-safe weighted average, which stays finite. + * Normalize w before multiplying to avoid overflowing the product. If the difference + * itself overflows, the inputs have opposite signs and need the scaled weighted average. */ private static double mergedMean(final double mean, final double value, final long w, final long weight) { - final double newMean = mean + (((value - mean) * w) / weight); - if (Double.isFinite(newMean)) { return newMean; } + final double delta = value - mean; + if (Double.isFinite(delta)) { return Math.fma(delta, (double) w / weight, mean); } return weightedAverage(mean, weight - w, value, w); } } diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleSerializationTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleSerializationTest.java new file mode 100644 index 000000000..6d64d06a3 --- /dev/null +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleSerializationTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.tdigest; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; + +import org.apache.datasketches.common.Family; +import org.apache.datasketches.common.SketchesArgumentException; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class TDigestDoubleSerializationTest { + + @DataProvider(name = "formats") + public Object[][] formats() { + return new Object[][] {{false, false}, {false, true}, {true, false}, {true, true}}; + } + + @DataProvider(name = "precisions") + public Object[][] precisions() { + return new Object[][] {{false}, {true}}; + } + + @Test(dataProvider = "formats") + public void validCentroids(final boolean compat, final boolean isFloat) { + // Equal means are valid and must remain in their original order. + final byte[] bytes = serialize(compat, isFloat, 0, 10, + new double[] {0, 5, 5, 10}, new double[] {1, 2, 3, 1}, new double[0]); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), isFloat); + assertEquals(td.getTotalWeight(), 7); + assertEquals(td.getMinValue(), 0.0); + assertEquals(td.getMaxValue(), 10.0); + assertEquals(td.getQuantile(0.5), 5.0); + final TDigestDouble restored = TDigestDouble.heapify(MemorySegment.ofArray(td.toByteArray())); + assertEquals(restored.getTotalWeight(), 7); + assertEquals(restored.getQuantile(0.5), 5.0); + } + + @Test + public void quantileInterpolationApproachesNearerCentroid() { + final byte[] bytes = serialize(false, false, -1, 21, + new double[] {0, 10, 20}, new double[] {4, 4, 4}, new double[0]); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); + // The first two centroids lie at ranks 2/12 and 6/12. Moving across this + // interval must approach the right centroid, not move back toward the left. + assertEquals(td.getQuantile(2.0 / 12), 0.0, 1e-12); + assertEquals(td.getQuantile(3.0 / 12), 2.5, 1e-12); + assertEquals(td.getQuantile(4.0 / 12), 5.0, 1e-12); + assertEquals(td.getQuantile(5.0 / 12), 7.5, 1e-12); + assertEquals(td.getQuantile(6.0 / 12), 10.0, 1e-12); + } + + @Test + public void queryTailsAreSymmetric() { + final byte[] bytes = serialize(false, false, 0, 100, + new double[] {10, 50, 90}, new double[] {10, 10, 10}, new double[0]); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); + assertEquals(td.getQuantile(0.9), 95.0, 1e-12); + assertEquals(td.getQuantile(29.0 / 30), 100.0, 1e-12); + assertEquals(td.getQuantile(1.0 / 30), 0.0, 1e-12); + assertEquals(td.getQuantile(5.0 / 30), 10.0, 1e-12); + assertEquals(td.getRank(5), 0.1, 1e-12); + assertEquals(td.getRank(95), 0.9, 1e-12); + assertEquals(td.getCDF(new double[] {5, 95}), new double[] {0.1, 0.9, 1.0}, 1e-12); + assertEquals(td.getPMF(new double[] {5, 95}), new double[] {0.1, 0.8, 0.1}, 1e-12); + } + + @Test(dataProvider = "formats") + public void centroidsBeyondLocalCapacity(final boolean compat, final boolean isFloat) { + // k=100 normally allocates 210 centroid slots, but that is not a wire-format limit. + final double[] means = new double[211]; + final double[] weights = new double[means.length]; + for (int i = 0; i < means.length; i++) { + means[i] = i; + weights[i] = 1; + } + final byte[] bytes = serialize(compat, isFloat, 0, 210, means, weights, new double[0]); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), isFloat); + assertEquals(td.toByteArray(), serialize(false, false, 0, 210, means, weights, new double[0])); + assertEquals(td.getQuantile(0.5), 105.0); + final TDigestDouble merged = new TDigestDouble((short) 100); + merged.merge(td); + assertEquals(merged.getTotalWeight(), means.length); + td.update(211); + assertEquals(TDigestDouble.heapify(MemorySegment.ofArray(td.toByteArray())).getTotalWeight(), means.length + 1); + } + + @Test(dataProvider = "precisions") + public void bufferedValuesBeyondLocalCapacity(final boolean isFloat) { + // A producer may serialize more than this implementation's 840 buffered values. + // Exercise both buffer-only and mixed images, then cross several update boundaries. + final double[] buffered = new double[841]; + Arrays.fill(buffered, 10); + buffered[0] = 0; + for (final double[] means : new double[][] {new double[0], {0, 10}}) { + final double[] weights = new double[means.length]; + Arrays.fill(weights, 1); + final byte[] bytes = serialize(false, isFloat, 0, 10, means, weights, buffered); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), isFloat); + assertEquals(td.getTotalWeight(), buffered.length + means.length); + for (int i = 0; i < 10000; i++) { td.update(10); } + assertEquals(td.getTotalWeight(), 10000 + buffered.length + means.length); + assertEquals(td.getMinValue(), 0.0); + assertEquals(td.getMaxValue(), 10.0); + assertEquals(td.getQuantile(0.5), 10.0); + final TDigestDouble restored = TDigestDouble.heapify(MemorySegment.ofArray(td.toByteArray())); + assertEquals(restored.getTotalWeight(), td.getTotalWeight()); + } + } + + @Test(dataProvider = "formats") + public void invalidExtremaAndMeans(final boolean compat, final boolean isFloat) { + final double[][] extrema = {{2, 1}, {Double.NaN, 1}, {0, Double.POSITIVE_INFINITY}}; + for (final double[] range : extrema) { + assertInvalid(serialize(compat, isFloat, range[0], range[1], + new double[] {0, 1}, new double[] {1, 1}, new double[0]), isFloat); + } + final double[][] means = {{1, 0}, {-1, 1}, {0, 2}, {Double.NaN, 1}, {0, Double.POSITIVE_INFINITY}}; + for (final double[] values : means) { + assertInvalid(serialize(compat, isFloat, 0, 1, values, new double[] {1, 1}, new double[0]), isFloat); + } + assertInvalid(serialize(compat, isFloat, 0, 1, new double[0], new double[0], new double[0]), isFloat); + } + + @Test(dataProvider = "formats") + public void nonpositiveWeights(final boolean compat, final boolean isFloat) { + for (final double weight : new double[] {0, -1}) { + assertInvalid(serialize(compat, isFloat, 0, 1, + new double[] {0, 1}, new double[] {weight, 1}, new double[0]), isFloat); + } + } + + @Test(dataProvider = "precisions") + public void invalidCompatibilityWeights(final boolean isFloat) { + for (final double weight : new double[] {Double.NaN, Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, 0.5, 1.5, 0x1p63}) { + assertInvalid(serialize(true, isFloat, 0, 1, + new double[] {0, 1}, new double[] {weight, 1}, new double[0]), isFloat); + } + assertInvalid(serialize(true, isFloat, 0, 1, + new double[] {0, 1}, new double[] {0x1p62, 0x1p62}, new double[0]), isFloat); + } + + @Test + public void compatibilityFloatMeansRoundedOutsideExtrema() { + // asSmallBytes() retains double extrema but rounds means to floats in either direction. + for (final double value : new double[] {0.1, 0.3, -0.1, -0.3}) { + final byte[] bytes = serialize(true, true, value, value, + new double[] {value, value}, new double[] {2, 2}, new double[0]); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); + assertEquals(td.getTotalWeight(), 4); + assertEquals(td.getMinValue(), value); + assertEquals(td.getMaxValue(), value); + assertEquals(td.getQuantile(0.5), value); + final TDigestDouble restored = TDigestDouble.heapify(MemorySegment.ofArray(td.toByteArray())); + assertEquals(restored.getQuantile(0.5), value); + } + } + + @Test + public void overflowingNativeWeights() { + final byte[] bytes = serialize(false, false, 0, 1, + new double[] {0, 1}, new double[] {1, 1}, new double[0]); + MemorySegment.ofArray(bytes).set(ValueLayout.JAVA_LONG_UNALIGNED, 40, Long.MAX_VALUE); + assertInvalid(bytes, false); + + final byte[] buffered = serialize(false, false, 0, 1, + new double[] {0}, new double[] {1}, new double[] {1}); + MemorySegment.ofArray(buffered).set(ValueLayout.JAVA_LONG_UNALIGNED, 40, Long.MAX_VALUE); + assertInvalid(buffered, false); + + // The exact boundary is representable, including an uncompressed value. + MemorySegment.ofArray(buffered).set(ValueLayout.JAVA_LONG_UNALIGNED, 40, Long.MAX_VALUE - 1); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(buffered)); + assertEquals(td.getTotalWeight(), Long.MAX_VALUE); + final TDigestDouble restored = TDigestDouble.heapify(MemorySegment.ofArray(td.toByteArray())); + assertEquals(restored.getTotalWeight(), Long.MAX_VALUE); + } + + @Test + public void weightOverflowDoesNotChangeDigest() { + final byte[] bytes = serialize(false, false, 0, 0, + new double[] {0}, new double[] {1}, new double[0]); + MemorySegment.ofArray(bytes).set(ValueLayout.JAVA_LONG_UNALIGNED, 40, Long.MAX_VALUE - 1); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); + td.update(1); + assertEquals(td.getTotalWeight(), Long.MAX_VALUE); + final byte[] before = td.toByteArray(); + assertThrows(ArithmeticException.class, () -> td.update(2)); + final TDigestDouble other = new TDigestDouble(); + other.update(2); + assertThrows(ArithmeticException.class, () -> td.merge(other)); + assertThrows(ArithmeticException.class, () -> td.merge(td)); + assertEquals(td.toByteArray(), before); + assertEquals(other.getTotalWeight(), 1); + } + + @Test(dataProvider = "precisions") + public void bufferedValues(final boolean isFloat) { + // Buffered values may be unsorted, and may be the only stored values. + for (final double[] means : new double[][] {new double[0], {0, 10}}) { + final double[] weights = new double[means.length]; + Arrays.fill(weights, 1); + final byte[] bytes = serialize(false, isFloat, 0, 10, means, weights, new double[] {10, 0, 5}); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), isFloat); + assertEquals(td.getTotalWeight(), means.length + 3); + assertEquals(td.getQuantile(0.5), 5.0); + assertEquals(TDigestDouble.heapify(MemorySegment.ofArray(td.toByteArray())).getTotalWeight(), means.length + 3); + } + for (final double value : new double[] {-1, 11, Double.NaN, Double.POSITIVE_INFINITY}) { + assertInvalid(serialize(false, isFloat, 0, 10, + new double[] {0, 10}, new double[] {1, 1}, new double[] {value}), isFloat); + } + } + + @Test + public void invalidFlags() { + final byte[] bytes = new TDigestDouble().toByteArray(); + bytes[5] = 3; // mutually exclusive empty and single-value flags + assertInvalid(bytes, false); + bytes[5] = (byte) 0x81; // unknown flag on an otherwise valid empty image + assertInvalid(bytes, false); + } + + @Test(dataProvider = "formats") + public void invalidCountsAndTruncatedPayload(final boolean compat, final boolean isFloat) { + final byte[] bytes = serialize(compat, isFloat, 0, 1, + new double[] {0, 1}, new double[] {1, 1}, new double[0]); + for (int length = 0; length < bytes.length; length++) { + assertInvalid(Arrays.copyOf(bytes, length), isFloat); + } + final ByteBuffer buffer = ByteBuffer.wrap(bytes).order(compat ? ByteOrder.BIG_ENDIAN : ByteOrder.nativeOrder()); + if (compat && isFloat) { + buffer.putShort(28, (short) -1); + } else { + buffer.putInt(compat ? 28 : 8, Integer.MAX_VALUE); + } + assertInvalid(bytes, isFloat); + if (!(compat && isFloat)) { + buffer.putInt(compat ? 28 : 8, -1); + assertInvalid(bytes, isFloat); + } + if (!compat) { + buffer.putInt(8, 2); + buffer.putInt(12, -1); + assertInvalid(bytes, isFloat); + buffer.putInt(12, Integer.MAX_VALUE); + assertInvalid(bytes, isFloat); + } + } + + private static void assertInvalid(final byte[] bytes, final boolean isFloat) { + assertThrows(SketchesArgumentException.class, () -> TDigestDouble.heapify(MemorySegment.ofArray(bytes), isFloat)); + } + + private static byte[] serialize(final boolean compat, final boolean isFloat, final double min, final double max, + final double[] means, final double[] weights, final double[] buffered) { + final int valueBytes = isFloat ? Float.BYTES : Double.BYTES; + final int headerBytes = compat ? (isFloat ? 30 : 32) : 16 + (2 * valueBytes); + final ByteBuffer buffer = ByteBuffer.allocate(headerBytes + (means.length * 2 * valueBytes) + + (buffered.length * valueBytes)).order(compat ? ByteOrder.BIG_ENDIAN : ByteOrder.nativeOrder()); + if (compat) { + buffer.putInt(isFloat ? 2 : 1).putDouble(min).putDouble(max); + putValue(buffer, 100, isFloat); + if (isFloat) { + buffer.putInt(0).putShort((short) means.length); + } else { + buffer.putInt(means.length); + } + } else { + buffer.put((byte) 2).put((byte) 1).put((byte) Family.TDIGEST.getID()).putShort((short) 100); + buffer.put((byte) 0).putShort((short) 0).putInt(means.length).putInt(buffered.length); + putValue(buffer, min, isFloat); + putValue(buffer, max, isFloat); + } + for (int i = 0; i < means.length; i++) { + if (compat) { + putValue(buffer, weights[i], isFloat); + putValue(buffer, means[i], isFloat); + } else { + putValue(buffer, means[i], isFloat); + if (isFloat) { + buffer.putInt((int) weights[i]); + } else { + buffer.putLong((long) weights[i]); + } + } + } + for (final double value : buffered) { + putValue(buffer, value, isFloat); + } + return buffer.array(); + } + + private static void putValue(final ByteBuffer buffer, final double value, final boolean isFloat) { + if (isFloat) { + buffer.putFloat((float) value); + } else { + buffer.putDouble(value); + } + } +} diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java index e436025be..05dd73806 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java @@ -64,6 +64,21 @@ public void oneValue() { assertEquals(td.getQuantile(1), 1); } + @Test + public void repeatedValuesAtSingletonInterpolationBoundary() { + final TDigestDouble td = new TDigestDouble(); + for (int i = 0; i < 20; i++) { td.update(1); } + assertEquals(td.getQuantile(0.9), 1.0); + } + + @Test + public void emptySplitPointsDefineOneBin() { + final TDigestDouble td = new TDigestDouble(); + td.update(1); + assertEquals(td.getCDF(new double[0]), new double[] {1}); + assertEquals(td.getPMF(new double[0]), new double[] {1}); + } + @Test public void manyValues() { final TDigestDouble td = new TDigestDouble(); @@ -186,6 +201,67 @@ public void extremeValuesDoNotProduceNaN() { assertEquals(td2.getMaxValue(), Double.MAX_VALUE); } + @Test + public void sameSignExtremeQuantilesStayFinite() { + for (final double sign : new double[] {1, -1}) { + final TDigestDouble td = new TDigestDouble(); + td.update(sign * Math.nextDown(Double.MAX_VALUE)); + td.update(sign * Double.MAX_VALUE); + final MemorySegment seg = MemorySegment.ofArray(td.toByteArray()); + // The independently rounded normalized weights used to overflow the sum of + // two finite terms, even though the result must lie between these means. + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, 40, (1L << 52) - 1); + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, 56, 1L << 52); + final TDigestDouble restored = TDigestDouble.heapify(seg); + for (final double rank : new double[] {0.25, 0.5, 0.75}) { + final double quantile = restored.getQuantile(rank); + assertTrue(Double.isFinite(quantile), "non-finite quantile: " + quantile); + assertTrue(quantile >= restored.getMinValue()); + assertTrue(quantile <= restored.getMaxValue()); + } + } + } + + @Test + public void mergedExtremeCentroidStaysFinite() { + final double lower = Math.nextDown(Double.MAX_VALUE); + final TDigestDouble td = new TDigestDouble((short) 10); + td.update(lower); + td.update(lower); + td.update(Double.MAX_VALUE); + td.update(Double.MAX_VALUE); + final MemorySegment seg = MemorySegment.ofArray(td.toByteArray()); + // Merging the middle centroids overflows (value - mean) * weight. The old + // fallback also overflowed because its independently rounded ratios summed above one. + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, 40, 1L << 55); + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, 56, (1L << 52) + 1); + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, 72, 5L << 52); + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, 88, 1L << 55); + final TDigestDouble source = TDigestDouble.heapify(seg); + final TDigestDouble merged = new TDigestDouble((short) 10); + merged.merge(source); + // Round-tripping validates every stored mean, including centroids away from the queried rank. + final TDigestDouble restored = TDigestDouble.heapify(MemorySegment.ofArray(merged.toByteArray())); + assertEquals(restored.getTotalWeight(), source.getTotalWeight()); + assertEquals(restored.getMinValue(), lower); + assertEquals(restored.getMaxValue(), Double.MAX_VALUE); + assertTrue(Double.isFinite(restored.getQuantile(0.5))); + } + + @Test + public void quantileWithTwoSampleLastCentroid() { + final TDigestDouble td = new TDigestDouble(); + td.update(0); + td.update(50); + td.update(90); + final MemorySegment seg = MemorySegment.ofArray(td.toByteArray()); + seg.set(ValueLayout.JAVA_DOUBLE_UNALIGNED, 24, 100); + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, 72, 2); + final TDigestDouble restored = TDigestDouble.heapify(seg); + assertEquals(restored.getTotalWeight(), 4); + assertEquals(restored.getQuantile(0.75), 100.0); + } + // serialized layout: preamble 16 bytes, min 8 bytes, max 8 bytes, // then (mean 8 bytes, weight 8 bytes) per centroid private static byte[] serializeNonEmpty() {