[server] Add HLL sketch aggregation merge function

LICENSE-bin

@@ -914,6 +914,33 @@ This product bundles Apache Datasketches.
 Project URL: https://datasketches.apache.org/
 License: Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt
 
+Apache DataSketches Memory includes xxHash code under the following license:
+License: BSD-2-Clause
+| Copyright (C) 2012-2016, Yann Collet.
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions
+| are met:
+|
+| 1. Redistributions of source code must retain the above copyright
+| notice, this list of conditions and the following disclaimer.
+|
+| 2. Redistributions in binary form must reproduce the above copyright
+| notice, this list of conditions and the following disclaimer in the
+| documentation and/or other materials provided with the distribution.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+| HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 ------------------------------------------------------------------------------------
 
 This product bundles Scala.

fluss-common/src/main/java/org/apache/fluss/metadata/AggFunctionType.java

@@ -67,6 +67,9 @@ public enum AggFunctionType {
     RBM32,
     RBM64,
 
+    // Apache DataSketches aggregation
+    HLL_SKETCH,
+
     // Array aggregation
     LAST_N;
 
@@ -153,6 +156,7 @@ public DataTypeRoot[] getSupportedDataTypeRoots() {
                 return new DataTypeRoot[] {DataTypeRoot.BOOLEAN};
             case RBM32:
             case RBM64:
+            case HLL_SKETCH:
                 return new DataTypeRoot[] {DataTypeRoot.BYTES};
             case LAST_N:
             case COLLECT:

fluss-common/src/main/java/org/apache/fluss/metadata/AggFunctions.java

@@ -319,6 +319,22 @@ public static AggFunction RBM64() {
         return new AggFunction(AggFunctionType.RBM64, null);
     }
 
+    // ===================================================================================
+    // Apache DataSketches Aggregation Functions
+    // ===================================================================================
+
+    /**
+     * Creates a HLL_SKETCH aggregation function that merges serialized Apache DataSketches HLL
+     * sketches.
+     *
+     * <p>Supported data types: BYTES
+     *
+     * @return a HLL_SKETCH aggregation function
+     */
+    public static AggFunction HLL_SKETCH() {
+        return new AggFunction(AggFunctionType.HLL_SKETCH, null);
+    }
+
     // ===================================================================================
     // Array Aggregation Functions
     // ===================================================================================

fluss-common/src/test/java/org/apache/fluss/metadata/TableDescriptorTest.java

@@ -402,8 +402,19 @@ void testValidateAggFunctionWithDataType() {
                 .hasMessageContaining(
                         Arrays.deepToString(AggFunctionType.MAX.getSupportedDataTypeRoots()));
 
+        assertThatThrownBy(
+                        () ->
+                                AggFunctions.of(AggFunctionType.HLL_SKETCH, params)
+                                        .validateDataType(DataTypes.STRING()))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining("column must be part of")
+                .hasMessageContaining(
+                        Arrays.deepToString(
+                                AggFunctionType.HLL_SKETCH.getSupportedDataTypeRoots()));
+
         // valid case
         AggFunctions.of(AggFunctionType.LAST_VALUE, params).validateDataType(DataTypes.STRING());
         AggFunctions.of(AggFunctionType.LISTAGG, params).validateDataType(DataTypes.STRING());
+        AggFunctions.of(AggFunctionType.HLL_SKETCH, params).validateDataType(DataTypes.BYTES());
     }
 }

fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkAggFunctionParserTest.java

@@ -55,6 +55,18 @@ void testParseFunctionWithoutParameters() {
         assertThat(result.get().hasParameters()).isFalse();
     }
 
+    @Test
+    void testParseHllSketchFunction() {
+        Configuration options = new Configuration();
+        options.setString("fields.user_hll.agg", "hll_sketch");
+
+        Optional<AggFunction> result =
+                FlinkAggFunctionParser.parseAggFunction("user_hll", DataTypes.BYTES(), options);
+
+        assertThat(result).isPresent();
+        assertThat(result.get()).isEqualTo(AggFunctions.HLL_SKETCH());
+    }
+
     @Test
     void testParseFunctionWithParameters() {
         Configuration options = new Configuration();
@@ -131,6 +143,20 @@ void testParseInvalidFunctionName() {
                 .hasMessageContaining("invalid_function");
     }
 
+    @Test
+    void testParseHllSketchFunctionWithInvalidType() {
+        Configuration options = new Configuration();
+        options.setString("fields.user_hll.agg", "hll_sketch");
+
+        assertThatThrownBy(
+                        () ->
+                                FlinkAggFunctionParser.parseAggFunction(
+                                        "user_hll", DataTypes.STRING(), options))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining("Invalid aggregation function configuration")
+                .hasMessageContaining("hll_sketch");
+    }
+
     @Test
     void testParseEmptyFunctionName() {
         Configuration options = new Configuration();

fluss-server/pom.xml

@@ -78,6 +78,12 @@
             <version>${roaringbitmap.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.datasketches</groupId>
+            <artifactId>datasketches-java</artifactId>
+            <version>${datasketches-java.version}</version>
+        </dependency>
+
         <!--
         we will start zookeeper server when start a local cluster,
         need include this module for starting zookeeper requires it -->
@@ -147,6 +153,10 @@
                                 </includes>
                             </artifactSet>
                             <relocations>
+                                <relocation>
+                                    <pattern>org.apache.datasketches</pattern>
+                                    <shadedPattern>org.apache.fluss.shaded.org.apache.datasketches</shadedPattern>
+                                </relocation>
                                 <relocation>
                                     <pattern>org.apache.commons</pattern>
                                     <shadedPattern>org.apache.fluss.shaded.org.apache.commons</shadedPattern>

fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/aggregate/factory/FieldHllSketchAggFactory.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.kv.rowmerger.aggregate.factory;
+
+import org.apache.fluss.metadata.AggFunction;
+import org.apache.fluss.metadata.AggFunctionType;
+import org.apache.fluss.server.kv.rowmerger.aggregate.functions.FieldHllSketchAgg;
+import org.apache.fluss.types.DataType;
+import org.apache.fluss.types.DataTypeRoot;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+
+/** Factory for {@link FieldHllSketchAgg}. */
+public class FieldHllSketchAggFactory implements FieldAggregatorFactory {
+
+    @Override
+    public FieldHllSketchAgg create(DataType fieldType, AggFunction aggFunction) {
+        checkArgument(
+                fieldType.getTypeRoot() == DataTypeRoot.BYTES,
+                "Data type for hll_sketch column must be 'BytesType' but was '%s'.",
+                fieldType);
+        return new FieldHllSketchAgg(fieldType);
+    }
+
+    @Override
+    public String identifier() {
+        return AggFunctionType.HLL_SKETCH.toString();
+    }
+}

fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/aggregate/functions/FieldHllSketchAgg.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.kv.rowmerger.aggregate.functions;
+
+import org.apache.fluss.types.DataType;
+
+import org.apache.datasketches.hll.HllSketch;
+import org.apache.datasketches.hll.Union;
+
+/** HLL sketch aggregator for serialized Apache DataSketches HLL sketches. */
+public class FieldHllSketchAgg extends FieldAggregator {
+
+    private static final long serialVersionUID = 1L;
+
+    public FieldHllSketchAgg(DataType dataType) {
+        super(dataType);
+    }
+
+    @Override
+    public Object agg(Object accumulator, Object inputField) {
+        if (accumulator == null || inputField == null) {
+            return accumulator == null ? inputField : accumulator;
+        }
+
+        try {
+            HllSketch accumulatorSketch = HllSketch.heapify((byte[]) accumulator);
+            HllSketch inputSketch = HllSketch.heapify((byte[]) inputField);
+            int lgMaxK = Math.max(accumulatorSketch.getLgConfigK(), inputSketch.getLgConfigK());
+            Union union = new Union(lgMaxK);
+            union.update(accumulatorSketch);
+            union.update(inputSketch);
+            return union.getResult().toCompactByteArray();
+        } catch (RuntimeException e) {
+            throw new RuntimeException("Unable to deserialize or merge HLL sketch bytes.", e);
+        }
+    }
+}

fluss-server/src/main/resources/META-INF/NOTICE

@@ -13,6 +13,8 @@ This project bundles the following dependencies under the Apache Software Licens
 - commons-cli:commons-cli:1.5.0
 - org.apache.commons:commons-lang3:3.18.0
 - org.apache.commons:commons-math3:3.6.1
+- org.apache.datasketches:datasketches-java:6.2.0
+- org.apache.datasketches:datasketches-memory:3.0.2
 - org.roaringbitmap:RoaringBitmap:1.3.0
 - at.yawk.lz4:lz4-java:1.10.2
 - org.xerial.snappy:snappy-java:1.1.10.4