diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java index 6ff3de4d9495..a79be559dd86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.enums.PipeRateAverage; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.metric.PipeRemainingOperator; +import org.apache.iotdb.db.pipe.source.dataregion.IoTDBDataRegionSource; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; import org.apache.iotdb.metrics.core.IoTDBMetricManager; import org.apache.iotdb.metrics.core.type.IoTDBHistogram; @@ -42,6 +43,9 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator { + private final Set dataRegionSources = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + // Calculate from schema region extractors directly for it requires less computation private final Set schemaRegionSources = Collections.newSetFromMap(new ConcurrentHashMap<>()); @@ -105,6 +109,10 @@ public long getRemainingNonHeartbeatEvents() { tsfileEventCount.get() + rawTabletEventCount.get() + insertNodeEventCount.get() + + dataRegionSources.stream() + .map(IoTDBDataRegionSource::getHistoricalTsFileInsertionEventCount) + .reduce(Integer::sum) + .orElse(0) + schemaRegionSources.stream() .map(IoTDBSchemaRegionSource::getUnTransferredEventCount) .reduce(Long::sum) @@ -186,6 +194,10 @@ public double getRemainingTime() { //////////////////////////// Register & deregister (pipe integration) //////////////////////////// + void register(final IoTDBDataRegionSource source) { + dataRegionSources.add(source); + } + void register(final IoTDBSchemaRegionSource source) { schemaRegionSources.add(source); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 1599fe78ebae..b6efd64ed2c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -202,11 +202,13 @@ private void removeAutoGauge(final String pipeID) { public void register(final IoTDBDataRegionSource source) { // The metric is global thus the regionId is omitted final String pipeID = source.getPipeName() + "_" + source.getCreationTime(); - remainingEventAndTimeOperatorMap.computeIfAbsent( - pipeID, - k -> - new PipeDataNodeRemainingEventAndTimeOperator( - source.getPipeName(), source.getCreationTime())); + remainingEventAndTimeOperatorMap + .computeIfAbsent( + pipeID, + k -> + new PipeDataNodeRemainingEventAndTimeOperator( + source.getPipeName(), source.getCreationTime())) + .register(source); if (Objects.nonNull(metricService)) { createMetrics(pipeID); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperatorTest.java new file mode 100644 index 000000000000..1d4c927ac8b6 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperatorTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.metric.overview; + +import org.apache.iotdb.db.pipe.source.dataregion.IoTDBDataRegionSource; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PipeDataNodeRemainingEventAndTimeOperatorTest { + + @Test + public void testPendingHistoricalEventsAreIncludedInRemainingCount() { + final PipeDataNodeRemainingEventAndTimeOperator operator = + new PipeDataNodeRemainingEventAndTimeOperator("test_pipe", 1L); + final IoTDBDataRegionSource source = mock(IoTDBDataRegionSource.class); + when(source.getHistoricalTsFileInsertionEventCount()).thenReturn(2); + + operator.register(source); + + assertEquals(2, operator.getRemainingNonHeartbeatEvents()); + + operator.increaseTsFileEventCount(); + assertEquals(3, operator.getRemainingNonHeartbeatEvents()); + } +}