Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.apache.iotdb.itbase.exception.InconsistentDataException;

import org.awaitility.Awaitility;
import org.junit.After;
Expand Down Expand Up @@ -510,11 +511,17 @@ public void showSpaceQuotaTest1() throws SQLException {
}

private void validateResultSetEventually(Statement statement, String sql, String ans) {
// ClusterIT fans SHOW out to every DataNode and compares cells. Space-quota usage is updated
// asynchronously via heartbeat, so coordinators may briefly disagree (e.g. [0, 2, 2]) until CN
// converges. That mismatch surfaces as InconsistentDataException from getString(), which
// untilAsserted() does not retry by default — ignore only that type so genuine SQL errors
// still fail fast.
Awaitility.await()
.pollInSameThread()
.pollDelay(0, TimeUnit.MILLISECONDS)
.pollInterval(1, TimeUnit.SECONDS)
.atMost(30, TimeUnit.SECONDS)
.ignoreExceptionsMatching(e -> e instanceof InconsistentDataException)
.untilAsserted(
() -> {
try {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,252 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.it.quotas;

import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;

import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

@RunWith(IoTDBTestRunner.class)
@Category({LocalStandaloneIT.class, ClusterIT.class})
public class IoTDBUserResourceQuotaIT {

@Before
public void setUp() throws Exception {
EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setQuotaEnable(true);
EnvFactory.getEnv().initClusterEnvironment();
}

@After
public void tearDown() throws Exception {
EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setQuotaEnable(false);
EnvFactory.getEnv().cleanClusterEnvironment();
}

@Test
public void throttleQuotaMapsToReadCpuMax() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.execute("CREATE USER quota_user 'pass'");
statement.execute("GRANT READ_DATA ON root.** TO USER quota_user");
statement.execute("SET THROTTLE QUOTA cpu=1 ON quota_user");
statement.execute("CREATE DATABASE root.quota");
statement.execute("CREATE TIMESERIES root.quota.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN");
statement.execute("INSERT INTO root.quota.d1(time, s1) VALUES (1, 1)");
try (ResultSet rs = statement.executeQuery("SHOW THROTTLE QUOTA quota_user")) {
Assert.assertTrue(rs.next());
}
try (Connection userCon = EnvFactory.getEnv().getConnection("quota_user", "pass");
Statement userStmt = userCon.createStatement()) {
userStmt.executeQuery("SELECT s1 FROM root.quota.d1");
}
}
}

@Test
public void setAndShowUserQuota() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.execute("CREATE USER urq_user 'pass'");
statement.execute(
"SET USER QUOTA ON urq_user WITH read_cpu_min=1, read_cpu_max=4, write_memory_max=10M, write_disk_io_max=10485760, write_temp_disk_max=8589934592");
Map<String, String> maxByType = new HashMap<>();
Set<String> types = new HashSet<>();
try (ResultSet rs = statement.executeQuery("SHOW USER QUOTA urq_user")) {
ResultSetMetaData meta = rs.getMetaData();
int userIdx = columnIndex(meta, "User");
int nodeIdx = columnIndex(meta, "NodeID");
int rwIdx = columnIndex(meta, "Read/Write");
int typeIdx = columnIndex(meta, "QuotaType");
int minIdx = columnIndex(meta, "Min");
int maxIdx = columnIndex(meta, "Max");
int usedIdx = columnIndex(meta, "Used");
Assert.assertTrue(columnIndex(meta, "MinGap") > 0);
while (rs.next()) {
Assert.assertEquals("urq_user", rs.getString(userIdx));
String nodeId = rs.getString(nodeIdx);
Assert.assertNotNull(nodeId);
Assert.assertFalse("-".equals(nodeId));
Assert.assertTrue(Integer.parseInt(nodeId) > 0);
String key = rs.getString(rwIdx) + ":" + rs.getString(typeIdx);
maxByType.put(key, rs.getString(maxIdx));
types.add(rs.getString(typeIdx));
if ("read:cpu".equals(key)) {
Assert.assertEquals("1", rs.getString(minIdx));
Assert.assertEquals("4", rs.getString(maxIdx));
}
Assert.assertNotNull(rs.getString(usedIdx));
}
}
Assert.assertEquals("4", maxByType.get("read:cpu"));
Assert.assertTrue(maxByType.containsKey("write:memory"));
Assert.assertTrue(types.contains("temp_disk"));
Assert.assertTrue(types.contains("disk_io"));

// At least one alive DataNode id appears in SHOW (dedicated usage-report aggregation).
Set<Integer> nodeIds = new HashSet<>();
try (ResultSet rs = statement.executeQuery("SHOW USER QUOTA urq_user")) {
int nodeIdx = columnIndex(rs.getMetaData(), "NodeID");
while (rs.next()) {
nodeIds.add(Integer.parseInt(rs.getString(nodeIdx)));
}
}
Assert.assertFalse(nodeIds.isEmpty());
for (Integer id : nodeIds) {
Assert.assertTrue(id > 0);
}

statement.execute("DELETE USER QUOTA ON urq_user");
try (ResultSet rs = statement.executeQuery("SHOW USER QUOTA urq_user")) {
Assert.assertFalse(rs.next());
}
}
}

@Test
public void lowerMaxDoesNotBreakShowAndNewAcquirePath() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.execute("CREATE USER urq_lower 'pass'");
statement.execute("SET USER QUOTA ON urq_lower WITH read_cpu_max=8");
statement.execute("SET USER QUOTA ON urq_lower WITH read_cpu_max=2");
try (ResultSet rs = statement.executeQuery("SHOW USER QUOTA urq_lower")) {
boolean seen = false;
while (rs.next()) {
if ("cpu".equalsIgnoreCase(rs.getString(columnIndex(rs.getMetaData(), "QuotaType")))
&& "read"
.equalsIgnoreCase(rs.getString(columnIndex(rs.getMetaData(), "Read/Write")))) {
Assert.assertEquals("2", rs.getString(columnIndex(rs.getMetaData(), "Max")));
seen = true;
}
}
Assert.assertTrue(seen);
}
}
}

@Test
public void rejectRootAndPartialUpdateAndCrossThrottle() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
try {
statement.execute("SET USER QUOTA ON `root` WITH read_cpu_max=1");
Assert.fail("root SET should fail");
} catch (Exception e) {
// expected
}

statement.execute("CREATE USER urq_partial 'pass'");
statement.execute("SET USER QUOTA ON urq_partial WITH read_cpu_max=4, write_memory_max=10M");
statement.execute("SET USER QUOTA ON urq_partial WITH read_cpu_min=1");
Map<String, String> values = new HashMap<>();
try (ResultSet rs = statement.executeQuery("SHOW USER QUOTA urq_partial")) {
ResultSetMetaData meta = rs.getMetaData();
int rw = columnIndex(meta, "Read/Write");
int type = columnIndex(meta, "QuotaType");
int min = columnIndex(meta, "Min");
int max = columnIndex(meta, "Max");
while (rs.next()) {
values.put(
rs.getString(rw) + ":" + rs.getString(type),
rs.getString(min) + "/" + rs.getString(max));
}
}
Assert.assertEquals("1/4", values.get("read:cpu"));
Assert.assertTrue(values.containsKey("write:memory"));

statement.execute("CREATE USER urq_cross 'pass'");
statement.execute("SET THROTTLE QUOTA cpu=2 ON urq_cross");
statement.execute("SET USER QUOTA ON urq_cross WITH write_disk_io_max=1048576");
Assert.assertTrue(hasQuotaType(statement, "SHOW USER QUOTA urq_cross", "cpu"));
Assert.assertTrue(hasQuotaType(statement, "SHOW USER QUOTA urq_cross", "disk_io"));
Assert.assertTrue(hasAnyRow(statement, "SHOW THROTTLE QUOTA urq_cross"));
}
}

@Test
public void readCpuMaxEnforcementAndRelease() throws Exception {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
statement.execute("CREATE USER urq_cpu 'pass'");
statement.execute("GRANT READ_DATA ON root.** TO USER urq_cpu");
statement.execute("SET USER QUOTA ON urq_cpu WITH read_cpu_max=1");
statement.execute("CREATE DATABASE root.urq_cpu");
statement.execute("CREATE TIMESERIES root.urq_cpu.d1.s1 WITH DATATYPE=INT32,ENCODING=PLAIN");
statement.execute("INSERT INTO root.urq_cpu.d1(time,s1) VALUES(1,1)");

// Single query should succeed (and release).
try (Connection userCon = EnvFactory.getEnv().getConnection("urq_cpu", "pass");
Statement userStmt = userCon.createStatement()) {
try (ResultSet rs = userStmt.executeQuery("SELECT s1 FROM root.urq_cpu.d1")) {
Assert.assertTrue(rs.next());
}
try (ResultSet rs = userStmt.executeQuery("SELECT s1 FROM root.urq_cpu.d1")) {
Assert.assertTrue(rs.next());
}
}
}
}

private static boolean hasQuotaType(Statement statement, String sql, String quotaType)
throws Exception {
try (ResultSet rs = statement.executeQuery(sql)) {
int typeIdx = columnIndex(rs.getMetaData(), "QuotaType");
while (rs.next()) {
if (quotaType.equalsIgnoreCase(rs.getString(typeIdx))) {
return true;
}
}
}
return false;
}

private static boolean hasAnyRow(Statement statement, String sql) throws Exception {
try (ResultSet rs = statement.executeQuery(sql)) {
return rs.next();
}
}

private static int columnIndex(ResultSetMetaData meta, String name) throws Exception {
for (int i = 1; i <= meta.getColumnCount(); i++) {
if (name.equalsIgnoreCase(meta.getColumnLabel(i))
|| name.equalsIgnoreCase(meta.getColumnName(i))) {
return i;
}
}
throw new AssertionError("column not found: " + name);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ keyWords
| SUBSCRIPTION
| SUBSCRIPTIONS
| SUBSTRING
| SUMMARY
| SYSTEM
| TABLE
| TAG
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ ddlStatement
| callInference | loadModel | unloadModel
// Quota
| setSpaceQuota | showSpaceQuota | setThrottleQuota | showThrottleQuota
| setUserResourceQuota | showUserResourceQuota | deleteUserResourceQuota
// View
| createLogicalView | dropLogicalView | showLogicalView | renameLogicalView | alterLogicalView
// Table View
Expand Down Expand Up @@ -400,6 +401,18 @@ showThrottleQuota
: SHOW THROTTLE QUOTA (userName=identifier)?
;

setUserResourceQuota
: SET USER QUOTA ON userName=identifier WITH attributePair (COMMA attributePair)*
;

showUserResourceQuota
: SHOW USER QUOTA (userName=identifier)? (SUMMARY)? (ON DATANODE dataNodeId=INTEGER_LITERAL)?
;

deleteUserResourceQuota
: DELETE USER QUOTA ON userName=identifier
;

// Trigger =========================================================================================
// ---- Create Trigger
createTrigger
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -907,6 +907,10 @@ SUBSTRING
: S U B S T R I N G
;

SUMMARY
: S U M M A R Y
;

SYSTEM
: S Y S T E M
;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,10 @@ private ManagerMessages() {}
public static final String MESSAGE_DATABASE_LIMIT_THRESHOLD_45C23274 = "database_limit_threshold";
public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_SPACE_QUOTA_DATABASE_ARG_F6ED7586 = "Unexpected error happened while setting space quota on database: %s ";
public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_THROTTLE_QUOTA_USER_ARG_C111BE81 = "Unexpected error happened while setting throttle quota on user: %s ";
public static final String LOG_UNEXPECTED_ERROR_SETTING_USER_RESOURCE_QUOTA_91A4C2E8 =
"Unexpected error happened when setting user resource quota";
public static final String LOG_FAILED_TO_AGGREGATE_RUNNING_DATANODE_USAGE_FOR_SHOW_USER_QUOTA_00017902 =
"Failed to aggregate Running DataNode usage for SHOW USER QUOTA";
public static final String LOG_SCHEMA_TEMPLATE_NEED_TWO_FILES_1E57542A = "schema_template need two files";
public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_USE_ROLE_FILE_TYPE_ARG_1B548759 = "Got IOException when deserialize use&role file, type:{}";
public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_ROLELIST_1354F29E = "Got IOException when deserialize roleList";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,10 @@ private ManagerMessages() {}
public static final String MESSAGE_DATABASE_LIMIT_THRESHOLD_45C23274 = "database_limit_threshold";
public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_SPACE_QUOTA_DATABASE_ARG_F6ED7586 = "设置数据库 %s 的空间配额时发生意外错误 ";
public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_THROTTLE_QUOTA_USER_ARG_C111BE81 = "设置用户 %s 的限流配额时发生意外错误 ";
public static final String LOG_UNEXPECTED_ERROR_SETTING_USER_RESOURCE_QUOTA_91A4C2E8 =
"设置用户资源配额时发生意外错误";
public static final String LOG_FAILED_TO_AGGREGATE_RUNNING_DATANODE_USAGE_FOR_SHOW_USER_QUOTA_00017902 =
"汇总 Running DataNode 用量失败,无法完整展示 SHOW USER QUOTA";
public static final String LOG_SCHEMA_TEMPLATE_NEED_TWO_FILES_1E57542A = "schema_template 需要两个文件";
public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_USE_ROLE_FILE_TYPE_ARG_1B548759 = "反序列化 use&role 文件时发生 IOException,类型:{}";
public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_ROLELIST_1354F29E = "反序列化 roleList 时发生 IOException";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ public enum CnToDnAsyncRequestType {
// Quota
SET_SPACE_QUOTA,
SET_THROTTLE_QUOTA,
SET_USER_RESOURCE_QUOTA,

// Table
UPDATE_TABLE,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq;
import org.apache.iotdb.common.rpc.thrift.TSetTTLReq;
import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq;
import org.apache.iotdb.common.rpc.thrift.TSetUserResourceQuotaReq;
import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient;
import org.apache.iotdb.commons.client.request.AsyncRequestContext;
import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler;
Expand Down Expand Up @@ -410,6 +411,11 @@ protected void initActionMapBuilder() {
(req, client, handler) ->
client.setThrottleQuota(
(TSetThrottleQuotaReq) req, (DataNodeTSStatusRPCHandler) handler));
actionMapBuilder.put(
CnToDnAsyncRequestType.SET_USER_RESOURCE_QUOTA,
(req, client, handler) ->
client.setUserResourceQuota(
(TSetUserResourceQuotaReq) req, (DataNodeTSStatusRPCHandler) handler));
actionMapBuilder.put(
CnToDnAsyncRequestType.RESET_PEER_LIST,
(req, client, handler) ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,10 @@
import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2;
import org.apache.iotdb.confignode.consensus.request.write.procedure.DeleteProcedurePlan;
import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan;
import org.apache.iotdb.confignode.consensus.request.write.quota.DeleteUserResourceQuotaPlan;
import org.apache.iotdb.confignode.consensus.request.write.quota.SetSpaceQuotaPlan;
import org.apache.iotdb.confignode.consensus.request.write.quota.SetThrottleQuotaPlan;
import org.apache.iotdb.confignode.consensus.request.write.quota.SetUserResourceQuotaPlan;
import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan;
import org.apache.iotdb.confignode.consensus.request.write.region.PollRegionMaintainTaskPlan;
Expand Down Expand Up @@ -612,6 +614,12 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept
case setThrottleQuota:
plan = new SetThrottleQuotaPlan();
break;
case setUserResourceQuota:
plan = new SetUserResourceQuotaPlan();
break;
case deleteUserResourceQuota:
plan = new DeleteUserResourceQuotaPlan();
break;
case CreateExternalService:
plan = new CreateExternalServicePlan();
break;
Expand Down
Loading
Loading