SK-2972: Implement get, update, delete, detokenize, tokenize for flowvault

common/tests/utils/validations/test__validations.py

@@ -7,6 +7,7 @@
     validate_update_vault_config,
     validate_credentials,
     validate_log_level,
+    validate_non_empty_string_list,
 )
 
 VALID_VAULT_CONFIG = {
@@ -165,5 +166,31 @@ def test_uses_injected_messages(self):
         self.assertIn("FAKE", ctx.exception.message)
 
 
+class TestValidateNonEmptyStringList(unittest.TestCase):
+    def test_valid_list_passes(self):
+        validate_non_empty_string_list(None, ["a", "b"], "boom")  # should not raise
+
+    def test_non_list_raises_with_given_error(self):
+        with self.assertRaises(SkyflowError) as ctx:
+            validate_non_empty_string_list(None, "not-a-list", "boom")
+        self.assertEqual(ctx.exception.message, "boom")
+
+    def test_empty_list_raises(self):
+        with self.assertRaises(SkyflowError):
+            validate_non_empty_string_list(None, [], "boom")
+
+    def test_none_raises(self):
+        with self.assertRaises(SkyflowError):
+            validate_non_empty_string_list(None, None, "boom")
+
+    def test_non_string_entry_raises(self):
+        with self.assertRaises(SkyflowError):
+            validate_non_empty_string_list(None, ["a", 1], "boom")
+
+    def test_blank_string_entry_raises(self):
+        with self.assertRaises(SkyflowError):
+            validate_non_empty_string_list(None, ["a", "  "], "boom")
+
+
 if __name__ == "__main__":
     unittest.main()

common/utils/validations/__init__.py

@@ -4,6 +4,7 @@
     validate_credentials,
     validate_log_level,
     validate_keys,
+    validate_non_empty_string_list,
     validate_vault_config,
     validate_update_vault_config,
 )

common/utils/validations/_validations.py

@@ -173,6 +173,11 @@ def validate_keys(logger, config, config_keys, messages=None):
             raise SkyflowError(messages.Error.INVALID_KEY.value.format(key), invalid_input_error_code)
 
 
+def validate_non_empty_string_list(logger, value, error):
+    if not isinstance(value, list) or not value or not all(isinstance(item, str) and item.strip() for item in value):
+        raise SkyflowError(error, invalid_input_error_code)
+
+
 def validate_vault_config(logger, config, messages=None):
     messages = messages or SkyflowMessages
     log_info(messages.Info.VALIDATING_VAULT_CONFIG.value, logger)

flowvault/skyflow_flowvault/utils/_skyflow_messages.py

@@ -18,8 +18,8 @@ class SkyflowMessages:
     class Error(Enum):
         EMPTY_RECORDS_IN_INSERT = f"{error_prefix} Insert failed. Specify at least one record to insert."
         INVALID_RECORDS_TYPE_IN_INSERT = f"{error_prefix} Insert failed. 'records' must be a list of dicts."
-        INVALID_RECORD_DATA_IN_INSERT = f"{error_prefix} Insert failed. Each record's 'values' must be a non-empty dict."
-        INVALID_TABLE_NAME_IN_INSERT = f"{error_prefix} Insert failed. 'table' must be a non-empty string."
+        INVALID_RECORD_DATA_IN_INSERT = f"{error_prefix} Validation error. Each record's 'values' must be a non-empty dict."
+        INVALID_TABLE_NAME_IN_INSERT = f"{error_prefix} Validation error. 'table' must be a non-empty string."
         INVALID_UPSERT_TYPE_IN_INSERT = f"{error_prefix} Insert failed. 'upsert' must be a dict."
         INVALID_UPSERT_UNIQUE_COLUMNS_IN_INSERT = f"{error_prefix} Insert failed. Upsert's 'unique_columns' must be a non-empty list of strings."
         INVALID_UPSERT_UPDATE_TYPE_IN_INSERT = f"{error_prefix} Insert failed. Upsert's 'update_type' must be an UpsertType value."
@@ -44,14 +44,75 @@ class Error(Enum):
             "provided per-record -- InsertRequest's request-level 'upsert' cannot be used while "
             "'table' is set on individual records."
         )
-        EMPTY_KEY_IN_INSERT_DATA = f"{error_prefix} Insert failed. Each record's 'values' must not contain a null or empty key."
+        EMPTY_KEY_IN_INSERT_DATA = f"{error_prefix} Validation error. Each record's 'values' must not contain a null or empty key."
         EMPTY_VALUE_IN_INSERT_DATA = f"{error_prefix} Insert failed. Each record's 'values' must not contain a null or empty value."
 
+        MISSING_TABLE_NAME_IN_GET = f"{error_prefix} Get failed. Specify a table name."
+        MISSING_IDS_OR_UNIQUE_VALUES_IN_GET = f"{error_prefix} Get failed. Specify at least one of 'ids' or 'unique_values'."
+        INVALID_IDS_IN_GET = f"{error_prefix} Get failed. 'ids' must be a non-empty list of strings."
+
+        EMPTY_RECORDS_IN_UPDATE = f"{error_prefix} Update failed. Specify at least one record to update."
+        INVALID_RECORDS_TYPE_IN_UPDATE = f"{error_prefix} Update failed. 'records' must be a list of dicts."
+        MISSING_SKYFLOW_ID_IN_UPDATE = f"{error_prefix} Update failed. Each record must specify a non-empty 'skyflow_id'."
+        INVALID_UPDATE_TYPE_IN_UPDATE = f"{error_prefix} Update failed. 'update_type' must be an UpsertType value."
+        TABLE_NAME_IN_BOTH_PLACES_IN_UPDATE = (
+            f"{error_prefix} Update failed. 'table' cannot be set on UpdateRequest at the same "
+            "time as any record's 'table' -- specify a table name outside the records "
+            "(request-level, applying to all of them) or inside each record, but not both at once."
+        )
+        TABLE_NAME_MISSING_IN_UPDATE = (
+            f"{error_prefix} Update failed. 'table' is not set on UpdateRequest, so every record "
+            "must set its own 'table' -- either set 'table' once at the request level, or set it "
+            "individually on every record."
+        )
+
+        MISSING_TABLE_NAME_IN_DELETE = f"{error_prefix} Delete failed. Specify a table name."
+        MISSING_IDS_OR_UNIQUE_VALUES_IN_DELETE = f"{error_prefix} Delete failed. Specify at least one of 'ids' or 'unique_values'."
+        INVALID_IDS_IN_DELETE = f"{error_prefix} Delete failed. 'ids' must be a non-empty list of strings."
+
+        EMPTY_TOKENS_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. Specify at least one token to detokenize."
+        INVALID_TOKENS_TYPE_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. 'tokens' must be a non-empty list of strings."
+        INVALID_TOKEN_GROUP_REDACTIONS_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. 'token_group_redactions' must be a list of dicts with 'token_group_name' and 'redaction' keys."
+
+        EMPTY_VALUES_IN_TOKENIZE = f"{error_prefix} Tokenize failed. Specify at least one value to tokenize."
+        INVALID_VALUES_TYPE_IN_TOKENIZE = f"{error_prefix} Tokenize failed. 'values' must be a list of dicts."
+        MISSING_TOKEN_GROUP_NAMES_IN_TOKENIZE = f"{error_prefix} Tokenize failed. Each value must specify a non-empty 'token_group_names' list of strings."
+
     class Info(Enum):
         VALIDATE_INSERT_REQUEST = f"{INFO}: [{error_prefix}] Validating insert request."
         INSERT_TRIGGERED = f"{INFO}: [{error_prefix}] Insert method triggered."
         INSERT_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Insert request resolved."
         INSERT_SUCCESS = f"{INFO}: [{error_prefix}] Data inserted."
 
+        VALIDATE_GET_REQUEST = f"{INFO}: [{error_prefix}] Validating get request."
+        GET_TRIGGERED = f"{INFO}: [{error_prefix}] Get method triggered."
+        GET_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Get request resolved."
+        GET_SUCCESS = f"{INFO}: [{error_prefix}] Data fetched."
+
+        VALIDATE_UPDATE_REQUEST = f"{INFO}: [{error_prefix}] Validating update request."
+        UPDATE_TRIGGERED = f"{INFO}: [{error_prefix}] Update method triggered."
+        UPDATE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Update request resolved."
+        UPDATE_SUCCESS = f"{INFO}: [{error_prefix}] Data updated."
+
+        VALIDATE_DELETE_REQUEST = f"{INFO}: [{error_prefix}] Validating delete request."
+        DELETE_TRIGGERED = f"{INFO}: [{error_prefix}] Delete method triggered."
+        DELETE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Delete request resolved."
+        DELETE_SUCCESS = f"{INFO}: [{error_prefix}] Data deleted."
+
+        VALIDATE_DETOKENIZE_REQUEST = f"{INFO}: [{error_prefix}] Validating detokenize request."
+        DETOKENIZE_TRIGGERED = f"{INFO}: [{error_prefix}] Detokenize method triggered."
+        DETOKENIZE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Detokenize request resolved."
+        DETOKENIZE_SUCCESS = f"{INFO}: [{error_prefix}] Tokens detokenized."
+
+        VALIDATE_TOKENIZE_REQUEST = f"{INFO}: [{error_prefix}] Validating tokenize request."
+        TOKENIZE_TRIGGERED = f"{INFO}: [{error_prefix}] Tokenize method triggered."
+        TOKENIZE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Tokenize request resolved."
+        TOKENIZE_SUCCESS = f"{INFO}: [{error_prefix}] Values tokenized."
+
     class ErrorLogs(Enum):
         INSERT_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Insert call resulted in failure."
+        GET_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Get call resulted in failure."
+        UPDATE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Update call resulted in failure."
+        DELETE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Delete call resulted in failure."
+        DETOKENIZE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Detokenize call resulted in failure."
+        TOKENIZE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Tokenize call resulted in failure."

flowvault/skyflow_flowvault/utils/validations/__init__.py

@@ -1 +1,10 @@
-from ._validations import validate_vault_config, validate_update_vault_config, validate_insert_request
+from ._validations import (
+    validate_vault_config,
+    validate_update_vault_config,
+    validate_insert_request,
+    validate_get_request,
+    validate_update_request,
+    validate_delete_request,
+    validate_detokenize_request,
+    validate_tokenize_request,
+)

flowvault/skyflow_flowvault/utils/validations/_validations.py

@@ -3,6 +3,7 @@
 from common.utils.validations import (
     validate_keys,
     validate_credentials,
+    validate_non_empty_string_list,
     validate_vault_config,
     validate_update_vault_config,
 )
@@ -11,6 +12,7 @@
 
 VALID_INSERT_RECORD_KEYS = ["values", "table", "upsert"]
 VALID_UPSERT_KEYS = ["update_type", "unique_columns"]
+VALID_UPDATE_RECORD_KEYS = ["skyflow_id", "values", "tokens", "table"]
 
 invalid_input_error_code = CommonMessages.ErrorCodes.INVALID_INPUT.value
 
@@ -77,3 +79,88 @@ def validate_insert_request(logger, request):
     else:
         if request.upsert is not None:
             raise SkyflowError(SkyflowMessages.Error.REQUEST_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code)
+
+
+def validate_get_request(logger, request):
+    if not request.table:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_GET.value, invalid_input_error_code)
+
+    if not request.ids and not request.unique_values:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_GET.value, invalid_input_error_code)
+
+    if request.ids is not None:
+        validate_non_empty_string_list(logger, request.ids, SkyflowMessages.Error.INVALID_IDS_IN_GET.value)
+
+
+def validate_update_request(logger, request):
+    if not isinstance(request.records, list) or not all(isinstance(r, dict) for r in request.records):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_UPDATE.value, invalid_input_error_code)
+
+    if not request.records:
+        raise SkyflowError(SkyflowMessages.Error.EMPTY_RECORDS_IN_UPDATE.value, invalid_input_error_code)
+
+    if request.update_type is not None and not isinstance(request.update_type, UpsertType):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_UPDATE_TYPE_IN_UPDATE.value, invalid_input_error_code)
+
+    for record in request.records:
+        validate_keys(logger, record, VALID_UPDATE_RECORD_KEYS)
+        skyflow_id = record.get("skyflow_id")
+        if not isinstance(skyflow_id, str) or not skyflow_id.strip():
+            raise SkyflowError(SkyflowMessages.Error.MISSING_SKYFLOW_ID_IN_UPDATE.value, invalid_input_error_code)
+
+    table_at_request_level = request.table is not None
+
+    if table_at_request_level:
+        for record in request.records:
+            if record.get("table") is not None:
+                raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_IN_BOTH_PLACES_IN_UPDATE.value, invalid_input_error_code)
+    else:
+        for record in request.records:
+            if record.get("table") is None:
+                raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_MISSING_IN_UPDATE.value, invalid_input_error_code)
+
+
+def validate_delete_request(logger, request):
+    if not request.table:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_DELETE.value, invalid_input_error_code)
+
+    if not request.ids and not request.unique_values:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_DELETE.value, invalid_input_error_code)
+
+    if request.ids is not None:
+        validate_non_empty_string_list(logger, request.ids, SkyflowMessages.Error.INVALID_IDS_IN_DELETE.value)
+
+
+def validate_detokenize_request(logger, request):
+    if (
+        not isinstance(request.tokens, list) or not all(isinstance(t, str) and t.strip() for t in request.tokens)
+    ):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_TYPE_IN_DETOKENIZE.value, invalid_input_error_code)
+
+    if not request.tokens:
+        raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENS_IN_DETOKENIZE.value, invalid_input_error_code)
+
+    if request.token_group_redactions is not None:
+        valid = (
+            isinstance(request.token_group_redactions, list)
+            and all(
+                isinstance(entry, dict) and isinstance(entry.get("token_group_name"), str) and entry.get("token_group_name").strip()
+                for entry in request.token_group_redactions
+            )
+        )
+        if not valid:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_GROUP_REDACTIONS_IN_DETOKENIZE.value, invalid_input_error_code)
+
+
+def validate_tokenize_request(logger, request):
+    if not isinstance(request.values, list) or not all(isinstance(v, dict) for v in request.values):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_VALUES_TYPE_IN_TOKENIZE.value, invalid_input_error_code)
+
+    if not request.values:
+        raise SkyflowError(SkyflowMessages.Error.EMPTY_VALUES_IN_TOKENIZE.value, invalid_input_error_code)
+
+    for value in request.values:
+        token_group_names = value.get("token_group_names")
+        validate_non_empty_string_list(
+            logger, token_group_names, SkyflowMessages.Error.MISSING_TOKEN_GROUP_NAMES_IN_TOKENIZE.value
+        )

flowvault/skyflow_flowvault/vault/client/client.py

@@ -10,5 +10,5 @@ def resolve_vault_url(self, cluster_id, env, vault_id, logger=None):
     def initialize_api_client(self, vault_url, bearer_token):
         self._api_client = SkyflowAuth(base_url=vault_url)
 
-    def get_insert_api(self):
+    def get_flowservice_api(self):
         return self._api_client.flowservice