diff --git a/CHANGELOG.md b/CHANGELOG.md index dbdc9fb..16e8c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## v1.12.2 7/21/26 +- Enable S3 client to upload files in binary format if needed + ## v1.12.1 6/18/26 - Remove Azure client encoding and decoding diff --git a/pyproject.toml b/pyproject.toml index f60f648..b92b5b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "nypl_py_utils" -version = "1.12.1" +version = "1.12.2" authors = [ { name="Aaron Friedman", email="aaronfriedman@nypl.org" }, ] diff --git a/src/nypl_py_utils/classes/s3_client.py b/src/nypl_py_utils/classes/s3_client.py index c1b7e9d..9dab536 100644 --- a/src/nypl_py_utils/classes/s3_client.py +++ b/src/nypl_py_utils/classes/s3_client.py @@ -16,13 +16,14 @@ class S3Client: """ def __init__(self, bucket, resource=None): - self.logger = create_log('s3_client') + self.logger = create_log("s3_client") self.bucket = bucket self.resource = resource try: self.s3_client = boto3.client( - 's3', region_name=os.environ.get('AWS_REGION', 'us-east-1')) + "s3", region_name=os.environ.get("AWS_REGION", "us-east-1") + ) except ClientError as e: error_msg = f'Could not create S3 client: {e}' self.logger.error(error_msg) @@ -34,7 +35,7 @@ def close(self): def fetch_cache(self): """Fetches a JSON file from S3 and returns the resulting dictionary""" self.logger.info( - f'Fetching {self.resource} from S3 bucket {self.bucket}') + f"Fetching {self.resource} from S3 bucket {self.bucket}") try: output_stream = BytesIO() self.s3_client.download_fileobj( @@ -42,23 +43,25 @@ def fetch_cache(self): return json.loads(output_stream.getvalue()) except ClientError as e: error_msg = ( - f'Error retrieving {self.resource} from S3 bucket ' - f'{self.bucket}: {e}') + f"Error retrieving {self.resource} from S3 bucket " + f"{self.bucket}: {e}" + ) self.logger.error(error_msg) raise S3ClientError(error_msg) from None def set_cache(self, state): """Writes a dictionary to JSON and uploads the resulting file to S3""" self.logger.info( - f'Setting {self.resource} in S3 bucket {self.bucket} to {state}') + f"Setting {self.resource} in S3 bucket {self.bucket} to {state}" + ) try: input_stream = BytesIO(json.dumps(state).encode()) self.s3_client.upload_fileobj( input_stream, self.bucket, self.resource) except ClientError as e: error_msg = ( - f'Error uploading {self.resource} to S3 bucket ' - f'{self.bucket}: {e}') + f"Error uploading {self.resource} to S3 bucket " + f"{self.bucket}: {e}") self.logger.error(error_msg) raise S3ClientError(error_msg) from None @@ -69,21 +72,23 @@ def upload_file(self, content, file_path): Parameters ---------- - content: str - The string that should be written to the file. Must be utf-8. + content: str | bytes + The data that should be written to the file file_path: str The full path of the file that should be written not including the bucket. Example: "subdirectory/example_file.csv" """ - self.logger.info( - f'Writing {file_path} in S3 bucket {self.bucket}') + self.logger.info(f"Writing {file_path} in S3 bucket {self.bucket}") try: - input_stream = BytesIO(content.encode()) - self.s3_client.upload_fileobj(input_stream, self.bucket, file_path) + if isinstance(content, str): + content = content.encode("utf-8") + self.s3_client.upload_fileobj( + BytesIO(content), self.bucket, file_path) except ClientError as e: error_msg = ( - f'Error uploading {file_path} to S3 bucket ' - f'{self.bucket}: {e}') + f"Error uploading {file_path} to S3 bucket " + f"{self.bucket}: {e}" + ) self.logger.error(error_msg) raise S3ClientError(error_msg) from None diff --git a/tests/test_s3_client.py b/tests/test_s3_client.py index cdc8293..d82541f 100644 --- a/tests/test_s3_client.py +++ b/tests/test_s3_client.py @@ -3,20 +3,20 @@ from nypl_py_utils.classes.s3_client import S3Client -_TEST_STATE = {'key1': 'val1', 'key2': 'val2', 'key3': 'val3'} +_TEST_STATE = {"key1": "val1", "key2": "val2", "key3": "val3"} class TestS3Client: @pytest.fixture def test_instance(self, mocker): - mocker.patch('boto3.client') - return S3Client('test_s3_bucket', 'test_s3_resource') + mocker.patch("boto3.client") + return S3Client("test_s3_bucket", "test_s3_resource") def test_fetch_cache(self, test_instance): def mock_download(bucket, resource, stream): - assert bucket == 'test_s3_bucket' - assert resource == 'test_s3_resource' + assert bucket == "test_s3_bucket" + assert resource == "test_s3_resource" stream.write(json.dumps(_TEST_STATE).encode()) test_instance.s3_client.download_fileobj.side_effect = mock_download @@ -26,12 +26,25 @@ def test_set_cache(self, test_instance): test_instance.set_cache(_TEST_STATE) arguments = test_instance.s3_client.upload_fileobj.call_args.args assert arguments[0].getvalue() == json.dumps(_TEST_STATE).encode() - assert arguments[1] == 'test_s3_bucket' - assert arguments[2] == 'test_s3_resource' + assert arguments[1] == "test_s3_bucket" + assert arguments[2] == "test_s3_resource" - def test_upload_file(self, test_instance): - test_instance.upload_file('test_content', 'test_filename.txt') + def test_upload_file_encoded(self, test_instance, mocker): + test_instance.upload_file("test_content", "test_filename.txt") arguments = test_instance.s3_client.upload_fileobj.call_args.args - assert arguments[0].getvalue() == b'test_content' - assert arguments[1] == 'test_s3_bucket' - assert arguments[2] == 'test_filename.txt' + expected_content = "test_content".encode("utf-8") + + # check that the content is encoded as utf-8 before being sent to S3 + assert arguments[0].getvalue() == expected_content + assert arguments[1] == "test_s3_bucket" + assert arguments[2] == "test_filename.txt" + + def test_upload_file_binary(self, test_instance): + binary_content = b"PAR1\x00\x01\xff\xfe" + test_instance.upload_file(binary_content, "test_filename.parquet") + arguments = test_instance.s3_client.upload_fileobj.call_args.args + + # check bytes aren't re-encoded; they should reach S3 as-is + assert arguments[0].getvalue() == binary_content + assert arguments[1] == "test_s3_bucket" + assert arguments[2] == "test_filename.parquet"