diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/aws/IceAwsClientFactory.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/aws/IceAwsClientFactory.java new file mode 100644 index 00000000..f8aa16a8 --- /dev/null +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/aws/IceAwsClientFactory.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog.internal.aws; + +import java.util.Map; +import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.HttpClientProperties; +import org.apache.iceberg.aws.s3.S3FileIOAwsClientFactory; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import software.amazon.awssdk.core.client.builder.SdkClientBuilder; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * S3FileIOAwsClientFactory to be set as {@link S3FileIOProperties#CLIENT_FACTORY} + * (s3.client-factory-impl) so that ice controls how S3 clients are built. + */ +public class IceAwsClientFactory implements S3FileIOAwsClientFactory { + + private AwsClientProperties awsClientProperties; + private S3FileIOProperties s3FileIOProperties; + private HttpClientProperties httpClientProperties; + private Map objectMetadata; + + public IceAwsClientFactory() { + this.awsClientProperties = new AwsClientProperties(); + this.s3FileIOProperties = new S3FileIOProperties(); + this.httpClientProperties = new HttpClientProperties(); + this.objectMetadata = Map.of(); + } + + @Override + public S3Client s3() { + return S3Client.builder() + .applyMutation(awsClientProperties::applyClientRegionConfiguration) + .applyMutation(httpClientProperties::applyHttpClientConfigurations) + .applyMutation(s3FileIOProperties::applyEndpointConfigurations) + .applyMutation(s3FileIOProperties::applyServiceConfigurations) + .applyMutation( + b -> s3FileIOProperties.applyCredentialConfigurations(awsClientProperties, b)) + .applyMutation(s3FileIOProperties::applySignerConfiguration) + .applyMutation(s3FileIOProperties::applyS3AccessGrantsConfigurations) + .applyMutation(s3FileIOProperties::applyUserAgentConfigurations) + .applyMutation(s3FileIOProperties::applyRetryConfigurations) + .applyMutation(this::applyObjectMetadataConfiguration) + .build(); + } + + @Override + public S3AsyncClient s3Async() { + if (s3FileIOProperties.isS3CRTEnabled()) { + return S3AsyncClient.crtBuilder() + .applyMutation(awsClientProperties::applyClientRegionConfiguration) + .applyMutation(awsClientProperties::applyClientCredentialConfigurations) + .applyMutation(s3FileIOProperties::applyEndpointConfigurations) + .applyMutation(s3FileIOProperties::applyS3CrtConfigurations) + .build(); + } + return S3AsyncClient.builder() + .applyMutation(awsClientProperties::applyClientRegionConfiguration) + .applyMutation(awsClientProperties::applyClientCredentialConfigurations) + .applyMutation(s3FileIOProperties::applyEndpointConfigurations) + .applyMutation(this::applyObjectMetadataConfiguration) + .build(); + } + + private void applyObjectMetadataConfiguration(SdkClientBuilder builder) { + if (objectMetadata.isEmpty()) { + return; + } + builder.overrideConfiguration( + c -> c.addExecutionInterceptor(new S3ObjectMetadataInterceptor(objectMetadata))); + } + + @Override + public void initialize(Map properties) { + this.awsClientProperties = new AwsClientProperties(properties); + this.s3FileIOProperties = new S3FileIOProperties(properties); + this.httpClientProperties = new HttpClientProperties(properties); + this.objectMetadata = S3ObjectMetadataInterceptor.metadataFromProperties(properties); + } +} diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/aws/S3ObjectMetadataInterceptor.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/aws/S3ObjectMetadataInterceptor.java new file mode 100644 index 00000000..6abcfdd7 --- /dev/null +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/aws/S3ObjectMetadataInterceptor.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog.internal.aws; + +import com.altinity.ice.internal.strings.Strings; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +/** + * Adds user-defined object metadata (sent as x-amz-meta-* headers) to every S3 request that creates + * an object. + */ +public final class S3ObjectMetadataInterceptor implements ExecutionInterceptor { + + public static final String METADATA_PREFIX = "s3.metadata."; + + private static final String HEADER_PREFIX = "x-amz-meta-"; + + private final Map metadata; + + public S3ObjectMetadataInterceptor(Map metadata) { + this.metadata = Map.copyOf(metadata); + } + + /** + * Extracts object metadata from catalog properties. Keys are stripped of METADATA_PREFIX (both + * x-amz-meta-foo and foo result in the x-amz-meta-foo header). + */ + public static Map metadataFromProperties(Map properties) { + Map m = new LinkedHashMap<>(); + for (Map.Entry e : properties.entrySet()) { + String k = e.getKey(); + if (!k.startsWith(METADATA_PREFIX)) { + continue; + } + k = Strings.removePrefix(k, METADATA_PREFIX); + if (k.toLowerCase().startsWith(HEADER_PREFIX)) { + k = k.substring(HEADER_PREFIX.length()); + } + if (k.isEmpty() || e.getValue() == null) { + continue; + } + m.put(k, e.getValue()); + } + return m; + } + + @Override + public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes attrs) { + SdkRequest request = context.request(); + if (request instanceof PutObjectRequest r) { + return r.toBuilder().metadata(merge(r.hasMetadata() ? r.metadata() : Map.of())).build(); + } + if (request instanceof CreateMultipartUploadRequest r) { + return r.toBuilder().metadata(merge(r.hasMetadata() ? r.metadata() : Map.of())).build(); + } + return request; + } + + private Map merge(Map requestMetadata) { + Map m = new HashMap<>(metadata); + m.putAll(requestMetadata); + return m; + } +} diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java index cb98cded..ca6bcb4c 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java @@ -13,6 +13,8 @@ import com.altinity.ice.internal.iceberg.io.SchemeFileIO; import com.altinity.ice.internal.strings.Strings; import com.altinity.ice.rest.catalog.internal.aws.CustomS3TablesCatalog; +import com.altinity.ice.rest.catalog.internal.aws.IceAwsClientFactory; +import com.altinity.ice.rest.catalog.internal.aws.S3ObjectMetadataInterceptor; import com.altinity.ice.rest.catalog.internal.etcd.EtcdCatalog; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; @@ -127,7 +129,26 @@ public record S3( String secretAccessKey, @JsonPropertyDescription( "AWS_REGION (see https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html#envvars-list)") - String region) {} + String region, + @JsonPropertyDescription( + "User-defined metadata to attach to every object created by the catalog, e.g. \"x-amz-meta-expiration-seconds: 1000\" (the x-amz-meta- prefix is optional). Empty by default") + Map metadata) { + + public S3( + String endpoint, + boolean pathStyleAccess, + String accessKeyID, + String secretAccessKey, + String region, + Map metadata) { + this.endpoint = endpoint; + this.pathStyleAccess = pathStyleAccess; + this.accessKeyID = accessKeyID; + this.secretAccessKey = secretAccessKey; + this.region = region; + this.metadata = Objects.requireNonNullElse(metadata, Map.of()); + } + } public record Token( @JsonPropertyDescription("Name") String name, @@ -249,6 +270,13 @@ public void putNotNullOrEmpty(String key, String value) { if (s3.pathStyleAccess) { m.putNotNullOrEmpty(S3FileIOProperties.PATH_STYLE_ACCESS, "true"); } + if (!s3.metadata.isEmpty()) { + for (Map.Entry e : s3.metadata.entrySet()) { + m.putNotNullOrEmpty( + S3ObjectMetadataInterceptor.METADATA_PREFIX + e.getKey(), e.getValue()); + } + m.put(S3FileIOProperties.CLIENT_FACTORY, IceAwsClientFactory.class.getName()); + } } if (localFileIOBaseDir != null) { diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java index 00e98d04..eaca1415 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java @@ -95,7 +95,7 @@ public void setUp() throws Exception { "jdbc:sqlite::memory:", // uri "s3://test-bucket/warehouse", // warehouse null, // localFileIOBaseDir - new Config.S3(minioEndpoint, true, "minioadmin", "minioadmin", "us-east-1"), // s3 + new Config.S3(minioEndpoint, true, "minioadmin", "minioadmin", "us-east-1", null), // s3 null, // bearerTokens new Config.AnonymousAccess( true, diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/internal/aws/S3ObjectMetadataInterceptorTest.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/internal/aws/S3ObjectMetadataInterceptorTest.java new file mode 100644 index 00000000..e46354c0 --- /dev/null +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/internal/aws/S3ObjectMetadataInterceptorTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog.internal.aws; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; +import org.junit.Test; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +public class S3ObjectMetadataInterceptorTest { + + @Test + public void metadataFromPropertiesIgnoresUnrelatedPropertiesAndHeaderPrefix() { + var m = + S3ObjectMetadataInterceptor.metadataFromProperties( + Map.of( + "s3.endpoint", "http://localhost:9000", + "s3.metadata.x-amz-meta-expiration-seconds", "1000", + "s3.metadata.owner", "ice", + "s3.metadata.", "ignored")); + assertThat(m).containsOnly(Map.entry("expiration-seconds", "1000"), Map.entry("owner", "ice")); + } + + @Test + public void putObjectAndCreateMultipartUploadGetMetadata() { + var i = new S3ObjectMetadataInterceptor(Map.of("owner", "ice")); + + var put = (PutObjectRequest) modify(i, PutObjectRequest.builder().bucket("b").key("k").build()); + assertThat(put.metadata()).containsExactly(Map.entry("owner", "ice")); + + var mpu = + (CreateMultipartUploadRequest) + modify(i, CreateMultipartUploadRequest.builder().bucket("b").key("k").build()); + assertThat(mpu.metadata()).containsExactly(Map.entry("owner", "ice")); + } + + @Test + public void requestMetadataWins() { + var i = new S3ObjectMetadataInterceptor(Map.of("owner", "ice", "env", "prod")); + var put = + (PutObjectRequest) + modify( + i, + PutObjectRequest.builder() + .bucket("b") + .key("k") + .metadata(Map.of("owner", "explicit")) + .build()); + assertThat(put.metadata()) + .containsOnly(Map.entry("owner", "explicit"), Map.entry("env", "prod")); + } + + @Test + public void otherRequestsAreLeftAlone() { + var i = new S3ObjectMetadataInterceptor(Map.of("owner", "ice")); + var get = GetObjectRequest.builder().bucket("b").key("k").build(); + assertThat(modify(i, get)).isSameAs(get); + } + + private static SdkRequest modify(S3ObjectMetadataInterceptor i, SdkRequest request) { + return i.modifyRequest(() -> request, new ExecutionAttributes()); + } +} diff --git a/ice-rest-catalog/src/test/pyiceberg/uv.lock b/ice-rest-catalog/src/test/pyiceberg/uv.lock index 37a544a0..a4fef937 100644 --- a/ice-rest-catalog/src/test/pyiceberg/uv.lock +++ b/ice-rest-catalog/src/test/pyiceberg/uv.lock @@ -13,16 +13,15 @@ wheels = [ [[package]] name = "anyio" -version = "4.9.0" +version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, - { name = "sniffio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] [[package]] @@ -539,15 +538,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "sniffio" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, -] - [[package]] name = "strictyaml" version = "1.7.3"