Skip to content

Commit f151f40

Browse files
CodingBangbooCodingBangboo
authored
Use AccessConfigProvider.getAccessConfig in DefaultFileIOFactory (#2852)
1 parent 1e356ea commit f151f40

File tree

11 files changed

+52
-74
lines changed

11 files changed

+52
-74
lines changed

‎runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java‎

Lines changed: 14 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,9 @@
3131
import org.apache.iceberg.catalog.TableIdentifier;
3232
import org.apache.iceberg.io.FileIO;
3333
import org.apache.polaris.core.context.CallContext;
34-
import org.apache.polaris.core.context.RealmContext;
35-
import org.apache.polaris.core.entity.PolarisEntity;
36-
import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
3734
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
3835
import org.apache.polaris.core.storage.AccessConfig;
39-
import org.apache.polaris.core.storage.PolarisCredentialVendor;
4036
import org.apache.polaris.core.storage.PolarisStorageActions;
41-
import org.apache.polaris.core.storage.cache.StorageCredentialCache;
4237

4338
/**
4439
* A default FileIO factory implementation for creating Iceberg {@link FileIO} instances with
@@ -52,15 +47,11 @@
5247
@Identifier("default")
5348
public class DefaultFileIOFactory implements FileIOFactory {
5449

55-
private final StorageCredentialCache storageCredentialCache;
56-
private final MetaStoreManagerFactory metaStoreManagerFactory;
50+
private final AccessConfigProvider accessConfigProvider;
5751

5852
@Inject
59-
public DefaultFileIOFactory(
60-
StorageCredentialCache storageCredentialCache,
61-
MetaStoreManagerFactory metaStoreManagerFactory) {
62-
this.storageCredentialCache = storageCredentialCache;
63-
this.metaStoreManagerFactory = metaStoreManagerFactory;
53+
public DefaultFileIOFactory(AccessConfigProvider accessConfigProvider) {
54+
this.accessConfigProvider = accessConfigProvider;
6455
}
6556

6657
@Override
@@ -72,36 +63,25 @@ public FileIO loadFileIO(
7263
@Nonnull Set<String> tableLocations,
7364
@Nonnull Set<PolarisStorageActions> storageActions,
7465
@Nonnull PolarisResolvedPathWrapper resolvedEntityPath) {
75-
RealmContext realmContext = callContext.getRealmContext();
76-
PolarisCredentialVendor credentialVendor =
77-
metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext);
7866

7967
// Get subcoped creds
8068
properties = new HashMap<>(properties);
81-
Optional<PolarisEntity> storageInfoEntity =
82-
FileIOUtil.findStorageInfoFromHierarchy(resolvedEntityPath);
83-
Optional<AccessConfig> accessConfig =
84-
storageInfoEntity.map(
85-
storageInfo ->
86-
FileIOUtil.refreshAccessConfig(
87-
callContext,
88-
storageCredentialCache,
89-
credentialVendor,
90-
identifier,
91-
tableLocations,
92-
storageActions,
93-
storageInfo,
94-
Optional.empty()));
69+
AccessConfig accessConfig =
70+
accessConfigProvider.getAccessConfig(
71+
callContext,
72+
identifier,
73+
tableLocations,
74+
storageActions,
75+
Optional.empty(),
76+
resolvedEntityPath);
9577

9678
// Update the FileIO with the subscoped credentials
9779
// Update with properties in case there are table-level overrides the credentials should
9880
// always override table-level properties, since storage configuration will be found at
9981
// whatever entity defines it
100-
if (accessConfig.isPresent()) {
101-
properties.putAll(accessConfig.get().credentials());
102-
properties.putAll(accessConfig.get().extraProperties());
103-
properties.putAll(accessConfig.get().internalProperties());
104-
}
82+
properties.putAll(accessConfig.credentials());
83+
properties.putAll(accessConfig.extraProperties());
84+
properties.putAll(accessConfig.internalProperties());
10585

10686
return loadFileIOInternal(ioImplClassName, properties);
10787
}

‎runtime/service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,8 @@
2727
import org.apache.iceberg.catalog.TableIdentifier;
2828
import org.apache.iceberg.io.FileIO;
2929
import org.apache.polaris.core.context.CallContext;
30-
import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
3130
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
3231
import org.apache.polaris.core.storage.PolarisStorageActions;
33-
import org.apache.polaris.core.storage.cache.StorageCredentialCache;
3432

3533
/** A {@link FileIOFactory} that translates WASB paths to ABFS ones */
3634
@ApplicationScoped
@@ -40,11 +38,8 @@ public class WasbTranslatingFileIOFactory implements FileIOFactory {
4038
private final FileIOFactory defaultFileIOFactory;
4139

4240
@Inject
43-
public WasbTranslatingFileIOFactory(
44-
StorageCredentialCache storageCredentialCache,
45-
MetaStoreManagerFactory metaStoreManagerFactory) {
46-
defaultFileIOFactory =
47-
new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory);
41+
public WasbTranslatingFileIOFactory(AccessConfigProvider accessConfigProvider) {
42+
defaultFileIOFactory = new DefaultFileIOFactory(accessConfigProvider);
4843
}
4944

5045
@Override

‎runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisS3InteroperabilityTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ private static String makeTableLocation(
7373

7474
public PolarisS3InteroperabilityTest() {
7575
TestServices.FileIOFactorySupplier fileIOFactorySupplier =
76-
(storageCredentialCache, metaStoreManagerFactory) ->
76+
(accessConfigProvider) ->
7777
(FileIOFactory)
7878
(callContext,
7979
ioImplClassName,

‎runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import org.apache.polaris.service.admin.PolarisAdminService;
6868
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
6969
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
70+
import org.apache.polaris.service.catalog.io.AccessConfigProvider;
7071
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
7172
import org.apache.polaris.service.catalog.io.FileIOFactory;
7273
import org.apache.polaris.service.config.ReservedProperties;
@@ -120,6 +121,7 @@ public abstract class AbstractPolarisGenericTableCatalogTest {
120121
private PolarisPrincipal authenticatedRoot;
121122
private PolarisEntity catalogEntity;
122123
private SecurityContext securityContext;
124+
private AccessConfigProvider accessConfigProvider;
123125

124126
protected static final Schema SCHEMA =
125127
new Schema(
@@ -156,6 +158,8 @@ public void before(TestInfo testInfo) {
156158
metaStoreManagerFactory.getOrCreateSession(realmContext),
157159
configurationStore);
158160
realmConfig = polarisContext.getRealmConfig();
161+
accessConfigProvider =
162+
new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory);
159163

160164
PrincipalEntity rootPrincipal =
161165
metaStoreManager.findRootPrincipal(polarisContext).orElseThrow();
@@ -211,7 +215,7 @@ public void before(TestInfo testInfo) {
211215
new PolarisPassthroughResolutionView(
212216
resolutionManifestFactory, securityContext, CATALOG_NAME);
213217
TaskExecutor taskExecutor = Mockito.mock();
214-
this.fileIOFactory = new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory);
218+
this.fileIOFactory = new DefaultFileIOFactory(accessConfigProvider);
215219

216220
StsClient stsClient = Mockito.mock(StsClient.class);
217221
when(stsClient.assumeRole(isA(AssumeRoleRequest.class)))

‎runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@
135135
import org.apache.polaris.service.admin.PolarisAdminService;
136136
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
137137
import org.apache.polaris.service.catalog.Profiles;
138+
import org.apache.polaris.service.catalog.io.AccessConfigProvider;
138139
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
139140
import org.apache.polaris.service.catalog.io.ExceptionMappingFileIO;
140141
import org.apache.polaris.service.catalog.io.FileIOFactory;
@@ -249,6 +250,7 @@ public Map<String, String> getConfigOverrides() {
249250
private SecurityContext securityContext;
250251
private TestPolarisEventListener testPolarisEventListener;
251252
private ReservedProperties reservedProperties;
253+
private AccessConfigProvider accessConfigProvider;
252254

253255
@BeforeAll
254256
public static void setUpMocks() {
@@ -286,7 +288,8 @@ public void before(TestInfo testInfo) {
286288
metaStoreManagerFactory.getOrCreateSession(realmContext),
287289
configurationStore);
288290
realmConfig = polarisContext.getRealmConfig();
289-
291+
accessConfigProvider =
292+
new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory);
290293
EntityCache entityCache = createEntityCache(diagServices, realmConfig, metaStoreManager);
291294
resolverFactory =
292295
(securityContext, referenceCatalogName) ->
@@ -352,7 +355,7 @@ public void before(TestInfo testInfo) {
352355
.build()
353356
.asCatalog(serviceIdentityProvider)));
354357

355-
this.fileIOFactory = new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory);
358+
this.fileIOFactory = new DefaultFileIOFactory(accessConfigProvider);
356359

357360
StsClient stsClient = Mockito.mock(StsClient.class);
358361
when(stsClient.assumeRole(isA(AssumeRoleRequest.class)))
@@ -996,8 +999,7 @@ public void testValidateNotificationFailToCreateFileIO() {
996999
// filename.
9971000
final String tableLocation = "s3://externally-owned-bucket/validate_table/";
9981001
final String tableMetadataLocation = tableLocation + "metadata/";
999-
FileIOFactory fileIOFactory =
1000-
spy(new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory));
1002+
FileIOFactory fileIOFactory = spy(new DefaultFileIOFactory(accessConfigProvider));
10011003
IcebergCatalog catalog = newIcebergCatalog(catalog().name(), metaStoreManager, fileIOFactory);
10021004
catalog.initialize(
10031005
CATALOG_NAME,
@@ -1914,8 +1916,7 @@ public void testDropTableWithPurge() {
19141916
.containsEntry(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), SECRET_ACCESS_KEY)
19151917
.containsEntry(StorageAccessProperty.AWS_TOKEN.getPropertyName(), SESSION_TOKEN);
19161918
FileIO fileIO =
1917-
new TaskFileIOSupplier(
1918-
new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory))
1919+
new TaskFileIOSupplier(new DefaultFileIOFactory(accessConfigProvider))
19191920
.apply(taskEntity, TABLE, polarisContext);
19201921
Assertions.assertThat(fileIO).isNotNull().isInstanceOf(ExceptionMappingFileIO.class);
19211922
Assertions.assertThat(((ExceptionMappingFileIO) fileIO).getInnerIo())
@@ -2041,8 +2042,7 @@ static Stream<Arguments> testRetriableException() {
20412042

20422043
@Test
20432044
public void testFileIOWrapper() {
2044-
MeasuredFileIOFactory measured =
2045-
new MeasuredFileIOFactory(storageCredentialCache, metaStoreManagerFactory);
2045+
MeasuredFileIOFactory measured = new MeasuredFileIOFactory(accessConfigProvider);
20462046
IcebergCatalog catalog = newIcebergCatalog(CATALOG_NAME, metaStoreManager, measured);
20472047
catalog.initialize(
20482048
CATALOG_NAME,

‎runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.apache.polaris.service.admin.PolarisAdminService;
6161
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
6262
import org.apache.polaris.service.catalog.Profiles;
63+
import org.apache.polaris.service.catalog.io.AccessConfigProvider;
6364
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
6465
import org.apache.polaris.service.catalog.io.FileIOFactory;
6566
import org.apache.polaris.service.config.ReservedProperties;
@@ -123,6 +124,7 @@ public Map<String, String> getConfigOverrides() {
123124
private UserSecretsManager userSecretsManager;
124125
private PolarisCallContext polarisContext;
125126
private RealmConfig realmConfig;
127+
private AccessConfigProvider accessConfigProvider;
126128

127129
private TestPolarisEventListener testPolarisEventListener;
128130

@@ -163,7 +165,8 @@ public void before(TestInfo testInfo) {
163165
metaStoreManagerFactory.getOrCreateSession(realmContext),
164166
configurationStore);
165167
realmConfig = polarisContext.getRealmConfig();
166-
168+
accessConfigProvider =
169+
new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory);
167170
PrincipalEntity rootPrincipal =
168171
metaStoreManager.findRootPrincipal(polarisContext).orElseThrow();
169172
PolarisPrincipal authenticatedRoot = PolarisPrincipal.of(rootPrincipal, Set.of());
@@ -207,8 +210,7 @@ public void before(TestInfo testInfo) {
207210
PolarisPassthroughResolutionView passthroughView =
208211
new PolarisPassthroughResolutionView(
209212
resolutionManifestFactory, securityContext, CATALOG_NAME);
210-
FileIOFactory fileIOFactory =
211-
new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory);
213+
FileIOFactory fileIOFactory = new DefaultFileIOFactory(accessConfigProvider);
212214

213215
testPolarisEventListener = (TestPolarisEventListener) polarisEventListener;
214216
testPolarisEventListener.clear();

‎runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1899,7 +1899,7 @@ public void testSendNotificationSufficientPrivileges() {
18991899
resolverFactory,
19001900
managerFactory,
19011901
Mockito.mock(),
1902-
new DefaultFileIOFactory(storageCredentialCache, managerFactory),
1902+
new DefaultFileIOFactory(accessConfigProvider),
19031903
polarisEventListener) {
19041904
@Override
19051905
public Catalog createCallContextCatalog(

‎runtime/service/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,9 @@ public void before(TestInfo testInfo) {
102102

103103
// Spy FileIOFactory and check if the credentials are passed to the FileIO
104104
TestServices.FileIOFactorySupplier fileIOFactorySupplier =
105-
(storageCredentialCache, metaStoreManagerFactory) ->
105+
(accessConfigProvider) ->
106106
Mockito.spy(
107-
new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory) {
107+
new DefaultFileIOFactory(accessConfigProvider) {
108108
@Override
109109
FileIO loadFileIOInternal(
110110
@Nonnull String ioImplClassName, @Nonnull Map<String, String> properties) {

‎runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import org.apache.polaris.service.admin.PolarisAdminService;
8080
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
8181
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
82+
import org.apache.polaris.service.catalog.io.AccessConfigProvider;
8283
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
8384
import org.apache.polaris.service.catalog.io.FileIOFactory;
8485
import org.apache.polaris.service.config.ReservedProperties;
@@ -146,6 +147,7 @@ public abstract class AbstractPolicyCatalogTest {
146147
private PolarisPrincipal authenticatedRoot;
147148
private PolarisEntity catalogEntity;
148149
private SecurityContext securityContext;
150+
private AccessConfigProvider accessConfigProvider;
149151

150152
@BeforeAll
151153
public static void setUpMocks() {
@@ -177,6 +179,8 @@ public void before(TestInfo testInfo) {
177179
metaStoreManagerFactory.getOrCreateSession(realmContext),
178180
configurationStore);
179181
realmConfig = polarisContext.getRealmConfig();
182+
accessConfigProvider =
183+
new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory);
180184

181185
PrincipalEntity rootPrincipal =
182186
metaStoreManager.findRootPrincipal(polarisContext).orElseThrow();
@@ -230,7 +234,7 @@ public void before(TestInfo testInfo) {
230234
new PolarisPassthroughResolutionView(
231235
resolutionManifestFactory, securityContext, CATALOG_NAME);
232236
TaskExecutor taskExecutor = Mockito.mock();
233-
this.fileIOFactory = new DefaultFileIOFactory(storageCredentialCache, metaStoreManagerFactory);
237+
this.fileIOFactory = new DefaultFileIOFactory(accessConfigProvider);
234238

235239
StsClient stsClient = Mockito.mock(StsClient.class);
236240
when(stsClient.assumeRole(isA(AssumeRoleRequest.class)))

‎runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import java.util.Map;
3535
import java.util.Optional;
3636
import java.util.Set;
37-
import java.util.function.BiFunction;
37+
import java.util.function.Function;
3838
import org.apache.polaris.core.PolarisCallContext;
3939
import org.apache.polaris.core.PolarisDefaultDiagServiceImpl;
4040
import org.apache.polaris.core.PolarisDiagnostics;
@@ -116,8 +116,7 @@ public record TestServices(
116116
private static final String GCP_ACCESS_TOKEN = "abc";
117117

118118
@FunctionalInterface
119-
public interface FileIOFactorySupplier
120-
extends BiFunction<StorageCredentialCache, MetaStoreManagerFactory, FileIOFactory> {}
119+
public interface FileIOFactorySupplier extends Function<AccessConfigProvider, FileIOFactory> {}
121120

122121
private static class MockedConfigurationStore implements PolarisConfigurationStore {
123122
private final Map<String, Object> defaults;
@@ -144,7 +143,8 @@ public static class Builder {
144143
private RealmContext realmContext = TEST_REALM;
145144
private Map<String, Object> config = Map.of();
146145
private StsClient stsClient;
147-
private FileIOFactorySupplier fileIOFactorySupplier = MeasuredFileIOFactory::new;
146+
private FileIOFactorySupplier fileIOFactorySupplier =
147+
metaStoreManagerFactory1 -> new MeasuredFileIOFactory(metaStoreManagerFactory1);
148148

149149
private Builder() {
150150
stsClient = Mockito.mock(StsClient.class, RETURNS_DEEP_STUBS);
@@ -242,8 +242,9 @@ public TestServices build() {
242242
PolarisCredentialManager credentialManager =
243243
new DefaultPolarisCredentialManager(realmContext, mockCredentialVendors);
244244

245-
FileIOFactory fileIOFactory =
246-
fileIOFactorySupplier.apply(storageCredentialCache, metaStoreManagerFactory);
245+
AccessConfigProvider accessConfigProvider =
246+
new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory);
247+
FileIOFactory fileIOFactory = fileIOFactorySupplier.apply(accessConfigProvider);
247248

248249
TaskExecutor taskExecutor = Mockito.mock(TaskExecutor.class);
249250

@@ -257,9 +258,6 @@ public TestServices build() {
257258
fileIOFactory,
258259
polarisEventListener);
259260

260-
AccessConfigProvider accessConfigProvider =
261-
new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory);
262-
263261
ReservedProperties reservedProperties = ReservedProperties.NONE;
264262

265263
CatalogHandlerUtils catalogHandlerUtils = new CatalogHandlerUtils(realmConfig);

0 commit comments

Comments
 (0)