partitionValues)
- {
- try {
- GetPartitionResult result = stats.getGetPartition().call(() ->
- glueClient.getPartition(new GetPartitionRequest()
- .withDatabaseName(table.getDatabaseName())
- .withTableName(table.getTableName())
- .withPartitionValues(partitionValues)));
- return Optional.of(new GluePartitionConverter(table.getDatabaseName(), table.getTableName()).apply(result.getPartition()));
- }
- catch (EntityNotFoundException e) {
- return Optional.empty();
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- /**
- *
- * Ex: Partition keys = ['a', 'b']
- * Partition names = ['a=1/b=2', 'a=2/b=2']
- *
- *
- * @param partitionNames List of full partition names
- * @return Mapping of partition name to the partition object
- */
- @Override
- public Map> getPartitionsByNames(Table table, List partitionNames)
- {
- return getPartitionsByNamesInternal(table, partitionNames);
- }
-
- private Map> getPartitionsByNamesInternal(Table table, Collection partitionNames)
- {
- requireNonNull(partitionNames, "partitionNames is null");
- if (partitionNames.isEmpty()) {
- return ImmutableMap.of();
- }
-
- List partitions = batchGetPartition(table, partitionNames);
-
- Map> partitionNameToPartitionValuesMap = partitionNames.stream()
- .collect(toMap(identity(), Partitions::toPartitionValues));
- Map, Partition> partitionValuesToPartitionMap = partitions.stream()
- .collect(toMap(Partition::getValues, identity()));
-
- ImmutableMap.Builder> resultBuilder = ImmutableMap.builder();
- for (Entry> entry : partitionNameToPartitionValuesMap.entrySet()) {
- Partition partition = partitionValuesToPartitionMap.get(entry.getValue());
- resultBuilder.put(entry.getKey(), Optional.ofNullable(partition));
- }
- return resultBuilder.buildOrThrow();
- }
-
- private List batchGetPartition(Table table, Collection partitionNames)
- {
- List> batchGetPartitionFutures = new ArrayList<>();
- try {
- List pendingPartitions = partitionNames.stream()
- .map(partitionName -> new PartitionValueList().withValues(toPartitionValues(partitionName)))
- .collect(toCollection(ArrayList::new));
-
- ImmutableList.Builder resultsBuilder = ImmutableList.builderWithExpectedSize(partitionNames.size());
-
- // Reuse immutable field instances opportunistically between partitions
- GluePartitionConverter converter = new GluePartitionConverter(table.getDatabaseName(), table.getTableName());
-
- while (!pendingPartitions.isEmpty()) {
- for (List partitions : Lists.partition(pendingPartitions, BATCH_GET_PARTITION_MAX_PAGE_SIZE)) {
- batchGetPartitionFutures.add(glueClient.batchGetPartitionAsync(new BatchGetPartitionRequest()
- .withDatabaseName(table.getDatabaseName())
- .withTableName(table.getTableName())
- .withPartitionsToGet(partitions),
- new StatsRecordingAsyncHandler<>(stats.getGetPartitions())));
- }
- pendingPartitions.clear();
-
- for (Future future : batchGetPartitionFutures) {
- BatchGetPartitionResult batchGetPartitionResult = future.get();
- List partitions = batchGetPartitionResult.getPartitions();
- List unprocessedKeys = batchGetPartitionResult.getUnprocessedKeys();
-
- // In the unlikely scenario where batchGetPartition call cannot make progress on retrieving partitions, avoid infinite loop
- // We fail only in case there are still unprocessedKeys. Case with empty partitions and empty unprocessedKeys is correct in case partitions from request are not found.
- if (partitions.isEmpty() && !unprocessedKeys.isEmpty()) {
- throw new TrinoException(HIVE_METASTORE_ERROR, "Cannot make progress retrieving partitions. Unable to retrieve partitions: " + unprocessedKeys);
- }
-
- partitions.stream()
- .map(converter)
- .forEach(resultsBuilder::add);
- pendingPartitions.addAll(unprocessedKeys);
- }
- batchGetPartitionFutures.clear();
- }
-
- return resultsBuilder.build();
- }
- catch (AmazonServiceException | InterruptedException | ExecutionException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- finally {
- // Ensure any futures still running are canceled in case of failure
- batchGetPartitionFutures.forEach(future -> future.cancel(true));
- }
- }
-
- @Override
- public void addPartitions(String databaseName, String tableName, List partitions)
- {
- try {
- stats.getCreatePartitions().call(() -> {
- List> futures = new ArrayList<>();
-
- for (List partitionBatch : Lists.partition(partitions, BATCH_CREATE_PARTITION_MAX_PAGE_SIZE)) {
- List partitionInputs = mappedCopy(partitionBatch, GlueInputConverter::convertPartition);
- futures.add(glueClient.batchCreatePartitionAsync(
- new BatchCreatePartitionRequest()
- .withDatabaseName(databaseName)
- .withTableName(tableName)
- .withPartitionInputList(partitionInputs),
- new StatsRecordingAsyncHandler<>(stats.getBatchCreatePartition())));
- }
-
- for (Future future : futures) {
- try {
- BatchCreatePartitionResult result = future.get();
- propagatePartitionErrorToTrinoException(databaseName, tableName, result.getErrors());
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- Set updates = partitions.stream()
- .map(partitionWithStatistics -> new GlueColumnStatisticsProvider.PartitionStatisticsUpdate(
- partitionWithStatistics.getPartition(),
- partitionWithStatistics.getStatistics().columnStatistics()))
- .collect(toImmutableSet());
- columnStatisticsProvider.updatePartitionStatistics(updates);
-
- return null;
- });
- }
- catch (AmazonServiceException | ExecutionException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- private static void propagatePartitionErrorToTrinoException(String databaseName, String tableName, List partitionErrors)
- {
- if (partitionErrors != null && !partitionErrors.isEmpty()) {
- ErrorDetail errorDetail = partitionErrors.getFirst().getErrorDetail();
- String glueExceptionCode = errorDetail.getErrorCode();
-
- switch (glueExceptionCode) {
- case "AlreadyExistsException":
- throw new TrinoException(ALREADY_EXISTS, errorDetail.getErrorMessage());
- case "EntityNotFoundException":
- throw new TableNotFoundException(new SchemaTableName(databaseName, tableName), errorDetail.getErrorMessage());
- default:
- throw new TrinoException(HIVE_METASTORE_ERROR, errorDetail.getErrorCode() + ": " + errorDetail.getErrorMessage());
- }
- }
- }
-
- @Override
- public void dropPartition(String databaseName, String tableName, List parts, boolean deleteData)
- {
- Table table = getExistingTable(databaseName, tableName);
- Partition partition = getPartition(table, parts)
- .orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), parts));
-
- try {
- stats.getDeletePartition().call(() ->
- glueClient.deletePartition(new DeletePartitionRequest()
- .withDatabaseName(databaseName)
- .withTableName(tableName)
- .withPartitionValues(parts)));
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
-
- String partLocation = partition.getStorage().getLocation();
- if (deleteData && isManagedTable(table) && !isNullOrEmpty(partLocation)) {
- deleteDir(Location.of(partLocation));
- }
- }
-
- @Override
- public void alterPartition(String databaseName, String tableName, PartitionWithStatistics partition)
- {
- try {
- PartitionInput newPartition = convertPartition(partition);
- stats.getUpdatePartition().call(() ->
- glueClient.updatePartition(new UpdatePartitionRequest()
- .withDatabaseName(databaseName)
- .withTableName(tableName)
- .withPartitionInput(newPartition)
- .withPartitionValueList(partition.getPartition().getValues())));
- columnStatisticsProvider.updatePartitionStatistics(
- partition.getPartition(),
- partition.getStatistics().columnStatistics());
- }
- catch (EntityNotFoundException e) {
- throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partition.getPartition().getValues(), e);
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- @Override
- public Map> getPartitionColumnStatistics(
- String databaseName,
- String tableName,
- Set partitionNames,
- Set columnNames)
- {
- checkArgument(!columnNames.isEmpty(), "columnNames is empty");
- return columnStatisticsProvider.getPartitionColumnStatistics(databaseName, tableName, partitionNames, columnNames);
- }
-
- @Override
- public void updatePartitionStatistics(Table table, StatisticsUpdateMode mode, Map partitionUpdates)
- {
- Iterables.partition(partitionUpdates.entrySet(), BATCH_CREATE_PARTITION_MAX_PAGE_SIZE)
- .forEach(batch -> updatePartitionStatisticsBatch(table, mode, batch.stream().collect(toImmutableMap(Entry::getKey, Entry::getValue))));
- }
-
- private void updatePartitionStatisticsBatch(Table table, StatisticsUpdateMode mode, Map partitionUpdates)
- {
- // Missing partitions are ignored
- Map partitions = getPartitionsByNamesInternal(table, partitionUpdates.keySet()).entrySet().stream()
- .filter(entry -> entry.getValue().isPresent())
- .collect(toImmutableMap(Entry::getKey, entry -> entry.getValue().orElseThrow()));
- Map> currentColumnStats = columnStatisticsProvider.getPartitionColumnStatistics(
- table.getDatabaseName(),
- table.getTableName(),
- partitionUpdates.keySet(),
- table.getDataColumns().stream().map(Column::getName).collect(toImmutableSet()));
-
- ImmutableList.Builder partitionUpdateRequests = ImmutableList.builder();
- ImmutableSet.Builder columnStatisticsUpdates = ImmutableSet.builder();
- partitions.forEach((partitionName, partition) -> {
- PartitionStatistics update = partitionUpdates.get(partitionName);
-
- PartitionStatistics currentStatistics = new PartitionStatistics(getHiveBasicStatistics(partition.getParameters()), currentColumnStats.get(partitionName));
- PartitionStatistics updatedStatistics = mode.updatePartitionStatistics(currentStatistics, update);
-
- Map updatedStatisticsParameters = updateStatisticsParameters(partition.getParameters(), updatedStatistics.basicStatistics());
-
- partition = Partition.builder(partition).setParameters(updatedStatisticsParameters).build();
- Map updatedColumnStatistics = updatedStatistics.columnStatistics();
-
- PartitionInput partitionInput = convertPartition(partition);
- partitionInput.setParameters(partition.getParameters());
-
- partitionUpdateRequests.add(new BatchUpdatePartitionRequestEntry()
- .withPartitionValueList(partition.getValues())
- .withPartitionInput(partitionInput));
- columnStatisticsUpdates.add(new GlueColumnStatisticsProvider.PartitionStatisticsUpdate(partition, updatedColumnStatistics));
- });
-
- List> partitionUpdateRequestsPartitioned = Lists.partition(partitionUpdateRequests.build(), BATCH_UPDATE_PARTITION_MAX_PAGE_SIZE);
- List> partitionUpdateRequestsFutures = new ArrayList<>();
- partitionUpdateRequestsPartitioned.forEach(partitionUpdateRequestsPartition -> {
- // Update basic statistics
- partitionUpdateRequestsFutures.add(glueClient.batchUpdatePartitionAsync(new BatchUpdatePartitionRequest()
- .withDatabaseName(table.getDatabaseName())
- .withTableName(table.getTableName())
- .withEntries(partitionUpdateRequestsPartition),
- new StatsRecordingAsyncHandler<>(stats.getBatchUpdatePartition())));
- });
-
- try {
- // Update column statistics
- columnStatisticsProvider.updatePartitionStatistics(columnStatisticsUpdates.build());
- // Don't block on the batch update call until the column statistics have finished updating
- partitionUpdateRequestsFutures.forEach(MoreFutures::getFutureValue);
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- @Override
- public boolean functionExists(String databaseName, String functionName, String signatureToken)
- {
- try {
- stats.getGetUserDefinedFunction().call(() ->
- glueClient.getUserDefinedFunction(new GetUserDefinedFunctionRequest()
- .withDatabaseName(databaseName)
- .withFunctionName(metastoreFunctionName(functionName, signatureToken))));
- return true;
- }
- catch (EntityNotFoundException e) {
- return false;
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- @Override
- public Collection getAllFunctions(String databaseName)
- {
- return getFunctionsByPattern(databaseName, "trino__.*");
- }
-
- @Override
- public Collection getFunctions(String databaseName, String functionName)
- {
- return getFunctionsByPattern(databaseName, "trino__" + Pattern.quote(functionName) + "__.*");
- }
-
- private Collection getFunctionsByPattern(String databaseName, String functionNamePattern)
- {
- try {
- return getPaginatedResults(
- glueClient::getUserDefinedFunctions,
- new GetUserDefinedFunctionsRequest()
- .withDatabaseName(databaseName)
- .withPattern(functionNamePattern)
- .withMaxResults(AWS_GLUE_GET_FUNCTIONS_MAX_RESULTS),
- GetUserDefinedFunctionsRequest::setNextToken,
- GetUserDefinedFunctionsResult::getNextToken,
- stats.getGetUserDefinedFunctions())
- .map(GetUserDefinedFunctionsResult::getUserDefinedFunctions)
- .flatMap(List::stream)
- .map(GlueToTrinoConverter::convertFunction)
- .collect(toImmutableList());
- }
- catch (EntityNotFoundException | AccessDeniedException e) {
- return ImmutableList.of();
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- @Override
- public void createFunction(String databaseName, String functionName, LanguageFunction function)
- {
- if (functionName.contains("__")) {
- throw new TrinoException(NOT_SUPPORTED, "Function names with double underscore are not supported");
- }
- try {
- UserDefinedFunctionInput functionInput = convertFunction(functionName, function);
- stats.getCreateUserDefinedFunction().call(() ->
- glueClient.createUserDefinedFunction(new CreateUserDefinedFunctionRequest()
- .withDatabaseName(databaseName)
- .withFunctionInput(functionInput)));
- }
- catch (AlreadyExistsException e) {
- throw new TrinoException(ALREADY_EXISTS, "Function already exists", e);
- }
- catch (EntityNotFoundException e) {
- throw new SchemaNotFoundException(databaseName, e);
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- @Override
- public void replaceFunction(String databaseName, String functionName, LanguageFunction function)
- {
- try {
- UserDefinedFunctionInput functionInput = convertFunction(functionName, function);
- stats.getUpdateUserDefinedFunction().call(() ->
- glueClient.updateUserDefinedFunction(new UpdateUserDefinedFunctionRequest()
- .withDatabaseName(databaseName)
- .withFunctionName(metastoreFunctionName(functionName, function.signatureToken()))
- .withFunctionInput(functionInput)));
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- @Override
- public void dropFunction(String databaseName, String functionName, String signatureToken)
- {
- try {
- stats.getDeleteUserDefinedFunction().call(() ->
- glueClient.deleteUserDefinedFunction(new DeleteUserDefinedFunctionRequest()
- .withDatabaseName(databaseName)
- .withFunctionName(metastoreFunctionName(functionName, signatureToken))));
- }
- catch (EntityNotFoundException e) {
- throw new TrinoException(FUNCTION_NOT_FOUND, "Function not found", e);
- }
- catch (AmazonServiceException e) {
- throw new TrinoException(HIVE_METASTORE_ERROR, e);
- }
- }
-
- @Override
- public void createRole(String role, String grantor)
- {
- throw new TrinoException(NOT_SUPPORTED, "createRole is not supported by Glue");
- }
-
- @Override
- public void dropRole(String role)
- {
- throw new TrinoException(NOT_SUPPORTED, "dropRole is not supported by Glue");
- }
-
- @Override
- public Set listRoles()
- {
- return ImmutableSet.of(PUBLIC_ROLE_NAME);
- }
-
- @Override
- public void grantRoles(Set roles, Set grantees, boolean adminOption, HivePrincipal grantor)
- {
- throw new TrinoException(NOT_SUPPORTED, "grantRoles is not supported by Glue");
- }
-
- @Override
- public void revokeRoles(Set roles, Set grantees, boolean adminOption, HivePrincipal grantor)
- {
- throw new TrinoException(NOT_SUPPORTED, "revokeRoles is not supported by Glue");
- }
-
- @Override
- public Set listRoleGrants(HivePrincipal principal)
- {
- if (principal.getType() == USER) {
- return ImmutableSet.of(new RoleGrant(principal.toTrinoPrincipal(), PUBLIC_ROLE_NAME, false));
- }
- return ImmutableSet.of();
- }
-
- @Override
- public void grantTablePrivileges(String databaseName, String tableName, String tableOwner, HivePrincipal grantee, HivePrincipal grantor, Set privileges, boolean grantOption)
- {
- throw new TrinoException(NOT_SUPPORTED, "grantTablePrivileges is not supported by Glue");
- }
-
- @Override
- public void revokeTablePrivileges(String databaseName, String tableName, String tableOwner, HivePrincipal grantee, HivePrincipal grantor, Set privileges, boolean grantOption)
- {
- throw new TrinoException(NOT_SUPPORTED, "revokeTablePrivileges is not supported by Glue");
- }
-
- @Override
- public Set listTablePrivileges(String databaseName, String tableName, Optional tableOwner, Optional principal)
- {
- return ImmutableSet.of();
- }
-
- @Override
- public void checkSupportsTransactions()
- {
- throw new TrinoException(NOT_SUPPORTED, "Glue does not support ACID tables");
- }
-
- private Stream getGlueTables(String databaseName)
- {
- return getPaginatedResults(
- glueClient::getTables,
- new GetTablesRequest()
- .withDatabaseName(databaseName)
- .withMaxResults(AWS_GLUE_GET_TABLES_MAX_RESULTS),
- GetTablesRequest::setNextToken,
- GetTablesResult::getNextToken,
- stats.getGetTables())
- .map(GetTablesResult::getTableList)
- .flatMap(List::stream);
- }
-
- static class StatsRecordingAsyncHandler
- implements AsyncHandler
- {
- private final AwsApiCallStats stats;
- private final Stopwatch stopwatch;
-
- public StatsRecordingAsyncHandler(AwsApiCallStats stats)
- {
- this.stats = requireNonNull(stats, "stats is null");
- this.stopwatch = Stopwatch.createStarted();
- }
-
- @Override
- public void onError(Exception e)
- {
- stats.recordCall(stopwatch.elapsed(NANOSECONDS), true);
- }
-
- @Override
- public void onSuccess(AmazonWebServiceRequest request, Object o)
- {
- stats.recordCall(stopwatch.elapsed(NANOSECONDS), false);
- }
- }
-}
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreConfig.java
deleted file mode 100644
index 2fde27caaa62..000000000000
--- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreConfig.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.trino.plugin.hive.metastore.glue.v1;
-
-import io.airlift.configuration.Config;
-import io.airlift.configuration.ConfigDescription;
-import io.airlift.configuration.ConfigSecuritySensitive;
-import io.airlift.configuration.DefunctConfig;
-import io.airlift.configuration.LegacyConfig;
-import jakarta.validation.constraints.AssertTrue;
-import jakarta.validation.constraints.Max;
-import jakarta.validation.constraints.Min;
-
-import java.util.Optional;
-
-@DefunctConfig("hive.metastore.glue.use-instance-credentials")
-public class GlueHiveMetastoreConfig
-{
- private Optional glueRegion = Optional.empty();
- private Optional glueEndpointUrl = Optional.empty();
- private Optional glueStsRegion = Optional.empty();
- private Optional glueStsEndpointUrl = Optional.empty();
- private Optional glueProxyApiId = Optional.empty();
- private boolean pinGlueClientToCurrentRegion;
- private int maxGlueErrorRetries = 10;
- private int maxGlueConnections = 30;
- private Optional defaultWarehouseDir = Optional.empty();
- private Optional iamRole = Optional.empty();
- private Optional externalId = Optional.empty();
- private Optional awsAccessKey = Optional.empty();
- private Optional awsSecretKey = Optional.empty();
- private Optional awsCredentialsProvider = Optional.empty();
- private Optional catalogId = Optional.empty();
- private int partitionSegments = 5;
- private int getPartitionThreads = 20;
- private int readStatisticsThreads = 5;
- private int writeStatisticsThreads = 20;
- private boolean assumeCanonicalPartitionKeys;
- private boolean skipArchive;
-
- public Optional getGlueRegion()
- {
- return glueRegion;
- }
-
- @Config("hive.metastore.glue.region")
- @ConfigDescription("AWS Region for Glue Data Catalog")
- public GlueHiveMetastoreConfig setGlueRegion(String region)
- {
- this.glueRegion = Optional.ofNullable(region);
- return this;
- }
-
- public Optional getGlueEndpointUrl()
- {
- return glueEndpointUrl;
- }
-
- @Config("hive.metastore.glue.endpoint-url")
- @ConfigDescription("Glue API endpoint URL")
- public GlueHiveMetastoreConfig setGlueEndpointUrl(String glueEndpointUrl)
- {
- this.glueEndpointUrl = Optional.ofNullable(glueEndpointUrl);
- return this;
- }
-
- public Optional getGlueStsRegion()
- {
- return glueStsRegion;
- }
-
- @Config("hive.metastore.glue.sts.region")
- @ConfigDescription("AWS STS signing region for Glue authentication")
- public GlueHiveMetastoreConfig setGlueStsRegion(String glueStsRegion)
- {
- this.glueStsRegion = Optional.ofNullable(glueStsRegion);
- return this;
- }
-
- public Optional getGlueStsEndpointUrl()
- {
- return glueStsEndpointUrl;
- }
-
- @Config("hive.metastore.glue.sts.endpoint")
- @ConfigDescription("AWS STS endpoint for Glue authentication")
- public GlueHiveMetastoreConfig setGlueStsEndpointUrl(String glueStsEndpointUrl)
- {
- this.glueStsEndpointUrl = Optional.ofNullable(glueStsEndpointUrl);
- return this;
- }
-
- public Optional getGlueProxyApiId()
- {
- return glueProxyApiId;
- }
-
- @Config("hive.metastore.glue.proxy-api-id")
- @ConfigDescription("ID of Glue Proxy API")
- public GlueHiveMetastoreConfig setGlueProxyApiId(String glueProxyApiId)
- {
- this.glueProxyApiId = Optional.ofNullable(glueProxyApiId);
- return this;
- }
-
- public boolean getPinGlueClientToCurrentRegion()
- {
- return pinGlueClientToCurrentRegion;
- }
-
- @Config("hive.metastore.glue.pin-client-to-current-region")
- @ConfigDescription("Should the Glue client be pinned to the current EC2 region")
- public GlueHiveMetastoreConfig setPinGlueClientToCurrentRegion(boolean pinGlueClientToCurrentRegion)
- {
- this.pinGlueClientToCurrentRegion = pinGlueClientToCurrentRegion;
- return this;
- }
-
- @Min(1)
- public int getMaxGlueConnections()
- {
- return maxGlueConnections;
- }
-
- @Config("hive.metastore.glue.max-connections")
- @ConfigDescription("Max number of concurrent connections to Glue")
- public GlueHiveMetastoreConfig setMaxGlueConnections(int maxGlueConnections)
- {
- this.maxGlueConnections = maxGlueConnections;
- return this;
- }
-
- @Min(0)
- public int getMaxGlueErrorRetries()
- {
- return maxGlueErrorRetries;
- }
-
- @Config("hive.metastore.glue.max-error-retries")
- @ConfigDescription("Maximum number of error retries for the Glue client")
- public GlueHiveMetastoreConfig setMaxGlueErrorRetries(int maxGlueErrorRetries)
- {
- this.maxGlueErrorRetries = maxGlueErrorRetries;
- return this;
- }
-
- public Optional getDefaultWarehouseDir()
- {
- return defaultWarehouseDir;
- }
-
- @Config("hive.metastore.glue.default-warehouse-dir")
- @ConfigDescription("Hive Glue metastore default warehouse directory")
- public GlueHiveMetastoreConfig setDefaultWarehouseDir(String defaultWarehouseDir)
- {
- this.defaultWarehouseDir = Optional.ofNullable(defaultWarehouseDir);
- return this;
- }
-
- public Optional getIamRole()
- {
- return iamRole;
- }
-
- @Config("hive.metastore.glue.iam-role")
- @ConfigDescription("ARN of an IAM role to assume when connecting to Glue")
- public GlueHiveMetastoreConfig setIamRole(String iamRole)
- {
- this.iamRole = Optional.ofNullable(iamRole);
- return this;
- }
-
- public Optional getExternalId()
- {
- return externalId;
- }
-
- @Config("hive.metastore.glue.external-id")
- @ConfigDescription("External ID for the IAM role trust policy when connecting to Glue")
- public GlueHiveMetastoreConfig setExternalId(String externalId)
- {
- this.externalId = Optional.ofNullable(externalId);
- return this;
- }
-
- public Optional getAwsAccessKey()
- {
- return awsAccessKey;
- }
-
- @Config("hive.metastore.glue.aws-access-key")
- @ConfigDescription("Hive Glue metastore AWS access key")
- public GlueHiveMetastoreConfig setAwsAccessKey(String awsAccessKey)
- {
- this.awsAccessKey = Optional.ofNullable(awsAccessKey);
- return this;
- }
-
- public Optional getAwsSecretKey()
- {
- return awsSecretKey;
- }
-
- @Config("hive.metastore.glue.aws-secret-key")
- @ConfigDescription("Hive Glue metastore AWS secret key")
- @ConfigSecuritySensitive
- public GlueHiveMetastoreConfig setAwsSecretKey(String awsSecretKey)
- {
- this.awsSecretKey = Optional.ofNullable(awsSecretKey);
- return this;
- }
-
- public Optional getCatalogId()
- {
- return catalogId;
- }
-
- @Config("hive.metastore.glue.catalogid")
- @ConfigDescription("Hive Glue metastore catalog id")
- public GlueHiveMetastoreConfig setCatalogId(String catalogId)
- {
- this.catalogId = Optional.ofNullable(catalogId);
- return this;
- }
-
- public Optional getAwsCredentialsProvider()
- {
- return awsCredentialsProvider;
- }
-
- @Config("hive.metastore.glue.aws-credentials-provider")
- @ConfigDescription("Fully qualified name of the Java class to use for obtaining AWS credentials")
- public GlueHiveMetastoreConfig setAwsCredentialsProvider(String awsCredentialsProvider)
- {
- this.awsCredentialsProvider = Optional.ofNullable(awsCredentialsProvider);
- return this;
- }
-
- @Min(1)
- @Max(10)
- public int getPartitionSegments()
- {
- return partitionSegments;
- }
-
- @Config("hive.metastore.glue.partitions-segments")
- @ConfigDescription("Number of segments for partitioned Glue tables")
- public GlueHiveMetastoreConfig setPartitionSegments(int partitionSegments)
- {
- this.partitionSegments = partitionSegments;
- return this;
- }
-
- @Min(1)
- public int getGetPartitionThreads()
- {
- return getPartitionThreads;
- }
-
- @Config("hive.metastore.glue.get-partition-threads")
- @ConfigDescription("Number of threads for parallel partition fetches from Glue")
- public GlueHiveMetastoreConfig setGetPartitionThreads(int getPartitionThreads)
- {
- this.getPartitionThreads = getPartitionThreads;
- return this;
- }
-
- @Min(1)
- public int getReadStatisticsThreads()
- {
- return readStatisticsThreads;
- }
-
- @Config("hive.metastore.glue.read-statistics-threads")
- @ConfigDescription("Number of threads for parallel statistics reads from Glue")
- public GlueHiveMetastoreConfig setReadStatisticsThreads(int getReadStatisticsThreads)
- {
- this.readStatisticsThreads = getReadStatisticsThreads;
- return this;
- }
-
- @Min(1)
- public int getWriteStatisticsThreads()
- {
- return writeStatisticsThreads;
- }
-
- @Config("hive.metastore.glue.write-statistics-threads")
- @ConfigDescription("Number of threads for parallel statistics writes to Glue")
- public GlueHiveMetastoreConfig setWriteStatisticsThreads(int writeStatisticsThreads)
- {
- this.writeStatisticsThreads = writeStatisticsThreads;
- return this;
- }
-
- public boolean isAssumeCanonicalPartitionKeys()
- {
- return assumeCanonicalPartitionKeys;
- }
-
- @Config("hive.metastore.glue.assume-canonical-partition-keys")
- @ConfigDescription("Allow conversion of non-char types (eg BIGINT, timestamp) to canonical string formats")
- public GlueHiveMetastoreConfig setAssumeCanonicalPartitionKeys(boolean assumeCanonicalPartitionKeys)
- {
- this.assumeCanonicalPartitionKeys = assumeCanonicalPartitionKeys;
- return this;
- }
-
- public boolean isSkipArchive()
- {
- return skipArchive;
- }
-
- @Config("hive.metastore.glue.skip-archive")
- @LegacyConfig("iceberg.glue.skip-archive")
- @ConfigDescription("Skip archiving an old table version when updating a table in the Glue metastore")
- public GlueHiveMetastoreConfig setSkipArchive(boolean skipArchive)
- {
- this.skipArchive = skipArchive;
- return this;
- }
-
- @AssertTrue(message = "Both hive.metastore.glue.region and hive.metastore.glue.endpoint-url must be provided when Glue proxy API ID is present")
- public boolean isGlueProxyApiIdValid()
- {
- if (getGlueProxyApiId().isPresent()) {
- return getGlueRegion().isPresent() && getGlueEndpointUrl().isPresent();
- }
- return true;
- }
-}
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreFactory.java
deleted file mode 100644
index 8d6412b08f5f..000000000000
--- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueHiveMetastoreFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.trino.plugin.hive.metastore.glue.v1;
-
-import com.google.inject.Inject;
-import io.opentelemetry.api.trace.Tracer;
-import io.trino.metastore.HiveMetastore;
-import io.trino.metastore.HiveMetastoreFactory;
-import io.trino.metastore.tracing.TracingHiveMetastore;
-import io.trino.spi.security.ConnectorIdentity;
-
-import java.util.Optional;
-
-public class GlueHiveMetastoreFactory
- implements HiveMetastoreFactory
-{
- private final HiveMetastore metastore;
-
- // Glue metastore does not support impersonation, so just use single shared instance
- @Inject
- public GlueHiveMetastoreFactory(GlueHiveMetastore metastore, Tracer tracer)
- {
- this.metastore = new TracingHiveMetastore(tracer, metastore);
- }
-
- @Override
- public boolean isImpersonationEnabled()
- {
- return false;
- }
-
- @Override
- public HiveMetastore createMetastore(Optional identity)
- {
- return metastore;
- }
-}
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueInputConverter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueInputConverter.java
deleted file mode 100644
index 9d16be3760ce..000000000000
--- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueInputConverter.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.trino.plugin.hive.metastore.glue.v1;
-
-import com.amazonaws.services.glue.model.DatabaseInput;
-import com.amazonaws.services.glue.model.Order;
-import com.amazonaws.services.glue.model.PartitionInput;
-import com.amazonaws.services.glue.model.PrincipalType;
-import com.amazonaws.services.glue.model.ResourceType;
-import com.amazonaws.services.glue.model.ResourceUri;
-import com.amazonaws.services.glue.model.SerDeInfo;
-import com.amazonaws.services.glue.model.StorageDescriptor;
-import com.amazonaws.services.glue.model.TableInput;
-import com.amazonaws.services.glue.model.UserDefinedFunctionInput;
-import com.google.common.collect.ImmutableMap;
-import io.airlift.json.JsonCodec;
-import io.trino.metastore.Column;
-import io.trino.metastore.Database;
-import io.trino.metastore.HiveBucketProperty;
-import io.trino.metastore.Partition;
-import io.trino.metastore.PartitionStatistics;
-import io.trino.metastore.PartitionWithStatistics;
-import io.trino.metastore.Storage;
-import io.trino.metastore.Table;
-import io.trino.spi.function.LanguageFunction;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
-import static io.trino.metastore.Table.TABLE_COMMENT;
-import static io.trino.plugin.hive.ViewReaderUtil.isTrinoMaterializedView;
-import static io.trino.plugin.hive.ViewReaderUtil.isTrinoView;
-import static io.trino.plugin.hive.metastore.MetastoreUtil.metastoreFunctionName;
-import static io.trino.plugin.hive.metastore.MetastoreUtil.toResourceUris;
-import static io.trino.plugin.hive.metastore.MetastoreUtil.updateStatisticsParameters;
-import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getStorageDescriptor;
-import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableParameters;
-import static io.trino.plugin.hive.metastore.glue.v1.GlueToTrinoConverter.getTableTypeNullable;
-
-public final class GlueInputConverter
-{
- static final JsonCodec LANGUAGE_FUNCTION_CODEC = JsonCodec.jsonCodec(LanguageFunction.class);
-
- private GlueInputConverter() {}
-
- public static DatabaseInput convertDatabase(Database database)
- {
- DatabaseInput input = new DatabaseInput();
- input.setName(database.getDatabaseName());
- input.setParameters(database.getParameters());
- database.getComment().ifPresent(input::setDescription);
- database.getLocation().ifPresent(input::setLocationUri);
- return input;
- }
-
- public static TableInput convertTable(Table table)
- {
- Map tableParameters = table.getParameters();
- Optional comment = Optional.empty();
- if (!isTrinoView(table) && !isTrinoMaterializedView(table)) {
- comment = Optional.ofNullable(tableParameters.get(TABLE_COMMENT));
- tableParameters = tableParameters.entrySet().stream()
- .filter(entry -> !entry.getKey().equals(TABLE_COMMENT))
- .collect(toImmutableMap(Entry::getKey, Entry::getValue));
- }
-
- TableInput input = new TableInput();
- input.setName(table.getTableName());
- input.setOwner(table.getOwner().orElse(null));
- input.setTableType(table.getTableType());
- input.setStorageDescriptor(convertStorage(table.getStorage(), table.getDataColumns()));
- input.setPartitionKeys(table.getPartitionColumns().stream().map(GlueInputConverter::convertColumn).collect(toImmutableList()));
- input.setParameters(tableParameters);
- table.getViewOriginalText().ifPresent(input::setViewOriginalText);
- table.getViewExpandedText().ifPresent(input::setViewExpandedText);
- comment.ifPresent(input::setDescription);
- return input;
- }
-
- public static TableInput convertGlueTableToTableInput(com.amazonaws.services.glue.model.Table glueTable)
- {
- return new TableInput()
- .withName(glueTable.getName())
- .withDescription(glueTable.getDescription())
- .withOwner(glueTable.getOwner())
- .withLastAccessTime(glueTable.getLastAccessTime())
- .withLastAnalyzedTime(glueTable.getLastAnalyzedTime())
- .withRetention(glueTable.getRetention())
- .withStorageDescriptor(getStorageDescriptor(glueTable).orElse(null))
- .withPartitionKeys(glueTable.getPartitionKeys())
- .withViewOriginalText(glueTable.getViewOriginalText())
- .withViewExpandedText(glueTable.getViewExpandedText())
- .withTableType(getTableTypeNullable(glueTable))
- .withTargetTable(glueTable.getTargetTable())
- .withParameters(getTableParameters(glueTable));
- }
-
- public static PartitionInput convertPartition(PartitionWithStatistics partitionWithStatistics)
- {
- PartitionInput input = convertPartition(partitionWithStatistics.getPartition());
- PartitionStatistics statistics = partitionWithStatistics.getStatistics();
- input.setParameters(updateStatisticsParameters(input.getParameters(), statistics.basicStatistics()));
- return input;
- }
-
- public static PartitionInput convertPartition(Partition partition)
- {
- PartitionInput input = new PartitionInput();
- input.setValues(partition.getValues());
- input.setStorageDescriptor(convertStorage(partition.getStorage(), partition.getColumns()));
- input.setParameters(partition.getParameters());
- return input;
- }
-
- private static StorageDescriptor convertStorage(Storage storage, List columns)
- {
- if (storage.isSkewed()) {
- throw new IllegalArgumentException("Writing to skewed table/partition is not supported");
- }
- SerDeInfo serdeInfo = new SerDeInfo()
- .withSerializationLibrary(storage.getStorageFormat().getSerDeNullable())
- .withParameters(storage.getSerdeParameters());
-
- StorageDescriptor sd = new StorageDescriptor();
- sd.setLocation(storage.getLocation());
- sd.setColumns(columns.stream().map(GlueInputConverter::convertColumn).collect(toImmutableList()));
- sd.setSerdeInfo(serdeInfo);
- sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable());
- sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable());
- sd.setParameters(ImmutableMap.of());
-
- Optional bucketProperty = storage.getBucketProperty();
- if (bucketProperty.isPresent()) {
- sd.setNumberOfBuckets(bucketProperty.get().bucketCount());
- sd.setBucketColumns(bucketProperty.get().bucketedBy());
- if (!bucketProperty.get().sortedBy().isEmpty()) {
- sd.setSortColumns(bucketProperty.get().sortedBy().stream()
- .map(column -> new Order().withColumn(column.columnName()).withSortOrder(column.order().getHiveOrder()))
- .collect(toImmutableList()));
- }
- }
-
- return sd;
- }
-
- private static com.amazonaws.services.glue.model.Column convertColumn(Column trinoColumn)
- {
- return new com.amazonaws.services.glue.model.Column()
- .withName(trinoColumn.getName())
- .withType(trinoColumn.getType().toString())
- .withComment(trinoColumn.getComment().orElse(null))
- .withParameters(trinoColumn.getProperties());
- }
-
- public static UserDefinedFunctionInput convertFunction(String functionName, LanguageFunction function)
- {
- return new UserDefinedFunctionInput()
- .withFunctionName(metastoreFunctionName(functionName, function.signatureToken()))
- .withClassName("TrinoFunction")
- .withOwnerType(PrincipalType.USER)
- .withOwnerName(function.owner().orElse(null))
- .withResourceUris(toResourceUris(LANGUAGE_FUNCTION_CODEC.toJsonBytes(function)).stream()
- .map(uri -> new ResourceUri()
- .withResourceType(ResourceType.FILE)
- .withUri(uri.getUri()))
- .toList());
- }
-}
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java
deleted file mode 100644
index 1d1bbf384cf3..000000000000
--- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueMetastoreModule.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.trino.plugin.hive.metastore.glue.v1;
-
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.handlers.RequestHandler2;
-import com.amazonaws.services.glue.AWSGlueAsync;
-import com.amazonaws.services.glue.model.Table;
-import com.google.inject.Binder;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.Provider;
-import com.google.inject.Scopes;
-import com.google.inject.Singleton;
-import com.google.inject.TypeLiteral;
-import com.google.inject.multibindings.Multibinder;
-import com.google.inject.multibindings.ProvidesIntoSet;
-import io.airlift.concurrent.BoundedExecutor;
-import io.airlift.configuration.AbstractConfigurationAwareModule;
-import io.opentelemetry.api.OpenTelemetry;
-import io.opentelemetry.instrumentation.awssdk.v1_11.AwsSdkTelemetry;
-import io.trino.metastore.HiveMetastoreFactory;
-import io.trino.metastore.RawHiveMetastoreFactory;
-import io.trino.plugin.hive.AllowHiveTableRename;
-import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats;
-
-import java.lang.annotation.Annotation;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
-import java.util.function.Function;
-import java.util.function.Predicate;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
-import static com.google.inject.multibindings.Multibinder.newSetBinder;
-import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder;
-import static io.airlift.concurrent.Threads.daemonThreadsNamed;
-import static io.airlift.configuration.ConditionalModule.conditionalModule;
-import static io.trino.plugin.base.ClosingBinder.closingBinder;
-import static java.util.concurrent.Executors.newCachedThreadPool;
-import static org.weakref.jmx.guice.ExportBinder.newExporter;
-
-public class GlueMetastoreModule
- extends AbstractConfigurationAwareModule
-{
- @Override
- protected void setup(Binder binder)
- {
- GlueHiveMetastoreConfig glueConfig = buildConfigObject(GlueHiveMetastoreConfig.class);
- Multibinder requestHandlers = newSetBinder(binder, RequestHandler2.class, ForGlueHiveMetastore.class);
- glueConfig.getCatalogId().ifPresent(catalogId -> requestHandlers.addBinding().toInstance(new GlueCatalogIdRequestHandler(catalogId)));
- glueConfig.getGlueProxyApiId().ifPresent(glueProxyApiId -> requestHandlers.addBinding()
- .toInstance(new ProxyApiRequestHandler(glueProxyApiId)));
- binder.bind(AWSCredentialsProvider.class).toProvider(GlueCredentialsProvider.class).in(Scopes.SINGLETON);
-
- newOptionalBinder(binder, Key.get(new TypeLiteral>() {}, ForGlueHiveMetastore.class))
- .setDefault().toProvider(DefaultGlueMetastoreTableFilterProvider.class).in(Scopes.SINGLETON);
-
- binder.bind(GlueHiveMetastore.class).in(Scopes.SINGLETON);
- newOptionalBinder(binder, Key.get(HiveMetastoreFactory.class, RawHiveMetastoreFactory.class))
- .setDefault()
- .to(GlueHiveMetastoreFactory.class)
- .in(Scopes.SINGLETON);
-
- // export under the old name, for backwards compatibility
- binder.bind(GlueHiveMetastoreFactory.class).in(Scopes.SINGLETON);
- binder.bind(Key.get(GlueMetastoreStats.class, ForGlueHiveMetastore.class)).toInstance(new GlueMetastoreStats());
- binder.bind(AWSGlueAsync.class).toProvider(HiveGlueClientProvider.class).in(Scopes.SINGLETON);
- closingBinder(binder).registerResource(AWSGlueAsync.class, AWSGlueAsync::shutdown);
- newExporter(binder).export(GlueHiveMetastore.class).withGeneratedName();
-
- binder.bind(Key.get(boolean.class, AllowHiveTableRename.class)).toInstance(false);
-
- newOptionalBinder(binder, GlueColumnStatisticsProviderFactory.class)
- .setDefault().to(DefaultGlueColumnStatisticsProviderFactory.class).in(Scopes.SINGLETON);
-
- createExecutor(ForGlueHiveMetastore.class, "hive-glue-partitions-%s", GlueHiveMetastoreConfig::getGetPartitionThreads);
- createExecutor(ForGlueColumnStatisticsRead.class, "hive-glue-statistics-read-%s", GlueHiveMetastoreConfig::getReadStatisticsThreads);
- createExecutor(ForGlueColumnStatisticsWrite.class, "hive-glue-statistics-write-%s", GlueHiveMetastoreConfig::getWriteStatisticsThreads);
- }
-
- @ProvidesIntoSet
- @Singleton
- @ForGlueHiveMetastore
- public RequestHandler2 createSkipArchiveRequestHandler(GlueHiveMetastoreConfig config)
- {
- if (!config.isSkipArchive()) {
- return new RequestHandler2() {};
- }
- return new SkipArchiveRequestHandler();
- }
-
- @ProvidesIntoSet
- @Singleton
- @ForGlueHiveMetastore
- public RequestHandler2 createTelemetryRequestHandler(OpenTelemetry openTelemetry)
- {
- return AwsSdkTelemetry.builder(openTelemetry)
- .setCaptureExperimentalSpanAttributes(true)
- .build()
- .newRequestHandler();
- }
-
- private void createExecutor(Class annotationClass, String nameTemplate, Function threads)
- {
- install(conditionalModule(
- GlueHiveMetastoreConfig.class,
- config -> threads.apply(config) > 1,
- binder -> {
- binder.bind(Key.get(ExecutorService.class, annotationClass)).toInstance(newCachedThreadPool(daemonThreadsNamed(nameTemplate)));
- binder.bind(Key.get(Executor.class, annotationClass)).toProvider(new BoundedExecutorProvider(annotationClass, threads)).in(Scopes.SINGLETON);
- closingBinder(binder).registerExecutor(Key.get(ExecutorService.class, annotationClass));
- },
- binder -> binder.bind(Key.get(Executor.class, annotationClass))
- .toInstance(directExecutor())));
- }
-
- private static class BoundedExecutorProvider
- implements Provider
- {
- private final Class annotationClass;
- private final Function threads;
- private Injector injector;
-
- public BoundedExecutorProvider(Class annotationClass, Function threads)
- {
- this.annotationClass = annotationClass;
- this.threads = threads;
- }
-
- @Inject
- public void setInjector(Injector injector)
- {
- this.injector = injector;
- }
-
- @Override
- public BoundedExecutor get()
- {
- ExecutorService executor = injector.getInstance(Key.get(ExecutorService.class, annotationClass));
- int threads = this.threads.apply(injector.getInstance(GlueHiveMetastoreConfig.class));
- checkArgument(threads > 0, "Invalid number of threads: %s", threads);
- return new BoundedExecutor(executor, threads);
- }
- }
-}
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueStatConverter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueStatConverter.java
deleted file mode 100644
index fc71676c5a36..000000000000
--- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueStatConverter.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.trino.plugin.hive.metastore.glue.v1;
-
-import com.amazonaws.services.glue.model.BinaryColumnStatisticsData;
-import com.amazonaws.services.glue.model.BooleanColumnStatisticsData;
-import com.amazonaws.services.glue.model.ColumnStatistics;
-import com.amazonaws.services.glue.model.ColumnStatisticsData;
-import com.amazonaws.services.glue.model.ColumnStatisticsType;
-import com.amazonaws.services.glue.model.DateColumnStatisticsData;
-import com.amazonaws.services.glue.model.DecimalColumnStatisticsData;
-import com.amazonaws.services.glue.model.DecimalNumber;
-import com.amazonaws.services.glue.model.DoubleColumnStatisticsData;
-import com.amazonaws.services.glue.model.LongColumnStatisticsData;
-import com.amazonaws.services.glue.model.StringColumnStatisticsData;
-import io.trino.hive.thrift.metastore.Decimal;
-import io.trino.metastore.Column;
-import io.trino.metastore.HiveColumnStatistics;
-import io.trino.metastore.HiveType;
-import io.trino.metastore.Partition;
-import io.trino.metastore.Table;
-import io.trino.metastore.type.PrimitiveTypeInfo;
-import io.trino.metastore.type.TypeInfo;
-import io.trino.spi.TrinoException;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.time.LocalDate;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.OptionalDouble;
-import java.util.OptionalLong;
-import java.util.concurrent.TimeUnit;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static io.trino.metastore.HiveColumnStatistics.createBinaryColumnStatistics;
-import static io.trino.metastore.HiveColumnStatistics.createBooleanColumnStatistics;
-import static io.trino.metastore.HiveColumnStatistics.createDateColumnStatistics;
-import static io.trino.metastore.HiveColumnStatistics.createDecimalColumnStatistics;
-import static io.trino.metastore.HiveColumnStatistics.createDoubleColumnStatistics;
-import static io.trino.metastore.HiveColumnStatistics.createIntegerColumnStatistics;
-import static io.trino.metastore.HiveColumnStatistics.createStringColumnStatistics;
-import static io.trino.metastore.type.Category.PRIMITIVE;
-import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA;
-import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreNullsCount;
-
-public class GlueStatConverter
-{
- private GlueStatConverter() {}
-
- private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1);
-
- public static List toGlueColumnStatistics(Partition partition, Map trinoColumnStats)
- {
- return partition.getColumns().stream()
- .filter(column -> trinoColumnStats.containsKey(column.getName()))
- .map(c -> toColumnStatistics(c, trinoColumnStats.get(c.getName())))
- .collect(toImmutableList());
- }
-
- public static List toGlueColumnStatistics(Table table, Map trinoColumnStats)
- {
- return trinoColumnStats.entrySet().stream()
- .map(e -> toColumnStatistics(table.getColumn(e.getKey()).get(), e.getValue()))
- .collect(toImmutableList());
- }
-
- private static ColumnStatistics toColumnStatistics(Column column, HiveColumnStatistics statistics)
- {
- ColumnStatistics columnStatistics = new ColumnStatistics();
- HiveType columnType = column.getType();
- columnStatistics.setColumnName(column.getName());
- columnStatistics.setColumnType(columnType.toString());
- ColumnStatisticsData catalogColumnStatisticsData = toGlueColumnStatisticsData(statistics, columnType);
- columnStatistics.setStatisticsData(catalogColumnStatisticsData);
- columnStatistics.setAnalyzedTime(new Date());
- return columnStatistics;
- }
-
- public static HiveColumnStatistics fromGlueColumnStatistics(ColumnStatisticsData catalogColumnStatisticsData)
- {
- ColumnStatisticsType type = ColumnStatisticsType.fromValue(catalogColumnStatisticsData.getType());
- switch (type) {
- case BINARY: {
- BinaryColumnStatisticsData data = catalogColumnStatisticsData.getBinaryColumnStatisticsData();
- OptionalLong max = OptionalLong.of(data.getMaximumLength());
- OptionalDouble avg = OptionalDouble.of(data.getAverageLength());
- OptionalLong nulls = fromMetastoreNullsCount(data.getNumberOfNulls());
- return createBinaryColumnStatistics(max, avg, nulls);
- }
- case BOOLEAN: {
- BooleanColumnStatisticsData catalogBooleanData = catalogColumnStatisticsData.getBooleanColumnStatisticsData();
- return createBooleanColumnStatistics(
- OptionalLong.of(catalogBooleanData.getNumberOfTrues()),
- OptionalLong.of(catalogBooleanData.getNumberOfFalses()),
- fromMetastoreNullsCount(catalogBooleanData.getNumberOfNulls()));
- }
- case DATE: {
- DateColumnStatisticsData data = catalogColumnStatisticsData.getDateColumnStatisticsData();
- Optional min = dateToLocalDate(data.getMinimumValue());
- Optional max = dateToLocalDate(data.getMaximumValue());
- OptionalLong nullsCount = fromMetastoreNullsCount(data.getNumberOfNulls());
- OptionalLong distinctValuesWithNullCount = OptionalLong.of(data.getNumberOfDistinctValues());
- return createDateColumnStatistics(min, max, nullsCount, distinctValuesWithNullCount);
- }
- case DECIMAL: {
- DecimalColumnStatisticsData data = catalogColumnStatisticsData.getDecimalColumnStatisticsData();
- Optional min = glueDecimalToBigDecimal(data.getMinimumValue());
- Optional max = glueDecimalToBigDecimal(data.getMaximumValue());
- OptionalLong distinctValuesWithNullCount = OptionalLong.of(data.getNumberOfDistinctValues());
- OptionalLong nullsCount = fromMetastoreNullsCount(data.getNumberOfNulls());
- return createDecimalColumnStatistics(min, max, nullsCount, distinctValuesWithNullCount);
- }
- case DOUBLE: {
- DoubleColumnStatisticsData data = catalogColumnStatisticsData.getDoubleColumnStatisticsData();
- OptionalDouble min = OptionalDouble.of(data.getMinimumValue());
- OptionalDouble max = OptionalDouble.of(data.getMaximumValue());
- OptionalLong nulls = fromMetastoreNullsCount(data.getNumberOfNulls());
- OptionalLong distinctValuesWithNullCount = OptionalLong.of(data.getNumberOfDistinctValues());
- return createDoubleColumnStatistics(min, max, nulls, distinctValuesWithNullCount);
- }
- case LONG: {
- LongColumnStatisticsData data = catalogColumnStatisticsData.getLongColumnStatisticsData();
- OptionalLong min = OptionalLong.of(data.getMinimumValue());
- OptionalLong max = OptionalLong.of(data.getMaximumValue());
- OptionalLong nullsCount = fromMetastoreNullsCount(data.getNumberOfNulls());
- OptionalLong distinctValuesWithNullCount = OptionalLong.of(data.getNumberOfDistinctValues());
- return createIntegerColumnStatistics(min, max, nullsCount, distinctValuesWithNullCount);
- }
- case STRING: {
- StringColumnStatisticsData data = catalogColumnStatisticsData.getStringColumnStatisticsData();
- OptionalLong max = OptionalLong.of(data.getMaximumLength());
- OptionalDouble avg = OptionalDouble.of(data.getAverageLength());
- OptionalLong nullsCount = fromMetastoreNullsCount(data.getNumberOfNulls());
- OptionalLong distinctValuesWithNullCount = OptionalLong.of(data.getNumberOfDistinctValues());
- return createStringColumnStatistics(max, avg, nullsCount, distinctValuesWithNullCount);
- }
- }
-
- throw new TrinoException(HIVE_INVALID_METADATA, "Invalid column statistics data: " + catalogColumnStatisticsData);
- }
-
- private static ColumnStatisticsData toGlueColumnStatisticsData(HiveColumnStatistics statistics, HiveType columnType)
- {
- TypeInfo typeInfo = columnType.getTypeInfo();
- checkArgument(typeInfo.getCategory() == PRIMITIVE, "Unsupported statistics type: %s", columnType);
-
- ColumnStatisticsData catalogColumnStatisticsData = new ColumnStatisticsData();
-
- switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) {
- case BOOLEAN: {
- BooleanColumnStatisticsData data = new BooleanColumnStatisticsData();
- statistics.getNullsCount().ifPresent(data::setNumberOfNulls);
- statistics.getBooleanStatistics().ifPresent(booleanStatistics -> {
- booleanStatistics.getFalseCount().ifPresent(data::setNumberOfFalses);
- booleanStatistics.getTrueCount().ifPresent(data::setNumberOfTrues);
- });
- catalogColumnStatisticsData.setType(ColumnStatisticsType.BOOLEAN.toString());
- catalogColumnStatisticsData.setBooleanColumnStatisticsData(data);
- break;
- }
- case BINARY: {
- BinaryColumnStatisticsData data = new BinaryColumnStatisticsData();
- statistics.getNullsCount().ifPresent(data::setNumberOfNulls);
- data.setMaximumLength(statistics.getMaxValueSizeInBytes().orElse(0));
- data.setAverageLength(statistics.getAverageColumnLength().orElse(0));
- catalogColumnStatisticsData.setType(ColumnStatisticsType.BINARY.toString());
- catalogColumnStatisticsData.setBinaryColumnStatisticsData(data);
- break;
- }
- case DATE: {
- DateColumnStatisticsData data = new DateColumnStatisticsData();
- statistics.getDateStatistics().ifPresent(dateStatistics -> {
- dateStatistics.getMin().ifPresent(value -> data.setMinimumValue(localDateToDate(value)));
- dateStatistics.getMax().ifPresent(value -> data.setMaximumValue(localDateToDate(value)));
- });
- statistics.getNullsCount().ifPresent(data::setNumberOfNulls);
- statistics.getDistinctValuesWithNullCount().ifPresent(data::setNumberOfDistinctValues);
- catalogColumnStatisticsData.setType(ColumnStatisticsType.DATE.toString());
- catalogColumnStatisticsData.setDateColumnStatisticsData(data);
- break;
- }
- case DECIMAL: {
- DecimalColumnStatisticsData data = new DecimalColumnStatisticsData();
- statistics.getDecimalStatistics().ifPresent(decimalStatistics -> {
- decimalStatistics.getMin().ifPresent(value -> data.setMinimumValue(bigDecimalToGlueDecimal(value)));
- decimalStatistics.getMax().ifPresent(value -> data.setMaximumValue(bigDecimalToGlueDecimal(value)));
- });
- statistics.getNullsCount().ifPresent(data::setNumberOfNulls);
- statistics.getDistinctValuesWithNullCount().ifPresent(data::setNumberOfDistinctValues);
- catalogColumnStatisticsData.setType(ColumnStatisticsType.DECIMAL.toString());
- catalogColumnStatisticsData.setDecimalColumnStatisticsData(data);
- break;
- }
- case FLOAT:
- case DOUBLE: {
- DoubleColumnStatisticsData data = new DoubleColumnStatisticsData();
- statistics.getDoubleStatistics().ifPresent(doubleStatistics -> {
- doubleStatistics.getMin().ifPresent(data::setMinimumValue);
- doubleStatistics.getMax().ifPresent(data::setMaximumValue);
- });
- statistics.getNullsCount().ifPresent(data::setNumberOfNulls);
- statistics.getDistinctValuesWithNullCount().ifPresent(data::setNumberOfDistinctValues);
- catalogColumnStatisticsData.setType(ColumnStatisticsType.DOUBLE.toString());
- catalogColumnStatisticsData.setDoubleColumnStatisticsData(data);
- break;
- }
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- case TIMESTAMP: {
- LongColumnStatisticsData data = new LongColumnStatisticsData();
- statistics.getIntegerStatistics().ifPresent(stats -> {
- stats.getMin().ifPresent(data::setMinimumValue);
- stats.getMax().ifPresent(data::setMaximumValue);
- });
- statistics.getNullsCount().ifPresent(data::setNumberOfNulls);
- statistics.getDistinctValuesWithNullCount().ifPresent(data::setNumberOfDistinctValues);
- catalogColumnStatisticsData.setType(ColumnStatisticsType.LONG.toString());
- catalogColumnStatisticsData.setLongColumnStatisticsData(data);
- break;
- }
- case VARCHAR:
- case CHAR:
- case STRING: {
- StringColumnStatisticsData data = new StringColumnStatisticsData();
- statistics.getNullsCount().ifPresent(data::setNumberOfNulls);
- statistics.getDistinctValuesWithNullCount().ifPresent(data::setNumberOfDistinctValues);
- data.setMaximumLength(statistics.getMaxValueSizeInBytes().orElse(0));
- data.setAverageLength(statistics.getAverageColumnLength().orElse(0));
- catalogColumnStatisticsData.setType(ColumnStatisticsType.STRING.toString());
- catalogColumnStatisticsData.setStringColumnStatisticsData(data);
- break;
- }
- default:
- throw new TrinoException(HIVE_INVALID_METADATA, "Invalid column statistics type: " + ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
- }
- return catalogColumnStatisticsData;
- }
-
- private static DecimalNumber bigDecimalToGlueDecimal(BigDecimal decimal)
- {
- Decimal hiveDecimal = new Decimal((short) decimal.scale(), ByteBuffer.wrap(decimal.unscaledValue().toByteArray()));
- DecimalNumber catalogDecimal = new DecimalNumber();
- catalogDecimal.setUnscaledValue(ByteBuffer.wrap(hiveDecimal.getUnscaled()));
- catalogDecimal.setScale((int) hiveDecimal.getScale());
- return catalogDecimal;
- }
-
- private static Optional glueDecimalToBigDecimal(DecimalNumber catalogDecimal)
- {
- if (catalogDecimal == null) {
- return Optional.empty();
- }
- Decimal decimal = new Decimal();
- decimal.setUnscaled(catalogDecimal.getUnscaledValue());
- decimal.setScale(catalogDecimal.getScale().shortValue());
- return Optional.of(new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()));
- }
-
- private static Optional dateToLocalDate(Date date)
- {
- if (date == null) {
- return Optional.empty();
- }
- long daysSinceEpoch = date.getTime() / MILLIS_PER_DAY;
- return Optional.of(LocalDate.ofEpochDay(daysSinceEpoch));
- }
-
- private static Date localDateToDate(LocalDate date)
- {
- long millisecondsSinceEpoch = date.toEpochDay() * MILLIS_PER_DAY;
- return new Date(millisecondsSinceEpoch);
- }
-}
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueToTrinoConverter.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueToTrinoConverter.java
deleted file mode 100644
index 3ab9906c170c..000000000000
--- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/v1/GlueToTrinoConverter.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.trino.plugin.hive.metastore.glue.v1;
-
-import com.amazonaws.services.glue.model.SerDeInfo;
-import com.amazonaws.services.glue.model.StorageDescriptor;
-import com.amazonaws.services.glue.model.UserDefinedFunction;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import io.trino.hive.thrift.metastore.ResourceType;
-import io.trino.hive.thrift.metastore.ResourceUri;
-import io.trino.metastore.Column;
-import io.trino.metastore.Database;
-import io.trino.metastore.HiveBucketProperty;
-import io.trino.metastore.HiveType;
-import io.trino.metastore.Partition;
-import io.trino.metastore.SortingColumn;
-import io.trino.metastore.SortingColumn.Order;
-import io.trino.metastore.Storage;
-import io.trino.metastore.StorageFormat;
-import io.trino.metastore.Table;
-import io.trino.plugin.hive.HiveStorageFormat;
-import io.trino.spi.TrinoException;
-import io.trino.spi.connector.SchemaTableName;
-import io.trino.spi.function.LanguageFunction;
-import io.trino.spi.security.PrincipalType;
-import jakarta.annotation.Nullable;
-import org.gaul.modernizer_maven_annotations.SuppressModernizer;
-
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.UnaryOperator;
-
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static com.google.common.base.Strings.emptyToNull;
-import static com.google.common.base.Strings.nullToEmpty;
-import static io.trino.metastore.HiveType.HIVE_INT;
-import static io.trino.metastore.Table.TABLE_COMMENT;
-import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA;
-import static io.trino.plugin.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT;
-import static io.trino.plugin.hive.TableType.EXTERNAL_TABLE;
-import static io.trino.plugin.hive.ViewReaderUtil.isTrinoMaterializedView;
-import static io.trino.plugin.hive.metastore.glue.v1.Memoizers.memoizeLast;
-import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.decodeFunction;
-import static io.trino.plugin.hive.util.HiveUtil.isDeltaLakeTable;
-import static io.trino.plugin.hive.util.HiveUtil.isIcebergTable;
-import static java.lang.String.format;
-import static java.util.Objects.requireNonNull;
-
-public final class GlueToTrinoConverter
-{
- private static final String PUBLIC_OWNER = "PUBLIC";
-
- private GlueToTrinoConverter() {}
-
- @SuppressModernizer // Usage of `Table.getStorageDescriptor` is not allowed. Only this method can call that.
- public static Optional getStorageDescriptor(com.amazonaws.services.glue.model.Table glueTable)
- {
- return Optional.ofNullable(glueTable.getStorageDescriptor());
- }
-
- @SuppressModernizer // Usage of `Column.getParameters` is not allowed. Only this method can call that.
- public static Map getColumnParameters(com.amazonaws.services.glue.model.Column glueColumn)
- {
- return firstNonNull(glueColumn.getParameters(), ImmutableMap.of());
- }
-
- public static String getTableType(com.amazonaws.services.glue.model.Table glueTable)
- {
- // Athena treats missing table type as EXTERNAL_TABLE.
- return firstNonNull(getTableTypeNullable(glueTable), EXTERNAL_TABLE.name());
- }
-
- @Nullable
- @SuppressModernizer // Usage of `Table.getTableType` is not allowed. Only this method can call that.
- public static String getTableTypeNullable(com.amazonaws.services.glue.model.Table glueTable)
- {
- return glueTable.getTableType();
- }
-
- @SuppressModernizer // Usage of `Table.getParameters` is not allowed. Only this method can call that.
- public static Map getTableParameters(com.amazonaws.services.glue.model.Table glueTable)
- {
- return firstNonNull(glueTable.getParameters(), ImmutableMap.of());
- }
-
- @SuppressModernizer // Usage of `Partition.getParameters` is not allowed. Only this method can call that.
- public static Map getPartitionParameters(com.amazonaws.services.glue.model.Partition gluePartition)
- {
- return firstNonNull(gluePartition.getParameters(), ImmutableMap.of());
- }
-
- @SuppressModernizer // Usage of `SerDeInfo.getParameters` is not allowed. Only this method can call that.
- public static Map getSerDeInfoParameters(com.amazonaws.services.glue.model.SerDeInfo glueSerDeInfo)
- {
- return firstNonNull(glueSerDeInfo.getParameters(), ImmutableMap.of());
- }
-
- public static Database convertDatabase(com.amazonaws.services.glue.model.Database glueDb)
- {
- return Database.builder()
- .setDatabaseName(glueDb.getName())
- // Currently it's not possible to create a Glue database with empty location string ""
- // (validation error detected: Value '' at 'database.locationUri' failed to satisfy constraint: Member must have length greater than or equal to 1)
- // However, it has been observed that Glue databases with empty location do exist in the wild.
- .setLocation(Optional.ofNullable(emptyToNull(glueDb.getLocationUri())))
- .setComment(Optional.ofNullable(glueDb.getDescription()))
- .setParameters(firstNonNull(glueDb.getParameters(), ImmutableMap.of()))
- .setOwnerName(Optional.of(PUBLIC_OWNER))
- .setOwnerType(Optional.of(PrincipalType.ROLE))
- .build();
- }
-
- public static Table convertTable(com.amazonaws.services.glue.model.Table glueTable, String dbName)
- {
- SchemaTableName table = new SchemaTableName(dbName, glueTable.getName());
-
- String tableType = getTableType(glueTable);
-
- ImmutableMap.Builder parameters = ImmutableMap.builder();
- Optional description = Optional.ofNullable(glueTable.getDescription());
- description.ifPresent(comment -> parameters.put(TABLE_COMMENT, comment));
- getTableParameters(glueTable).entrySet().stream()
- // If the description was set we may have two "comment"s, prefer the description field
- .filter(entry -> description.isEmpty() || !entry.getKey().equals(TABLE_COMMENT))
- .forEach(parameters::put);
- Map tableParameters = parameters.buildOrThrow();
-
- Table.Builder tableBuilder = Table.builder()
- .setDatabaseName(table.getSchemaName())
- .setTableName(table.getTableName())
- .setOwner(Optional.ofNullable(glueTable.getOwner()))
- .setTableType(tableType)
- .setParameters(tableParameters)
- .setViewOriginalText(Optional.ofNullable(glueTable.getViewOriginalText()))
- .setViewExpandedText(Optional.ofNullable(glueTable.getViewExpandedText()));
-
- Optional storageDescriptor = getStorageDescriptor(glueTable);
-
- if (isIcebergTable(tableParameters) ||
- (storageDescriptor.isEmpty() && isTrinoMaterializedView(tableType, tableParameters))) {
- // Iceberg tables do not need to read the StorageDescriptor field, but we still need to return dummy properties for compatibility
- // Materialized views do not need to read the StorageDescriptor, but we still need to return dummy properties for compatibility
- tableBuilder.setDataColumns(ImmutableList.of(new Column("dummy", HIVE_INT, Optional.empty(), ImmutableMap.of())));
- tableBuilder.getStorageBuilder().setStorageFormat(HiveStorageFormat.PARQUET.toStorageFormat());
- }
- else if (isDeltaLakeTable(tableParameters)) {
- tableBuilder.setDataColumns(ImmutableList.of(new Column("dummy", HIVE_INT, Optional.empty(), ImmutableMap.of())));
- tableBuilder.setPartitionColumns(ImmutableList.of());
- if (storageDescriptor.isEmpty()) {
- tableBuilder.getStorageBuilder().setStorageFormat(HiveStorageFormat.PARQUET.toStorageFormat());
- }
- else {
- StorageDescriptor sd = storageDescriptor.get();
- if (sd.getSerdeInfo() == null) {
- throw new TrinoException(HIVE_UNSUPPORTED_FORMAT, "Table SerdeInfo is null for table '%s' %s".formatted(table, glueTable));
- }
- new StorageConverter().setStorageBuilder(sd, tableBuilder.getStorageBuilder());
- }
- }
- else {
- if (storageDescriptor.isEmpty()) {
- throw new TrinoException(HIVE_UNSUPPORTED_FORMAT, "Table StorageDescriptor is null for table '%s' %s".formatted(table, glueTable));
- }
- StorageDescriptor sd = storageDescriptor.get();
- if (sd.getSerdeInfo() == null) {
- throw new TrinoException(HIVE_UNSUPPORTED_FORMAT, "Table SerdeInfo is null for table '%s' %s".formatted(table, glueTable));
- }
- boolean isCsv = HiveStorageFormat.CSV.getSerde().equals(sd.getSerdeInfo().getSerializationLibrary());
- tableBuilder.setDataColumns(convertColumns(table, sd.getColumns(), ColumnType.DATA, isCsv));
- if (glueTable.getPartitionKeys() != null) {
- tableBuilder.setPartitionColumns(convertColumns(table, glueTable.getPartitionKeys(), ColumnType.PARTITION, isCsv));
- }
- else {
- tableBuilder.setPartitionColumns(ImmutableList.of());
- }
- // No benefit to memoizing here, just reusing the implementation
- new StorageConverter().setStorageBuilder(sd, tableBuilder.getStorageBuilder());
- }
-
- return tableBuilder.build();
- }
-
- private static Column convertColumn(SchemaTableName table, com.amazonaws.services.glue.model.Column glueColumn, ColumnType columnType, boolean isCsv)
- {
- // OpenCSVSerde deserializes columns from csv file into strings, so we set the column type from the metastore
- // to string to avoid cast exceptions.
- if (columnType == ColumnType.DATA && isCsv) {
- //TODO(https://github.com/trinodb/trino/issues/7240) Add tests
- return new Column(glueColumn.getName(), HiveType.HIVE_STRING, Optional.ofNullable(glueColumn.getComment()), getColumnParameters(glueColumn));
- }
- return new Column(glueColumn.getName(), convertType(table, glueColumn), Optional.ofNullable(glueColumn.getComment()), getColumnParameters(glueColumn));
- }
-
- private static HiveType convertType(SchemaTableName table, com.amazonaws.services.glue.model.Column column)
- {
- try {
- return HiveType.valueOf(column.getType().toLowerCase(Locale.ENGLISH));
- }
- catch (IllegalArgumentException e) {
- throw new TrinoException(HIVE_INVALID_METADATA, "Glue table '%s' column '%s' has invalid data type: %s".formatted(table, column.getName(), column.getType()), e);
- }
- }
-
- private static List convertColumns(SchemaTableName table, List glueColumns, ColumnType columnType, boolean isCsv)
- {
- return mappedCopy(glueColumns, glueColumn -> convertColumn(table, glueColumn, columnType, isCsv));
- }
-
- private static Function, Map> parametersConverter()
- {
- return memoizeLast(ImmutableMap::copyOf);
- }
-
- private static boolean isNullOrEmpty(List list)
- {
- return list == null || list.isEmpty();
- }
-
- public static final class GluePartitionConverter
- implements Function
- {
- private final BiFunction, Boolean, List> dataColumnsConverter;
- private final Function, Map> parametersConverter = parametersConverter();
- private final StorageConverter storageConverter = new StorageConverter();
- private final String databaseName;
- private final String tableName;
-
- public GluePartitionConverter(String databaseName, String tableName)
- {
- this.databaseName = requireNonNull(databaseName, "databaseName is null");
- this.tableName = requireNonNull(tableName, "tableName is null");
- this.dataColumnsConverter = memoizeLast((glueColumns, isCsv) -> convertColumns(new SchemaTableName(databaseName, tableName), glueColumns, ColumnType.DATA, isCsv));
- }
-
- @Override
- public Partition apply(com.amazonaws.services.glue.model.Partition gluePartition)
- {
- requireNonNull(gluePartition.getStorageDescriptor(), "Partition StorageDescriptor is null");
- StorageDescriptor sd = gluePartition.getStorageDescriptor();
-
- if (!databaseName.equals(gluePartition.getDatabaseName())) {
- throw new IllegalArgumentException(format("Unexpected databaseName, expected: %s, but found: %s", databaseName, gluePartition.getDatabaseName()));
- }
- if (!tableName.equals(gluePartition.getTableName())) {
- throw new IllegalArgumentException(format("Unexpected tableName, expected: %s, but found: %s", tableName, gluePartition.getTableName()));
- }
- boolean isCsv = sd.getSerdeInfo() != null && HiveStorageFormat.CSV.getSerde().equals(sd.getSerdeInfo().getSerializationLibrary());
- Partition.Builder partitionBuilder = Partition.builder()
- .setDatabaseName(databaseName)
- .setTableName(tableName)
- .setValues(gluePartition.getValues()) // No memoization benefit
- .setColumns(dataColumnsConverter.apply(sd.getColumns(), isCsv))
- .setParameters(parametersConverter.apply(getPartitionParameters(gluePartition)));
-
- storageConverter.setStorageBuilder(sd, partitionBuilder.getStorageBuilder());
-
- return partitionBuilder.build();
- }
- }
-
- private static final class StorageConverter
- {
- private final Function, List> bucketColumns = memoizeLast(ImmutableList::copyOf);
- private final Function, List> sortColumns = memoizeLast(StorageConverter::createSortingColumns);
- private final UnaryOperator> bucketProperty = memoizeLast();
- private final Function, Map> serdeParametersConverter = parametersConverter();
- private final StorageFormatConverter storageFormatConverter = new StorageFormatConverter();
-
- public void setStorageBuilder(StorageDescriptor sd, Storage.Builder storageBuilder)
- {
- requireNonNull(sd.getSerdeInfo(), "StorageDescriptor SerDeInfo is null");
- SerDeInfo serdeInfo = sd.getSerdeInfo();
-
- storageBuilder.setStorageFormat(storageFormatConverter.createStorageFormat(serdeInfo, sd))
- .setLocation(nullToEmpty(sd.getLocation()))
- .setBucketProperty(convertToBucketProperty(sd))
- .setSkewed(sd.getSkewedInfo() != null && !isNullOrEmpty(sd.getSkewedInfo().getSkewedColumnNames()))
- .setSerdeParameters(serdeParametersConverter.apply(getSerDeInfoParameters(serdeInfo)))
- .build();
- }
-
- private Optional convertToBucketProperty(StorageDescriptor sd)
- {
- if (sd.getNumberOfBuckets() > 0) {
- if (isNullOrEmpty(sd.getBucketColumns())) {
- throw new TrinoException(HIVE_INVALID_METADATA, "Table/partition metadata has 'numBuckets' set, but 'bucketCols' is not set");
- }
- List bucketColumns = this.bucketColumns.apply(sd.getBucketColumns());
- List sortedBy = this.sortColumns.apply(sd.getSortColumns());
- return bucketProperty.apply(Optional.of(new HiveBucketProperty(bucketColumns, sd.getNumberOfBuckets(), sortedBy)));
- }
- return Optional.empty();
- }
-
- private static List createSortingColumns(List sortColumns)
- {
- if (isNullOrEmpty(sortColumns)) {
- return ImmutableList.of();
- }
- return mappedCopy(sortColumns, column -> new SortingColumn(column.getColumn(), Order.fromMetastoreApiOrder(column.getSortOrder(), "unknown")));
- }
- }
-
- private static final class StorageFormatConverter
- {
- private static final StorageFormat ALL_NULLS = StorageFormat.createNullable(null, null, null);
- private final UnaryOperator serializationLib = memoizeLast();
- private final UnaryOperator inputFormat = memoizeLast();
- private final UnaryOperator outputFormat = memoizeLast();
- // Second phase to attempt memoization on the entire instance beyond just the fields
- private final UnaryOperator storageFormat = memoizeLast();
-
- public StorageFormat createStorageFormat(SerDeInfo serdeInfo, StorageDescriptor storageDescriptor)
- {
- String serializationLib = this.serializationLib.apply(serdeInfo.getSerializationLibrary());
- String inputFormat = this.inputFormat.apply(storageDescriptor.getInputFormat());
- String outputFormat = this.outputFormat.apply(storageDescriptor.getOutputFormat());
- if (serializationLib == null && inputFormat == null && outputFormat == null) {
- return ALL_NULLS;
- }
- return this.storageFormat.apply(StorageFormat.createNullable(serializationLib, inputFormat, outputFormat));
- }
- }
-
- public static LanguageFunction convertFunction(UserDefinedFunction function)
- {
- List uris = mappedCopy(function.getResourceUris(), uri -> new ResourceUri(ResourceType.FILE, uri.getUri()));
-
- LanguageFunction result = decodeFunction(function.getFunctionName(), uris);
-
- return new LanguageFunction(
- result.signatureToken(),
- result.sql(),
- result.path(),
- Optional.ofNullable(function.getOwnerName()));
- }
-
- public static List mappedCopy(List list, Function mapper)
- {
- requireNonNull(list, "list is null");
- requireNonNull(mapper, "mapper is null");
- // Uses a pre-sized builder to avoid intermediate allocations and copies, which is especially significant when the
- // number of elements is large and the size of the resulting list can be known in advance
- ImmutableList.Builder