Refresh and aws secrets

Makefile

@@ -5,7 +5,7 @@ EXT_NAME=quack
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
 # Build HTTPFS for testing
-CORE_EXTENSIONS='httpfs'
+CORE_EXTENSIONS=''
 
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile

extension_config.cmake

@@ -4,4 +4,10 @@
 duckdb_extension_load(aws
     SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
     LOAD_TESTS
+)
+
+duckdb_extension_load(httpfs
+        GIT_URL https://github.com/duckdb/duckdb-httpfs
+        GIT_TAG main
+        INCLUDE_DIR extension/httpfs/include
 )

src/aws_secret.cpp

@@ -89,12 +89,13 @@ static string TryGetStringParam(CreateSecretInput &input, const string &param_na
 //! This is the actual callback function
 static unique_ptr<BaseSecret> CreateAWSSecretFromCredentialChain(ClientContext &context, CreateSecretInput &input) {
 	Aws::Auth::AWSCredentials credentials;
+	string chain;
 
 	string profile = TryGetStringParam(input, "profile");
 	string assume_role = TryGetStringParam(input, "assume_role_arn");
 
 	if (input.options.find("chain") != input.options.end()) {
-		string chain = TryGetStringParam(input, "chain");
+		chain = TryGetStringParam(input, "chain");
 
 		DuckDBCustomAWSCredentialsProviderChain provider(chain, profile, assume_role);
 		credentials = provider.GetAWSCredentials();
@@ -126,6 +127,8 @@ static unique_ptr<BaseSecret> CreateAWSSecretFromCredentialChain(ClientContext &
 		} else if (input.type == "gcs") {
 			scope.push_back("gcs://");
 			scope.push_back("gs://");
+		} else if (input.type == "aws") {
+			scope.push_back("");
 		} else {
 			throw InternalException("Unknown secret type found in aws extension: '%s'", input.type);
 		}
@@ -137,6 +140,24 @@ static unique_ptr<BaseSecret> CreateAWSSecretFromCredentialChain(ClientContext &
 		result->secret_map["region"] = region;
 	}
 
+	// Only auto is supported
+	string refresh = TryGetStringParam(input, "refresh");
+
+	// We have sneaked in this special handling where if you set the STS chain, you automatically enable refresh
+	// TODO: remove this once refresh is set to auto by default for all credential_chain provider created secrets.
+	if (chain == "sts" && refresh.empty()) {
+		refresh = "auto";
+	}
+
+	if (refresh == "auto") {
+		child_list_t<Value> struct_fields;
+		for (const auto &named_param : input.options) {
+			auto lower_name = StringUtil::Lower(named_param.first);
+			struct_fields.push_back({lower_name, named_param.second});
+		}
+		result->secret_map["refresh_info"] = Value::STRUCT(struct_fields);
+	}
+
 	AwsSetCredentialsResult ret;
 	if (!credentials.IsExpiredOrEmpty()) {
 		result->secret_map["key_id"] = Value(credentials.GetAWSAccessKeyId());
@@ -157,6 +178,8 @@ static unique_ptr<BaseSecret> CreateAWSSecretFromCredentialChain(ClientContext &
 			}
 		} else if (input.type == "gcs") {
 			result->secret_map["endpoint"] = "storage.googleapis.com";
+		} else if (input.type == "aws") {
+			// this is a nop?
 		} else {
 			throw InternalException("Unknown secret type found in httpfs extension: '%s'", input.type);
 		}
@@ -174,7 +197,7 @@ static unique_ptr<BaseSecret> CreateAWSSecretFromCredentialChain(ClientContext &
 }
 
 void CreateAwsSecretFunctions::Register(DatabaseInstance &instance) {
-	vector<string> types = {"s3", "r2", "gcs"};
+	vector<string> types = {"s3", "r2", "gcs", "aws"};
 
 	for (const auto &type : types) {
 		// Register the credential_chain secret provider
@@ -192,7 +215,7 @@ void CreateAwsSecretFunctions::Register(DatabaseInstance &instance) {
 
 		cred_chain_function.named_parameters["assume_role_arn"] = LogicalType::VARCHAR;
 
-		cred_chain_function.named_parameters["refresh"] = LogicalType::BOOLEAN;
+		cred_chain_function.named_parameters["refresh"] = LogicalType::VARCHAR;
 
 		if (type == "r2") {
 			cred_chain_function.named_parameters["account_id"] = LogicalType::VARCHAR;

test/sql/aws_secret_aws.test

@@ -0,0 +1,32 @@
+# name: test/sql/aws_secret_aws.test
+# description: test aws extension aws r2 secret
+# group: [aws]
+
+require aws
+
+require httpfs
+
+# Note this test is not very intelligent since we dont assume any profiles to be available
+
+statement ok
+CREATE SECRET s1 (
+    TYPE AWS,
+    PROVIDER credential_chain
+);
+
+query I
+SELECT name FROM which_secret('s3://haha/hoehoe.parkoe', 'aws')
+----
+s1
+
+statement ok
+set s3_endpoint='localhost:12345';
+
+# Ensures this test is independent of the DUCKDB_S3_USE_SSL env variable
+statement ok
+set s3_use_ssl=true;
+
+statement error
+from "s3://blabla/file.csv"
+----
+IO Error: Could not establish connection error for HTTP HEAD to 'https://blabla.localhost:12345/file.csv' with status

test/sql/env/aws_secret_refresh.test

@@ -0,0 +1,38 @@
+# name: test/sql/env/aws_secret_refresh.test
+# description: Tests secret refreshing
+# group: [secrets]
+
+require aws
+
+require httpfs
+
+require-env DUCKDB_AWS_TESTING_ENV_AVAILABLE
+
+require-env AWS_ACCESS_KEY_ID
+
+require-env AWS_SECRET_ACCESS_KEY
+
+require-env AWS_DEFAULT_REGION
+
+
+# Let's try explicitly passing in the config provider
+statement ok
+CREATE SECRET env_test (
+    TYPE S3,
+    PROVIDER credential_chain,
+    REGION 'eu-west-2',
+    REFRESH 'auto'
+);
+
+statement ok
+SET enable_logging=true
+
+statement error
+FROM "s3://test-bucket-ceiveran/lololasdklasdjk.csv";
+----
+
+# Secret refresh has been triggered
+query II
+SELECT log_level, message FROM duckdb_logs WHERE type='httpfs.SecretRefresh'
+----
+INFO	<REGEX>:Successfully refreshed secret: env_test, new key_id:.*