Real redirects support

Directory.Packages.props

@@ -13,6 +13,8 @@
     <PackageVersion Include="Amazon.Lambda.S3Events" Version="3.1.0" />
     <PackageVersion Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4" />
     <PackageVersion Include="Amazon.Lambda.SQSEvents" Version="2.2.0" />
+    <PackageVersion Include="AWSSDK.CloudFront" Version="4.0.0.10" />
+    <PackageVersion Include="AWSSDK.CloudFrontKeyValueStore" Version="4.0.0.9" />
     <PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
@@ -70,4 +72,4 @@
     </PackageVersion>
     <PackageVersion Include="xunit.v3" Version="2.0.2" />
   </ItemGroup>
-</Project>
+</Project>

src/Elastic.Documentation/Serialization/SourceGenerationContext.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Collections.Frozen;
 using System.Text.Json.Serialization;
 using Elastic.Documentation.Links;
 using Elastic.Documentation.Search;
@@ -18,4 +19,5 @@ namespace Elastic.Documentation.Serialization;
 [JsonSerializable(typeof(LinkRegistry))]
 [JsonSerializable(typeof(LinkRegistryEntry))]
 [JsonSerializable(typeof(DocumentationDocument))]
+[JsonSerializable(typeof(FrozenDictionary<string, string>))]
 public sealed partial class SourceGenerationContext : JsonSerializerContext;

src/tooling/docs-assembler/Building/AssemblerBuilder.cs

@@ -3,10 +3,12 @@
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Frozen;
+using System.Text.Json;
 using Documentation.Assembler.Exporters;
 using Documentation.Assembler.Navigation;
 using Elastic.Documentation.Legacy;
 using Elastic.Documentation.Links;
+using Elastic.Documentation.Serialization;
 using Elastic.Markdown;
 using Elastic.Markdown.Exporters;
 using Elastic.Markdown.Links.CrossLinks;
@@ -85,6 +87,8 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
 			}
 		}
 
+		await OutputRedirectsAsync(redirects, ctx);
+
 		tasks = markdownExporters.Select(async e => await e.StopAsync(ctx));
 		await Task.WhenAll(tasks);
 	}
@@ -143,4 +147,16 @@ private void SetFeatureFlags(AssemblerDocumentationSet set)
 			set.DocumentationSet.Configuration.Features.Set(configurationFeatureFlag.Key, configurationFeatureFlag.Value);
 		}
 	}
+
+	private async Task OutputRedirectsAsync(Dictionary<string, string> redirects, Cancel ctx)
+	{
+		var uniqueRedirects = redirects
+			.Where(x => !x.Key.TrimEnd('/').Equals(x.Value.TrimEnd('/')))
+			.ToFrozenDictionary();
+		var redirectsFile = context.WriteFileSystem.FileInfo.New(Path.Combine(context.OutputDirectory.FullName, "redirects.json"));
+		_logger.LogInformation("Writing {Count} resolved redirects to {Path}", uniqueRedirects.Count, redirectsFile.FullName);
+
+		var redirectsJson = JsonSerializer.Serialize(uniqueRedirects, SourceGenerationContext.Default.FrozenDictionaryStringString);
+		await context.WriteFileSystem.File.WriteAllTextAsync(redirectsFile.FullName, redirectsJson, ctx);
+	}
 }

src/tooling/docs-assembler/Cli/DeployCommands.cs

@@ -4,13 +4,20 @@
 
 using System.Diagnostics.CodeAnalysis;
 using System.IO.Abstractions;
+using System.Text.Json;
 using Actions.Core.Services;
+using Amazon.CloudFront;
+using Amazon.CloudFrontKeyValueStore;
+using Amazon.CloudFrontKeyValueStore.Model;
 using Amazon.S3;
 using Amazon.S3.Transfer;
 using ConsoleAppFramework;
 using Documentation.Assembler.Deploying;
+using Elastic.Documentation.Serialization;
 using Elastic.Documentation.Tooling.Diagnostics.Console;
+using Elastic.Documentation.Tooling.Filters;
 using Microsoft.Extensions.Logging;
+using DescribeKeyValueStoreRequest = Amazon.CloudFront.Model.DescribeKeyValueStoreRequest;
 
 namespace Documentation.Assembler.Cli;
 
@@ -94,4 +101,112 @@ public async Task<int> Apply(
 		await collector.StopAsync(ctx);
 		return collector.Errors;
 	}
+
+	/// <summary>Refreshes the redirects mapping in Cloudfront's KeyValueStore</summary>
+	/// <param name="environment">The environment to build</param>
+	/// <param name="redirectsFile">Path to the redirects mapping pre-generated by docs-assembler</param>
+	/// <param name="ctx"></param>
+	[Command("update-redirects")]
+	[ConsoleAppFilter<StopwatchFilter>]
+	[ConsoleAppFilter<CatchExceptionFilter>]
+	public async Task<int> UpdateRedirects(
+		string environment,
+		string redirectsFile = ".artifacts/assembly/redirects.json",
+		Cancel ctx = default)
+	{
+		AssignOutputLogger();
+		await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService)
+		{
+			NoHints = true
+		}.StartAsync(ctx);
+
+		if (!File.Exists(redirectsFile))
+		{
+			collector.EmitError(redirectsFile, "Redirects mapping does not exist.");
+			await collector.StopAsync(ctx);
+			return collector.Errors;
+		}
+
+		ConsoleApp.Log("Parsing redirects mapping");
+		var jsonContent = await File.ReadAllTextAsync(redirectsFile, ctx);
+		var sourcedRedirects = JsonSerializer.Deserialize(jsonContent, SourceGenerationContext.Default.FrozenDictionaryStringString);
+
+		if (sourcedRedirects is null)
+		{
+			collector.EmitError(redirectsFile, "Redirects mapping is invalid.");
+			await collector.StopAsync(ctx);
+			return collector.Errors;
+		}
+
+		var kvsName = $"elastic-docs-v3-{environment}-redirects-kvs";
+
+		var cfClient = new AmazonCloudFrontClient();
+		var kvsClient = new AmazonCloudFrontKeyValueStoreClient();
+
+		ConsoleApp.Log("Describing KVS");
+		var describeResponse = await cfClient.DescribeKeyValueStoreAsync(new DescribeKeyValueStoreRequest { Name = kvsName }, ctx);
+
+		var kvsArn = describeResponse.KeyValueStore.ARN;
+		var eTag = describeResponse.ETag;
+		var existingRedirects = new HashSet<string>();
+
+		var listKeysRequest = new ListKeysRequest { KvsARN = kvsArn };
+		ListKeysResponse listKeysResponse;
+
+		do
+		{
+			listKeysResponse = await kvsClient.ListKeysAsync(listKeysRequest, ctx);
+			foreach (var item in listKeysResponse.Items)
+				_ = existingRedirects.Add(item.Key);
+			listKeysRequest.NextToken = listKeysResponse.NextToken;
+		}
+		while (!string.IsNullOrEmpty(listKeysResponse.NextToken));
+
+		var toPut = sourcedRedirects
+			.Select(kvp => new PutKeyRequestListItem { Key = kvp.Key, Value = kvp.Value });
+		var toDelete = existingRedirects
+			.Except(sourcedRedirects.Keys)
+			.Select(k => new DeleteKeyRequestListItem { Key = k });
+
+		ConsoleApp.Log("Updating redirects in KVS");
+		const int batchSize = 500;
+
+		eTag = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toPut, batchSize, "Puts", ctx);
+		_ = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toDelete, batchSize, "Deletes", ctx);
+
+		await collector.StopAsync(ctx);
+		return collector.Errors;
+	}
+
+	private static async Task<string> ProcessBatchUpdatesAsync(
+		IAmazonCloudFrontKeyValueStore kvsClient,
+		string kvsArn,
+		string eTag,
+		IEnumerable<object> items,
+		int batchSize,
+		string operation,
+		Cancel ctx)
+	{
+		var enumerable = items.ToList();
+		for (var i = 0; i < enumerable.Count; i += batchSize)
+		{
+			var batch = enumerable.Skip(i).Take(batchSize);
+			var updateRequest = new UpdateKeysRequest
+			{
+				KvsARN = kvsArn,
+				IfMatch = eTag
+			};
+
+			if (operation.Equals("Puts", StringComparison.InvariantCulture))
+				updateRequest.Puts = batch.Cast<PutKeyRequestListItem>().ToList();
+			else if (operation.Equals("Deletes", StringComparison.InvariantCulture))
+				updateRequest.Deletes = batch.Cast<DeleteKeyRequestListItem>().ToList();
+
+			var update = await kvsClient.UpdateKeysAsync(updateRequest, ctx);
+			eTag = update.ETag;
+		}
+
+		return eTag;
+	}
+
 }

src/tooling/docs-assembler/Cli/RepositoryCommands.cs

@@ -136,6 +136,10 @@ public async Task<int> BuildAll(
 
 		await cloner.WriteLinkRegistrySnapshot(checkoutResult.LinkRegistrySnapshot, ctx);
 
+		var redirectsPath = Path.Combine(assembleContext.OutputDirectory.FullName, "redirects.json");
+		if (File.Exists(redirectsPath))
+			await githubActionsService.SetOutputAsync("redirects_artifact_path", redirectsPath);
+
 		var sitemapBuilder = new SitemapBuilder(navigation.NavigationItems, assembleContext.WriteFileSystem, assembleContext.OutputDirectory);
 		sitemapBuilder.Generate();
 

src/tooling/docs-assembler/docs-assembler.csproj

@@ -17,6 +17,8 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.CloudFront" />
+    <PackageReference Include="AWSSDK.CloudFrontKeyValueStore" />
     <PackageReference Include="AWSSDK.S3"/>
     <PackageReference Include="ConsoleAppFramework.Abstractions"/>
     <PackageReference Include="ConsoleAppFramework" />