Merge pull request backstage#16519 from JosiahCraw/feature/add-proxy-support-for-techdocs-awsS3

feat: add support for HTTPS proxy for AWS S3 requests in Techdocs

Author: agentbellnorm

.changeset/wicked-lions-repeat.md

@@ -0,0 +1,6 @@
+---
+'@techdocs/cli': minor
+'@backstage/plugin-techdocs-node': minor
+---
+
+Added support for an HTTPS proxy for techdocs AWS S3 requests

docs/features/techdocs/cli.md

@@ -176,29 +176,28 @@ Usage: techdocs-cli publish [options]
 Publish generated TechDocs site to an external storage AWS S3, Google GCS, etc.
 
 Options:
-  --publisher-type <TYPE>                  (Required always) awsS3 | googleGcs | azureBlobStorage
-                                           - same as techdocs.publisher.type in Backstage
-                                           app-config.yaml
-  --storage-name <BUCKET/CONTAINER NAME>   (Required always) In case of AWS/GCS, use the bucket
-                                           name. In case of Azure, use container name. Same as
-                                           techdocs.publisher.[TYPE].bucketName
-  --entity <NAMESPACE/KIND/NAME>           (Required always) Entity uid separated by / in
-                                           namespace/kind/name order (case-sensitive). Example:
-                                           default/Component/myEntity
-  --legacyUseCaseSensitiveTripletPaths     Publishes objects with cased entity triplet prefix when set (e.g. namespace/Kind/name).
-                                           Only use if your TechDocs backend is configured the same way
-  --azureAccountName <AZURE ACCOUNT NAME>  (Required for Azure) specify when --publisher-type
-                                           azureBlobStorage
-  --azureAccountKey <AZURE ACCOUNT KEY>    Azure Storage Account key to use for authentication.
-                                           If not specified, you must set AZURE_TENANT_ID,
-                                           AZURE_CLIENT_ID & AZURE_CLIENT_SECRET as environment
-                                           variables.
-  --awsRoleArn <AWS ROLE ARN>              Optional AWS ARN of role to be assumed.
-  --awsEndpoint <AWS ENDPOINT>             Optional AWS endpoint to send requests to.
-  --awsS3ForcePathStyle                    Optional AWS S3 option to force path style.
-  --directory <PATH>                       Path of the directory containing generated files to
-                                           publish (default: "./site/")
-  -h, --help                               display help for command
+  --publisher-type <TYPE>                                       (Required always) awsS3 | googleGcs | azureBlobStorage | openStackSwift - same as techdocs.publisher.type in Backstage app-config.yaml
+  --storage-name <BUCKET/CONTAINER NAME>                        (Required always) In case of AWS/GCS, use the bucket name. In case of Azure, use container name. Same as
+                                                                techdocs.publisher.[TYPE].bucketName
+  --entity <NAMESPACE/KIND/NAME>                                (Required always) Entity uid separated by / in namespace/kind/name order (case-sensitive). Example: default/Component/myEntity
+  --legacyUseCaseSensitiveTripletPaths                          Publishes objects with cased entity triplet prefix when set (e.g. namespace/Kind/name). Only use if your TechDocs backend is configured
+                                                                the same way. (default: false)
+  --azureAccountName <AZURE ACCOUNT NAME>                       (Required for Azure) specify when --publisher-type azureBlobStorage
+  --azureAccountKey <AZURE ACCOUNT KEY>                         Azure Storage Account key to use for authentication. If not specified, you must set AZURE_TENANT_ID, AZURE_CLIENT_ID &
+                                                                AZURE_CLIENT_SECRET as environment variables.
+  --awsRoleArn <AWS ROLE ARN>                                   Optional AWS ARN of role to be assumed.
+  --awsEndpoint <AWS ENDPOINT>                                  Optional AWS endpoint to send requests to.
+  --awsProxy <HTTPS Proxy>                                      Optional Proxy to use for AWS requests.
+  --awsS3sse <AWS SSE>                                          Optional AWS S3 Server Side Encryption.
+  --awsS3ForcePathStyle                                         Optional AWS S3 option to force path style.
+  --awsBucketRootPath <AWS BUCKET ROOT PATH>                    Optional sub-directory to store files in Amazon S3
+  --osCredentialId <OPENSTACK SWIFT APPLICATION CREDENTIAL ID>  (Required for OpenStack) specify when --publisher-type openStackSwift
+  --osSecret <OPENSTACK SWIFT APPLICATION CREDENTIAL SECRET>    (Required for OpenStack) specify when --publisher-type openStackSwift
+  --osAuthUrl <OPENSTACK SWIFT AUTHURL>                         (Required for OpenStack) specify when --publisher-type openStackSwift
+  --osSwiftUrl <OPENSTACK SWIFT SWIFTURL>                       (Required for OpenStack) specify when --publisher-type openStackSwift
+  --gcsBucketRootPath <GCS BUCKET ROOT PATH>                    Optional sub-directory to store files in Google cloud storage
+  --directory <PATH>                                            Path of the directory containing generated files to publish (default: "./site/")
+  -h, --help                                                    display help for command
 ```
 
 ### Migrate content for case-insensitive access

docs/features/techdocs/configuration.md

@@ -136,6 +136,11 @@ techdocs:
       # https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html#endpoint
       endpoint: ${AWS_ENDPOINT}
 
+      # (Optional) HTTPS proxy to use for S3 Requests
+      # Defaults to using no proxy
+      # This allows docs to be published and read from behind a proxy
+      httpsProxy: ${HTTPS_PROXY}
+
       # (Optional) Whether to use path style URLs when communicating with S3.
       # Defaults to false.
       # This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used to host tech docs.

packages/techdocs-cli/cli-report.md

@@ -77,6 +77,7 @@ Options:
   --azureAccountKey <AZURE ACCOUNT KEY>
   --awsRoleArn <AWS ROLE ARN>
   --awsEndpoint <AWS ENDPOINT>
+  --awsProxy <HTTPS Proxy>
   --awsS3sse <AWS SSE>
   --awsS3ForcePathStyle
   --awsBucketRootPath <AWS BUCKET ROOT PATH>

packages/techdocs-cli/src/commands/index.ts

@@ -173,6 +173,10 @@ export function registerCommands(program: Command) {
       '--awsEndpoint <AWS ENDPOINT>',
       'Optional AWS endpoint to send requests to.',
     )
+    .option(
+      '--awsProxy <HTTPS Proxy>',
+      'Optional Proxy to use for AWS requests.',
+    )
     .option('--awsS3sse <AWS SSE>', 'Optional AWS S3 Server Side Encryption.')
     .option(
       '--awsS3ForcePathStyle',

packages/techdocs-cli/src/lib/PublisherConfig.ts

@@ -94,6 +94,7 @@ export class PublisherConfig {
         ...(opts.awsEndpoint && { endpoint: opts.awsEndpoint }),
         ...(opts.awsS3ForcePathStyle && { s3ForcePathStyle: true }),
         ...(opts.awsS3sse && { sse: opts.awsS3sse }),
+        ...(opts.awsProxy && { httpsProxy: opts.awsProxy }),
       },
     };
   }

plugins/techdocs-node/package.json

@@ -42,6 +42,7 @@
     "@aws-sdk/client-s3": "^3.208.0",
     "@aws-sdk/credential-providers": "^3.208.0",
     "@aws-sdk/lib-storage": "^3.208.0",
+    "@aws-sdk/node-http-handler": "^3.208.0",
     "@aws-sdk/types": "^3.208.0",
     "@azure/identity": "^2.1.0",
     "@azure/storage-blob": "^12.5.0",
@@ -58,6 +59,7 @@
     "express": "^4.17.1",
     "fs-extra": "10.1.0",
     "git-url-parse": "^13.0.0",
+    "hpagent": "^1.2.0",
     "js-yaml": "^4.0.0",
     "json5": "^2.1.3",
     "mime-types": "^2.1.27",

plugins/techdocs-node/src/stages/publish/awsS3.ts

@@ -32,8 +32,10 @@ import {
   S3Client,
 } from '@aws-sdk/client-s3';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
+import { NodeHttpHandler } from '@aws-sdk/node-http-handler';
 import { Upload } from '@aws-sdk/lib-storage';
 import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
+import { HttpsProxyAgent } from 'hpagent';
 import express from 'express';
 import fs from 'fs-extra';
 import JSON5 from 'json5';
@@ -150,6 +152,11 @@ export class AwsS3Publish implements PublisherBase {
       'techdocs.publisher.awsS3.endpoint',
     );
 
+    // AWS HTTPS proxy is an optional config. If missing, no proxy is used
+    const httpsProxy = config.getOptionalString(
+      'techdocs.publisher.awsS3.httpsProxy',
+    );
+
     // AWS forcePathStyle is an optional config. If missing, it defaults to false. Needs to be enabled for cases
     // where endpoint url points to locally hosted S3 compatible storage like Localstack
     const forcePathStyle = config.getOptionalBoolean(
@@ -162,6 +169,11 @@ export class AwsS3Publish implements PublisherBase {
       ...(region && { region }),
       ...(endpoint && { endpoint }),
       ...(forcePathStyle && { forcePathStyle }),
+      ...(httpsProxy && {
+        requestHandler: new NodeHttpHandler({
+          httpsAgent: new HttpsProxyAgent({ proxy: httpsProxy }),
+        }),
+      }),
     });
 
     const legacyPathCasing =

yarn.lock

@@ -1211,7 +1211,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@aws-sdk/node-http-handler@npm:3.272.0":
+"@aws-sdk/node-http-handler@npm:3.272.0, @aws-sdk/node-http-handler@npm:^3.208.0":
   version: 3.272.0
   resolution: "@aws-sdk/node-http-handler@npm:3.272.0"
   dependencies:
@@ -8517,6 +8517,7 @@ __metadata:
     "@aws-sdk/client-s3": ^3.208.0
     "@aws-sdk/credential-providers": ^3.208.0
     "@aws-sdk/lib-storage": ^3.208.0
+    "@aws-sdk/node-http-handler": ^3.208.0
     "@aws-sdk/types": ^3.208.0
     "@azure/identity": ^2.1.0
     "@azure/storage-blob": ^12.5.0
@@ -8541,6 +8542,7 @@ __metadata:
     express: ^4.17.1
     fs-extra: 10.1.0
     git-url-parse: ^13.0.0
+    hpagent: ^1.2.0
     js-yaml: ^4.0.0
     json5: ^2.1.3
     mime-types: ^2.1.27