Add support for system params when deploying extensions (#5414)

* Add support for configuring system params when deploying extensions

* Allow . and / in env keys

* Make partitionRecord more flexible

Author: joehan

src/commands/ext-configure.ts

@@ -78,7 +78,7 @@ export const command = new Command("ext:configure <extensionInstanceId>")
     });
 
     const [immutableParams, tbdParams] = partition(
-      spec.params,
+      spec.params.concat(spec.systemParams ?? []),
       (param) => param.immutable ?? false
     );
     infoImmutableParams(immutableParams, oldParamValues);

src/commands/ext-install.ts

@@ -212,7 +212,7 @@ async function installToManifest(options: InstallExtensionOptions): Promise<void
 
   const paramBindingOptions = await paramHelper.getParams({
     projectId,
-    paramSpecs: spec.params,
+    paramSpecs: spec.params.concat(spec.systemParams ?? []),
     nonInteractive,
     paramsEnvPath,
     instanceId,

src/deploy/extensions/planner.ts

@@ -10,9 +10,10 @@ import {
 } from "../../extensions/extensionsHelper";
 import { logger } from "../../logger";
 import { readInstanceParam } from "../../extensions/manifest";
-import { ParamBindingOptions } from "../../extensions/paramHelper";
+import { isSystemParam, ParamBindingOptions } from "../../extensions/paramHelper";
 import { readExtensionYaml, readPostinstall } from "../../extensions/emulator/specHelper";
 import { ExtensionVersion, Extension, ExtensionSpec } from "../../extensions/types";
+import { partitionRecord } from "../../functional";
 
 export interface InstanceSpec {
   instanceId: string;
@@ -46,6 +47,7 @@ export interface ManifestInstanceSpec extends InstanceSpec {
  */
 export interface DeploymentInstanceSpec extends InstanceSpec {
   params: Record<string, string>;
+  systemParams: Record<string, string>;
   allowedEventTypes?: string[];
   eventarcChannel?: string;
   etag?: string;
@@ -107,6 +109,7 @@ export async function have(projectId: string): Promise<DeploymentInstanceSpec[]>
     const dep: DeploymentInstanceSpec = {
       instanceId: i.name.split("/").pop()!,
       params: i.config.params,
+      systemParams: i.config.systemParams ?? {},
       allowedEventTypes: i.config.allowedEventTypes,
       eventarcChannel: i.config.eventarcChannel,
       etag: i.etag,
@@ -145,7 +148,7 @@ export async function want(args: {
     try {
       const instanceId = e[0];
 
-      const params = readInstanceParam({
+      const rawParams = readInstanceParam({
         projectDir: args.projectDir,
         instanceId,
         projectId: args.projectId,
@@ -154,26 +157,28 @@ export async function want(args: {
         checkLocal: args.emulatorMode,
       });
       const autoPopulatedParams = await getFirebaseProjectParams(args.projectId, args.emulatorMode);
-      const subbedParams = substituteParams(params, autoPopulatedParams);
+      const subbedParams = substituteParams(rawParams, autoPopulatedParams);
+      const [systemParams, params] = partitionRecord(subbedParams, isSystemParam);
 
       // ALLOWED_EVENT_TYPES can be undefined (user input not provided) or empty string (no events selected).
       // If empty string, we want to pass an empty array. If it's undefined we want to pass through undefined.
       const allowedEventTypes =
-        subbedParams.ALLOWED_EVENT_TYPES !== undefined
-          ? subbedParams.ALLOWED_EVENT_TYPES.split(",").filter((e) => e !== "")
+        params.ALLOWED_EVENT_TYPES !== undefined
+          ? params.ALLOWED_EVENT_TYPES.split(",").filter((e) => e !== "")
           : undefined;
-      const eventarcChannel = subbedParams.EVENTARC_CHANNEL;
+      const eventarcChannel = params.EVENTARC_CHANNEL;
 
       // Remove special params that are stored in the .env file but aren't actually params specified by the publisher.
       // Currently, only environment variables needed for Events features are considered special params stored in .env files.
-      delete subbedParams["EVENTARC_CHANNEL"];
-      delete subbedParams["ALLOWED_EVENT_TYPES"];
+      delete params["EVENTARC_CHANNEL"];
+      delete params["ALLOWED_EVENT_TYPES"];
 
       if (isLocalPath(e[1])) {
         instanceSpecs.push({
           instanceId,
           localPath: e[1],
-          params: subbedParams,
+          params,
+          systemParams,
           allowedEventTypes: allowedEventTypes,
           eventarcChannel: eventarcChannel,
         });
@@ -183,7 +188,8 @@ export async function want(args: {
         instanceSpecs.push({
           instanceId,
           ref,
-          params: subbedParams,
+          params,
+          systemParams,
           allowedEventTypes: allowedEventTypes,
           eventarcChannel: eventarcChannel,
         });

src/deploy/extensions/tasks.ts

@@ -49,6 +49,7 @@ export function createExtensionInstanceTask(
         projectId,
         instanceId: instanceSpec.instanceId,
         params: instanceSpec.params,
+        systemParams: instanceSpec.systemParams,
         extensionVersionRef: refs.toExtensionVersionRef(instanceSpec.ref),
         allowedEventTypes: instanceSpec.allowedEventTypes,
         eventarcChannel: instanceSpec.eventarcChannel,
@@ -60,6 +61,7 @@ export function createExtensionInstanceTask(
         projectId,
         instanceId: instanceSpec.instanceId,
         params: instanceSpec.params,
+        systemParams: instanceSpec.systemParams,
         extensionSource,
         allowedEventTypes: instanceSpec.allowedEventTypes,
         eventarcChannel: instanceSpec.eventarcChannel,
@@ -92,6 +94,7 @@ export function updateExtensionInstanceTask(
         instanceId: instanceSpec.instanceId,
         extRef: refs.toExtensionVersionRef(instanceSpec.ref!),
         params: instanceSpec.params,
+        systemParams: instanceSpec.systemParams,
         canEmitEvents: !!instanceSpec.allowedEventTypes,
         allowedEventTypes: instanceSpec.allowedEventTypes,
         eventarcChannel: instanceSpec.eventarcChannel,
@@ -104,6 +107,8 @@ export function updateExtensionInstanceTask(
         instanceId: instanceSpec.instanceId,
         extensionSource,
         validateOnly,
+        params: instanceSpec.params,
+        systemParams: instanceSpec.systemParams,
         canEmitEvents: !!instanceSpec.allowedEventTypes,
         allowedEventTypes: instanceSpec.allowedEventTypes,
         eventarcChannel: instanceSpec.eventarcChannel,
@@ -134,6 +139,7 @@ export function configureExtensionInstanceTask(
         projectId,
         instanceId: instanceSpec.instanceId,
         params: instanceSpec.params,
+        systemParams: instanceSpec.systemParams,
         canEmitEvents: !!instanceSpec.allowedEventTypes,
         allowedEventTypes: instanceSpec.allowedEventTypes,
         eventarcChannel: instanceSpec.eventarcChannel,

src/extensions/extensionsApi.ts

@@ -73,13 +73,15 @@ export async function createInstance(args: {
   instanceId: string;
   extensionSource?: ExtensionSource;
   extensionVersionRef?: string;
-  params: { [key: string]: string };
+  params: Record<string, string>;
+  systemParams?: Record<string, string>;
   allowedEventTypes?: string[];
   eventarcChannel?: string;
   validateOnly?: boolean;
 }): Promise<ExtensionInstance> {
   const config: any = {
     params: args.params,
+    systemParams: args.systemParams ?? {},
     allowedEventTypes: args.allowedEventTypes,
     eventarcChannel: args.eventarcChannel,
   };
@@ -188,7 +190,8 @@ export async function listInstances(projectId: string): Promise<ExtensionInstanc
 export async function configureInstance(args: {
   projectId: string;
   instanceId: string;
-  params: { [option: string]: string };
+  params: Record<string, string>;
+  systemParams?: Record<string, string>;
   canEmitEvents: boolean;
   allowedEventTypes?: string[];
   eventarcChannel?: string;
@@ -215,6 +218,10 @@ export async function configureInstance(args: {
     reqBody.data.config.eventarcChannel = args.eventarcChannel;
   }
   reqBody.updateMask += ",config.allowed_event_types,config.eventarc_channel";
+  if (args.systemParams) {
+    reqBody.data.config.systemParams = args.systemParams;
+    reqBody.updateMask += ",config.system_params";
+  }
   return patchInstance(reqBody);
 }
 
@@ -233,7 +240,8 @@ export async function updateInstance(args: {
   projectId: string;
   instanceId: string;
   extensionSource: ExtensionSource;
-  params?: { [option: string]: string };
+  params?: Record<string, string>;
+  systemParams?: Record<string, string>;
   canEmitEvents: boolean;
   allowedEventTypes?: string[];
   eventarcChannel?: string;
@@ -249,6 +257,10 @@ export async function updateInstance(args: {
     body.config.params = args.params;
     updateMask += ",config.params";
   }
+  if (args.systemParams) {
+    body.config.systemParams = args.systemParams;
+    updateMask += ",config.system_params";
+  }
   if (args.canEmitEvents) {
     if (args.allowedEventTypes === undefined || args.eventarcChannel === undefined) {
       throw new FirebaseError(
@@ -283,7 +295,8 @@ export async function updateInstanceFromRegistry(args: {
   projectId: string;
   instanceId: string;
   extRef: string;
-  params?: { [option: string]: string };
+  params?: Record<string, string>;
+  systemParams?: Record<string, string>;
   canEmitEvents: boolean;
   allowedEventTypes?: string[];
   eventarcChannel?: string;
@@ -301,6 +314,10 @@ export async function updateInstanceFromRegistry(args: {
     body.config.params = args.params;
     updateMask += ",config.params";
   }
+  if (args.systemParams) {
+    body.config.systemParams = args.systemParams;
+    updateMask += ",config.system_params";
+  }
   if (args.canEmitEvents) {
     if (args.allowedEventTypes === undefined || args.eventarcChannel === undefined) {
       throw new FirebaseError(

src/extensions/paramHelper.ts

@@ -265,3 +265,8 @@ export function readEnvFile(envPath: string): Record<string, string> {
   }
   return result.envs;
 }
+
+export function isSystemParam(paramName: string): boolean {
+  const regex = /^firebaseextensions\.[a-zA-Z0-9\.]*\//;
+  return regex.test(paramName);
+}

src/extensions/types.ts

@@ -63,9 +63,8 @@ export interface ExtensionConfig {
   name: string;
   createTime: string;
   source: ExtensionSource;
-  params: {
-    [key: string]: any;
-  };
+  params: Record<string, string>;
+  systemParams: Record<string, string>;
   populatedPostinstallContent?: string;
   extensionRef?: string;
   extensionVersion?: string;
@@ -100,6 +99,7 @@ export interface ExtensionSpec {
   releaseNotesUrl?: string;
   sourceUrl?: string;
   params: Param[];
+  systemParams: Param[];
   preinstallContent?: string;
   postinstallContent?: string;
   readmeContent?: string;

src/functional.ts

@@ -97,20 +97,38 @@ export function assertExhaustive(val: never): never {
 }
 
 /**
- * Utility to partition an array into two based on callbackFn's truthiness for each element.
+ * Utility to partition an array into two based on predicate's truthiness for each element.
  * Returns a Array containing two Array<T>. The first array contains all elements that returned true,
  * the second contains all elements that returned false.
  */
-export function partition<T>(arr: T[], callbackFn: (elem: T) => boolean): [T[], T[]] {
+export function partition<T>(arr: T[], predicate: (elem: T) => boolean): [T[], T[]] {
   return arr.reduce<[T[], T[]]>(
     (acc, elem) => {
-      acc[callbackFn(elem) ? 0 : 1].push(elem);
+      acc[predicate(elem) ? 0 : 1].push(elem);
       return acc;
     },
     [[], []]
   );
 }
 
+/**
+ * Utility to partition a Record into two based on predicate's truthiness for each element.
+ * Returns a Array containing two Record<string, T>. The first array contains all elements that returned true,
+ * the second contains all elements that returned false.
+ */
+export function partitionRecord<T>(
+  rec: Record<string, T>,
+  predicate: (key: string, val: T) => boolean
+): [Record<string, T>, Record<string, T>] {
+  return Object.entries(rec).reduce<[Record<string, T>, Record<string, T>]>(
+    (acc, [key, val]) => {
+      acc[predicate(key, val) ? 0 : 1][key] = val;
+      return acc;
+    },
+    [{}, {}]
+  );
+}
+
 /**
  * Create a map of transformed values for all keys.
  */

src/functions/env.ts

@@ -45,7 +45,7 @@ const RESERVED_KEYS = [
 const LINE_RE = new RegExp(
   "^" +                      // begin line
   "\\s*" +                   //   leading whitespaces
-  "(\\w+)" +                 //   key
+  "([\\w./]+)" +                 //   key
   "\\s*=[\\f\\t\\v]*" +              //   separator (=)
   "(" +                      //   begin optional value
   "\\s*'(?:\\\\'|[^'])*'|" + //     single quoted or

src/test/deploy/extensions/planner.spec.ts

@@ -18,6 +18,7 @@ function extensionVersion(version?: string): any {
       resources: [],
       sourceUrl: "https://google.com",
       params: [],
+      systemParam: [],
     },
   };
 }
@@ -105,6 +106,7 @@ describe("Extensions Deployment Planner", () => {
       name: "",
       sourceUrl: "",
       params: [],
+      systemParams: [],
     };
 
     const INSTANCE_WITH_EVENTS: ExtensionInstance = {
@@ -116,6 +118,7 @@ describe("Extensions Deployment Planner", () => {
       etag: "123456",
       config: {
         params: {},
+        systemParams: {},
         extensionRef: "firebase/image-resizer",
         extensionVersion: "0.1.0",
         name: "projects/my-test-proj/instances/image-resizer/configurations/95355951-397f-4821-a5c2-9c9788b2cc63",
@@ -135,6 +138,7 @@ describe("Extensions Deployment Planner", () => {
     const INSTANCE_SPEC_WITH_EVENTS: planner.DeploymentInstanceSpec = {
       instanceId: "image-resizer",
       params: {},
+      systemParams: {},
       allowedEventTypes: ["google.firebase.custom-event-occurred"],
       eventarcChannel: "projects/my-test-proj/locations/us-central1/channels/firebase",
       etag: "123456",

src/test/emulators/extensions/validation.spec.ts

@@ -30,6 +30,7 @@ function fakeInstanceSpecWithAPI(instanceId: string, apiName: string): Deploymen
   return {
     instanceId,
     params: {},
+    systemParams: {},
     ref: {
       publisherId: "test",
       extensionId: "test",
@@ -47,6 +48,7 @@ function fakeInstanceSpecWithAPI(instanceId: string, apiName: string): Deploymen
         sourceUrl: "test.com",
         resources: [],
         params: [],
+        systemParams: [],
         apis: [{ apiName, reason: "because" }],
       },
     },

src/test/emulators/extensionsEmulator.spec.ts

@@ -45,6 +45,7 @@ const TEST_EXTENSION_VERSION: ExtensionVersion = {
       },
     ],
     params: [],
+    systemParams: [],
     version: "0.1.18",
     sourceUrl: "https://fake.test",
   },
@@ -80,6 +81,7 @@ describe("Extensions Emulator", () => {
               "google.firebase.image-resize-started,google.firebase.image-resize-completed",
             EVENTARC_CHANNEL: "projects/test-project/locations/us-central1/channels/firebase",
           },
+          systemParams: {},
           allowedEventTypes: [
             "google.firebase.image-resize-started",
             "google.firebase.image-resize-completed",

src/test/emulators/functionsEmulatorShared.spec.ts

@@ -158,6 +158,7 @@ describe("FunctionsEmulatorShared", () => {
       resources: [],
       sourceUrl: "test.com",
       params: [],
+      systemParams: [],
       postinstallContent: "Should subsitute ${param:KEY}",
     };
     const testSubbedSpec: ExtensionSpec = {
@@ -166,6 +167,7 @@ describe("FunctionsEmulatorShared", () => {
       resources: [],
       sourceUrl: "test.com",
       params: [],
+      systemParams: [],
       postinstallContent: "Should subsitute value",
     };
     const testExtension: Extension = {