GCP installer script

packages/external-db-google-sheets/tests/mock-server.js

@@ -0,0 +1,4 @@
+const { app }  = require('./mock_google_sheets_api');
+
+
+app.listen(1502, () => console.log('google-sheets mock server is running'))

packages/external-db-google-sheets/tests/templates.js

@@ -0,0 +1,44 @@
+
+const appendValuesResponse = (spreadsheetId, sheet, range, updatedRows, updatedColumns, updatedCells) => ({
+    spreadsheetId: `${spreadsheetId}`,
+    tableRange: `${sheet}!${range}`,
+    updates: updateValuesResponse(spreadsheetId, sheet, range, updatedRows, updatedColumns, updatedCells)
+})
+
+const batchGetValuesResponse = (spreadsheetId, sheet, range, majorDimension, values) => ({
+    spreadsheetId: `${spreadsheetId}`,
+    valueRanges: [
+      {
+        range: `${sheet}!${range}`,
+        majorDimension: `${majorDimension}`,
+        values: values
+      }
+    ]
+})
+
+const spreadsheet = (spreadsheetId, title, sheets = []) => ({
+    spreadsheetId: `${spreadsheetId}`,
+    properties: {
+      title: `${title}`
+    },
+    sheets: sheets
+})
+
+const updateValuesResponse = (spreadsheetId, sheet, range, updatedRows, updatedColumns, updatedCells) => ({
+    spreadsheetId: `${spreadsheetId}`,
+    updatedRange: `${sheet}!${range}`,
+    updatedRows: `${updatedRows}`,
+    updatedColumns: `${updatedColumns}`,
+    updatedCells: `${updatedCells}`,
+})
+
+
+const valueRange = (sheet, range, majorDimension) => ({
+    range: `${sheet}!${range}`,
+    majorDimension: `${majorDimension}`,
+    values: []
+})
+
+module.exports = { appendValuesResponse, batchGetValuesResponse, spreadsheet, updateValuesResponse, valueRange }
+
+

scripts/index.js

@@ -1,7 +1,7 @@
 const { askForUserInput } = require('./lib/cli/user-input')
 const { main } = require('./lib/core/provision')
 
-const process = async () => {
+const process = async() => {
     const userInput = await askForUserInput()
     await main(userInput)
 }

scripts/lib/cli/user-input.js

@@ -17,7 +17,8 @@ const dbSupportedOn = (vendor) => {
 
 const nonEmpty = input => !(!input || input.trim() === '')
 
-const Aws = require('../../.credentials.json')
+const Aws = { accessKeyId: '', secretAccessKey: '' } 
+const GCP = { gcpClientEmail: '', gcpPrivateKey: '', gcpProjectId: '' } 
 
 const credentialsFor = vendor => {
     switch (vendor) {
@@ -42,9 +43,25 @@ const credentialsFor = vendor => {
             return inquirer.prompt([
                 {
                     type: 'input',
-                    name: 'whatever',
-                    message: 'gcp credentials todo',
+                    name: 'gcpClientEmail',
+                    message: 'GCP Client email',
+                    validate: nonEmpty,
+                    default: GCP.gcpClientEmail
+                },
+                {
+                    type: 'input',
+                    name: 'gcpPrivateKey',
+                    message: 'GCP Private Key',
+                    validate: nonEmpty,
+                    default: GCP.gcpPrivateKey
                 },
+                {
+                    type: 'input',
+                    name: 'gcpProjectId',
+                    message: 'GCP project id',
+                    validate: nonEmpty,
+                    default: GCP.gcpProjectId
+                }
             ])
         case 'azure':
             return inquirer.prompt([

scripts/lib/core/provision.js

@@ -1,4 +1,5 @@
 const aws = require('../aws')
+const gcp = require('../gcp')
 const { randomCredentials, randomSecretKey } = require('../utils/password_utils')
 const { blockUntil, randomWithPrefix } = require('../utils/utils')
 const { info, blankLine, startSpinnerWith } = require('../cli/display')
@@ -7,29 +8,32 @@ const providerFor = (vendor) => {
     switch (vendor) {
         case 'aws':
         case 'gcp':
+            return gcp
         case 'azure':
             return aws
     }
 }
 
 const provisionDb = async (provider, configWriter, { engine, secretId, secretKey, dbName }) => {
     const dbCredentials = randomCredentials()
-    const instanceName = randomWithPrefix('velo-external-db')
+    const instanceName = randomWithPrefix('velo-external-db') 
 
     await startSpinnerWith(`Creating ${engine} DB Instance`, async () => await provider.createDb({ name: instanceName, engine: engine, credentials: dbCredentials}))
     await startSpinnerWith(`Waiting for db instance to start`, async () => await blockUntil( async () => (await provider.dbStatusAvailable(instanceName)).available ))
 
     const status = await provider.dbStatusAvailable(instanceName)
 
-    await startSpinnerWith(`Writing db config`, async () => await configWriter.writeConfig(secretId, dbCredentials, status.host, dbName, secretKey))
+    const secrets = await startSpinnerWith(`Writing db config`, async () => await configWriter.writeConfig(secretId, dbCredentials, status.host, status.connectionName, dbName, secretKey))
 
     await startSpinnerWith(`Provision Velo DB on db instance`, async () => await provider.postCreateDb(engine, dbName, status, dbCredentials))
+
+    return { status, secrets }
 }
 
-const provisionAdapter = async (provider, engine, secretId) => {
+const provisionAdapter = async (provider, engine, secretId, secrets) => {
     const instanceName = randomWithPrefix('velo-external-db-adapter')
 
-    const { serviceId } = await startSpinnerWith(`Provision Adapter`, async () => await provider.createAdapter(instanceName, engine, secretId))
+    const { serviceId } = await startSpinnerWith(`Provision Adapter`, async () => await provider.createAdapter(instanceName, engine, secretId, secrets))
     await startSpinnerWith(`Waiting adapter server instance to start`, async () => await blockUntil( async () => (await provider.adapterStatus(serviceId)).available ))
 
     const status = await provider.adapterStatus(serviceId)
@@ -54,12 +58,12 @@ const main = async ({ vendor, engine, credentials }) => {
     blankLine()
     blankLine()
     info('Provision DB Instance')
-    await provisionDb(dbProvision, configWriter, { engine, secretId, secretKey, dbName})
+    const {status, secrets} = await provisionDb(dbProvision, configWriter, { engine, secretId, secretKey, dbName})
 
     blankLine()
     blankLine()
     info('Provision Adapter')
-    await provisionAdapter(adapterProvision, engine, secretId)
+    await provisionAdapter(adapterProvision, engine, secretId, secrets, status.connectionName)
 
     blankLine()
     blankLine()

scripts/lib/gcp/adapter_gcp_provision.js

@@ -0,0 +1,88 @@
+const {GoogleAuth} = require('google-auth-library')
+const AdapterImageUrl = 'gcr.io/wix-velo-api/velo-external-db'
+
+class AdapterProvision {
+    constructor(credentials, region) {
+        this.authClient = new GoogleAuth({
+            credentials : { client_email: credentials.gcpClientEmail, private_key: credentials.gcpPrivateKey },
+            scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/service.management']
+        })
+        this.region = region
+    }
+
+    secretsToEnvs(secrets) { 
+        return Object.entries(secrets).map(([envVariable, secretName])=> {
+            return {
+                name: envVariable,
+                valueFrom: {
+                    secretKeyRef: { key: 'latest', name: secretName }
+                }
+            } 
+
+        })
+    }
+
+    async createAdapter(name, engine, secrets, connectionName) {
+        const client = await this.authClient.getClient()
+        const projectId = await this.authClient.getProjectId()
+        const apiUrl = `https://${this.region}-run.googleapis.com/apis/serving.knative.dev/v1/namespaces/${projectId}/services/`
+        const authApiUrl = `https://run.googleapis.com/v1/projects/${projectId}/locations/us-central1/services/${name}:setIamPolicy`
+
+        const env = [
+            { name: 'CLOUD_VENDOR', value: 'gcp' },
+            { name: 'TYPE', value: engine },
+            ...this.secretsToEnvs(secrets)
+        ]
+
+        const adapterInstanceProperties = {
+
+            apiVersion: 'serving.knative.dev/v1',
+            kind: 'Service',
+            metadata: {
+              annotations: {
+                  'run.googleapis.com/launch-stage' : 'BETA'
+              },
+              name: name,
+              namespace: projectId
+            },
+            spec: {
+                template: {
+                    metadata: {
+                        annotations: { 'run.googleapis.com/cloudsql-instances': `${projectId}:${this.region}:${connectionName}`}
+                },
+                spec: {
+                    containers: [
+                        { image: AdapterImageUrl, env }
+                    ]
+                }
+              }
+            }
+        }
+
+        const authData = {
+            policy: {
+              bindings: [
+                {
+                  members: [ 'allUsers' ],
+                  role: 'roles/run.invoker'
+                }
+              ],
+              etag: 'ACAB'
+            }
+          }
+
+        await client.request({ url: apiUrl, method: 'POST', data: adapterInstanceProperties })
+
+        await client.request({ url: authApiUrl, method: 'POST', data: authData })
+
+    }
+
+    async adapterStatus(serviceId) {
+    }
+
+    async rdsRoleArn() {
+    }
+}
+
+
+module.exports = AdapterProvision

scripts/lib/gcp/db_gcp_provision.js

@@ -0,0 +1,62 @@
+const {GoogleAuth} = require('google-auth-library')
+class DbProvision {
+    constructor({ gcpClientEmail, gcpPrivateKey }) {
+        this.authClient = new GoogleAuth({
+            credentials : { client_email: gcpClientEmail, private_key: gcpPrivateKey },
+            scopes: 'https://www.googleapis.com/auth/cloud-platform'
+        })
+    }
+
+    asGcp(engine) {
+        switch (engine) {
+            case 'mysql':
+                return 'MYSQL_5_7'
+            default:
+                break
+        }
+    }
+
+    async createDb( { name, engine, credentials }) {
+        const client = await this.authClient.getClient()
+        const projectId = await this.authClient.getProjectId()
+        const apiUrl = `https://sqladmin.googleapis.com/sql/v1beta4/projects/${projectId}/instances`
+
+        const dbInstanceProperties = {
+            databaseVersion: this.asGcp(engine),
+            name,
+            settings: { tier: 'db-f1-micro' },
+            rootPassword: credentials.passwd
+        }
+
+        const response = await client.request({ url: apiUrl, method: 'POST', data: dbInstanceProperties})
+        return response.operationType === 'CREATE' && response.status === 'PENDING'
+    }
+
+    async dbStatusAvailable(name) {
+        const client = await this.authClient.getClient()
+        const projectId = await this.authClient.getProjectId()
+        const apiUrl = `https://sqladmin.googleapis.com/sql/v1beta4/projects/${projectId}/instances/${name}`
+
+        const response = await client.request({ url: apiUrl })
+        const instance = response.data.items.find(i => i.name === name)
+
+        return {
+            instanceName: name,
+            available: instance.state === 'RUNNABLE',
+            host: instance.ipAddresses.find(i => i.type === 'PRIMARY').ipAddress,
+            connectionName: instance.connectionName,
+            databaseVersion: instance.databaseVersion,
+        }
+    }
+
+    async postCreateDb(engine, dbName, status, credentials) {
+        const client = await this.authClient.getClient()
+        const projectId = await this.authClient.getProjectId()
+        const apiUrl = `https://sqladmin.googleapis.com/sql/v1beta4/projects/${projectId}/instances/${status.instanceName}/databases`
+
+        await client.request({ url: apiUrl, method:'POST', data: { name:  dbName } })
+    }
+}
+
+
+module.exports = DbProvision

scripts/lib/gcp/gcp_config_writer.js

@@ -0,0 +1,43 @@
+const { SecretManagerServiceClient } = require('@google-cloud/secret-manager')
+const { randomWithPrefix } = require('../utils/utils')
+
+
+class ConfigWriter {
+    constructor({ gcpClientEmail, gcpPrivateKey }) {
+        this.client = new SecretManagerServiceClient({ credentials : { client_email: gcpClientEmail, private_key: gcpPrivateKey }})
+    }
+
+    extractSecretName(secretName) {
+        return secretName.split('/').splice(-1)[0]
+    }
+
+     async createSecret(secretKey, secretValue, projectId) {
+        const [secret] = await this.client.createSecret({
+            parent: `projects/${projectId}`,
+            secretId: randomWithPrefix(`VELO-EXTERNAL-DB-${secretKey}`),
+            secret: { replication: { automatic: {} } }
+        })
+
+        await this.client.addSecretVersion({ 
+            parent: secret.name, 
+            payload: { data: Buffer.from(secretValue, 'utf8') } 
+        })
+
+        return this.extractSecretName(secret.name)
+     }
+
+    async writeConfig(secretId, dbCredentials, host, connectionName, db, secretKey) {
+        const config = { USER: 'root', PASSWORD: dbCredentials.passwd, CLOUD_SQL_CONNECTION_NAME: connectionName, DB: db, SECRET_KEY: secretKey }
+        const projectId = await this.client.getProjectId()
+        const secrets = {}
+
+        for (const [secretKey, secretValue] of Object.entries(config)) {
+            const secretName = await this.createSecret(secretKey, secretValue, projectId)
+            secrets[secretKey] = secretName
+        }
+
+        return secrets
+    }
+}
+
+module.exports = ConfigWriter

scripts/lib/gcp/gcp_network_provision.js

@@ -0,0 +1,12 @@
+
+
+class NetworkProvision {
+    constructor(credentials, region) {
+    }
+
+    async addSecurityRule(groupId, port) {
+    }
+
+}
+
+module.exports = NetworkProvision

scripts/lib/gcp/index.js

@@ -0,0 +1,7 @@
+const DbProvision = require('./db_gcp_provision')
+const ConfigWriter = require('./gcp_config_writer')
+const AdapterProvision = require('./adapter_gcp_provision')
+const NetworkProvision = require('./gcp_network_provision')
+
+
+module.exports = { DbProvision, ConfigWriter, AdapterProvision, NetworkProvision}

scripts/lib/utils/utils.js

@@ -17,4 +17,5 @@ const blockUntil = async f => waitFor(
 const randomInt = () => Math.floor(Math.random() * 10000)
 const randomWithPrefix = prefix => `${prefix}-${randomInt()}`
 
+
 module.exports = { blockUntil, randomWithPrefix }