From 3ce2fc0ee68d56f321ac30610a4240b3693e13f0 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:16:29 +0200 Subject: [PATCH 01/47] testing main yml --- .github/workflows/main.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3463bf1fc..d76113a50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,3 +90,40 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down + + test_bigquery: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14 + uses: actions/setup-node@v2 + with: + node-version: 14 + + - name: Restore Dependencies + uses: actions/cache@v2 + with: + path: | + node_modules + */*/node_modules + key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + + - id: auth + uses: google-github-actions/auth@v0.4.0 + with: + credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.1 + + - name: Use gcloud CLI + run: gcloud info + + - name: Installing Dependencies + run: npm install + + - name: Install dependencies of the homemade packages. + run: npm run build:dev + + - name: Testing + run: npm run test:bigquery From 0723063032c49b834ce8a88843e15db0db3200bb Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:50:17 +0200 Subject: [PATCH 02/47] disable other tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d76113a50..4dd087a81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,7 +121,7 @@ jobs: - name: Installing Dependencies run: npm install - + - name: Install dependencies of the homemade packages. run: npm run build:dev From f543e414e06c89781a9cfeb7474066e3f1dddd69 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 15:05:12 +0200 Subject: [PATCH 03/47] print errors --- packages/external-db-bigquery/lib/sql_exception_translator.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 41bd1c54b..3e7c5bfe5 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,6 +15,7 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { + console.log(err) throw notThrowingTranslateErrorCodes(err) } From ef4efc8541ba4202b96b0a4475092738038d8985 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 15:43:47 +0200 Subject: [PATCH 04/47] log errors --- .../lib/bigquery_schema_provider.js | 69 ++++++++++++------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index 898a630d7..3c7ff6294 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -11,49 +11,72 @@ class SchemaProvider { } async list() { - const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') - const tables = parseTableData(res[0]) + try{ + const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') + const tables = parseTableData(res[0]) + return Object.entries(tables) + .map(([collectionName, rs]) => asWixSchema(rs.map( this.translateDbTypes.bind(this) ), collectionName)) + } catch(e) { + console.log(e) + } - return Object.entries(tables) - .map(([collectionName, rs]) => asWixSchema(rs.map( this.translateDbTypes.bind(this) ), collectionName)) } async create(collectionName, _columns) { - const columns = _columns || [] - const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) - await this.pool.createTable(collectionName, { schema: dbColumnsSql }) - .catch(translateErrorCodes) + try{ + const columns = _columns || [] + const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) + await this.pool.createTable(collectionName, { schema: dbColumnsSql }) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } } async drop(collectionName) { - await this.pool.table(collectionName).delete() - .catch(translateErrorCodes) + try { + await this.pool.table(collectionName).delete() + .catch(translateErrorCodes) + } catch(e) { + og.error(e) + } } async addColumn(collectionName, column) { - await validateSystemFields(column.name) - - await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) - .catch(translateErrorCodes) - + try { + await validateSystemFields(column.name) + await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) + .catch(translateErrorCodes) + + } catch(e) { + console.log(e) + } } async removeColumn(collectionName, columnName) { - await validateSystemFields(columnName) + try { + await validateSystemFields(columnName) + await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } + - await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) - .catch(translateErrorCodes) } async describeCollection(collectionName) { - const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) - - if (res[0].length === 0) { - throw new CollectionDoesNotExists('Collection does not exists') + try{ + const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) + if (res[0].length === 0) { + throw new CollectionDoesNotExists('Collection does not exists') + } + return asWixSchema(res[0].map( this.translateDbTypes.bind(this) ), collectionName) + } catch(e) { + console.log(e) } - return asWixSchema(res[0].map( this.translateDbTypes.bind(this) ), collectionName) } From 81d01ca3cb2e3ffc35dd6d5a1a8a0180405db17d Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:00:39 +0200 Subject: [PATCH 05/47] test fefactor bigquery --- .../lib/bigquery_data_provider.js | 9 ++- .../lib/bigquery_schema_provider.js | 78 +++++++------------ .../lib/bigquery_utils.js | 2 +- .../lib/connection_provider.js | 6 +- .../lib/sql_exception_translator.js | 1 - .../sql_filter_transformer_test_support.js | 4 +- .../lib/readers/gcp_config_reader.js | 6 +- packages/velo-external-db/package.json | 4 +- .../test/e2e/app_data.e2e.spec.js | 47 ++++++----- .../resources/engines/bigquery_resources.js | 12 +-- .../test/resources/provider_resources.js | 2 +- .../test/storage/data_provider.spec.js | 42 +++++----- .../test/storage/schema_provider.spec.js | 52 +++++++------ 13 files changed, 136 insertions(+), 129 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_data_provider.js b/packages/external-db-bigquery/lib/bigquery_data_provider.js index 828734a69..dd769879e 100644 --- a/packages/external-db-bigquery/lib/bigquery_data_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_data_provider.js @@ -34,6 +34,7 @@ class DataProvider { } async insert(collectionName, items) { + // const patchedItems = items.map(patchDateTime) // const itemsArr = patchedItems.map( item => `(${Object.values(item).map(i => typeof(i) === 'number' ? i : `"${i}"`).join(', ')})` ).join(', ') // const sql = `INSERT INTO \`${collectionName}\` (${Object.keys(items[0]).join(', ')}) VALUES ${itemsArr}` @@ -41,9 +42,9 @@ class DataProvider { // const res = await this.pool.query(sql).catch( translateErrorCodes ) const table = await this.pool.table(collectionName) - const res = await table.insert(items) - - return res + await table.insert(items) + + return items.length } async update(collectionName, items) { @@ -82,7 +83,7 @@ class DataProvider { const resultSet = await this.pool.query({ query: sql, params: [...whereParameters, ...parameters] }) .catch( translateErrorCodes ) - return resultSet[0] + return resultSet[0].map( unPatchDateTime ) } } diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index 3c7ff6294..75c36ffbc 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -5,79 +5,59 @@ const { SystemFields, validateSystemFields, asWixSchema, parseTableData } = requ const escapeIdentifier = i => i class SchemaProvider { - constructor(pool) { + constructor(pool, { projectId, databaseId }) { + this.projectId = projectId + this.databaseId = databaseId this.pool = pool this.sqlSchemaTranslator = new SchemaColumnTranslator() } async list() { - try{ - const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') - const tables = parseTableData(res[0]) - return Object.entries(tables) - .map(([collectionName, rs]) => asWixSchema(rs.map( this.translateDbTypes.bind(this) ), collectionName)) - } catch(e) { - console.log(e) - } - + const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM ${this.projectId}.${this.databaseId}.INFORMATION_SCHEMA.COLUMNS`) + const tables = parseTableData(res[0]) + return Object.entries(tables) + .map(([collectionName, rs]) => asWixSchema(rs.map(this.translateDbTypes.bind(this)), collectionName)) } async create(collectionName, _columns) { - try{ - const columns = _columns || [] - const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) - await this.pool.createTable(collectionName, { schema: dbColumnsSql }) - .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } + const columns = _columns || [] + const dbColumnsSql = [...SystemFields, ...columns].map(c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) + await this.pool.createTable(collectionName, { schema: dbColumnsSql }) + .catch(translateErrorCodes) } async drop(collectionName) { - try { - await this.pool.table(collectionName).delete() - .catch(translateErrorCodes) - } catch(e) { - og.error(e) - } + await this.pool.table(collectionName).delete() + .catch(translateErrorCodes) } async addColumn(collectionName, column) { - try { - await validateSystemFields(column.name) - await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) - .catch(translateErrorCodes) - - } catch(e) { - console.log(e) - } + await validateSystemFields(column.name) + try{ + await this.pool.query(`ALTER TABLE ${this.projectId}.${this.databaseId}.${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) + } catch (err) { + if (err.message.includes('was not found')) + throw new CollectionDoesNotExists('Collection does not exists') + else + translateErrorCodes(err) + } } async removeColumn(collectionName, columnName) { - try { - await validateSystemFields(columnName) - await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) + await validateSystemFields(columnName) + await this.pool.query(`CREATE OR REPLACE TABLE ${this.projectId}.${this.databaseId}.${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } - - - } async describeCollection(collectionName) { - try{ - const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) - if (res[0].length === 0) { - throw new CollectionDoesNotExists('Collection does not exists') - } - return asWixSchema(res[0].map( this.translateDbTypes.bind(this) ), collectionName) - } catch(e) { - console.log(e) + const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM ${this.projectId}.${this.databaseId}.INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) + + if (res[0].length === 0) { + throw new CollectionDoesNotExists('Collection does not exists') } + return asWixSchema(res[0].map(this.translateDbTypes.bind(this)), collectionName) } diff --git a/packages/external-db-bigquery/lib/bigquery_utils.js b/packages/external-db-bigquery/lib/bigquery_utils.js index d9b5dd3f0..8d032c376 100644 --- a/packages/external-db-bigquery/lib/bigquery_utils.js +++ b/packages/external-db-bigquery/lib/bigquery_utils.js @@ -24,7 +24,7 @@ const unPatchDateTime = (item) => { if (reISO.test(value)) { obj[key] = new Date(value) } else { - obj[key] = item[key] + obj[key] = item[key].toNumber ? item[key].toNumber() : item[key] } } diff --git a/packages/external-db-bigquery/lib/connection_provider.js b/packages/external-db-bigquery/lib/connection_provider.js index d269a42a0..c526f5272 100644 --- a/packages/external-db-bigquery/lib/connection_provider.js +++ b/packages/external-db-bigquery/lib/connection_provider.js @@ -4,14 +4,14 @@ const DatabaseOperations = require ('./bigquery_operations') const SchemaProvider = require('./bigquery_schema_provider') const DataProvider = require('./bigquery_data_provider') -const init = ( cfg ) => { +const init = ({ projectId, databaseId }) => { const bigquery = new BigQuery() - const pool = bigquery.dataset(cfg.databaseId) + const pool = bigquery.dataset(databaseId) const filterParser = new FilterParser() const databaseOperations = new DatabaseOperations(pool) const dataProvider = new DataProvider(pool, filterParser) - const schemaProvider = new SchemaProvider(pool) + const schemaProvider = new SchemaProvider(pool, { projectId, databaseId }) return { dataProvider: dataProvider, schemaProvider: schemaProvider, databaseOperations, connection: pool, cleanup: async() => {} } } diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 3e7c5bfe5..41bd1c54b 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,7 +15,6 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { - console.log(err) throw notThrowingTranslateErrorCodes(err) } diff --git a/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js b/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js index f2c667530..79a71f8de 100644 --- a/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js +++ b/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js @@ -33,11 +33,11 @@ const givenOrderByFor = (column, sort) => { const givenFilterByIdWith = (id, filter) => { when(filterParser.transform).calledWith(filter) - .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = $1`, parameters: [id], offset: 2 }) + .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = ?`, parameters: [id], offset: 2 }) } const givenAggregateQueryWith = (having, numericColumns, columnAliases, groupByColumns, filter, offest) => { - when(filterParser.parseAggregation).calledWith(having, filter, offest) + when(filterParser.parseAggregation).calledWith(having, filter) .mockReturnValue({ fieldsStatement: `${groupByColumns.map( escapeIdentifier ).join(', ')}, MAX(${escapeIdentifier(numericColumns[0].name)}) AS ${escapeIdentifier(columnAliases[0])}, SUM(${escapeIdentifier(numericColumns[1].name)}) AS ${escapeIdentifier(columnAliases[1])}`, groupByColumns: groupByColumns, diff --git a/packages/external-db-config/lib/readers/gcp_config_reader.js b/packages/external-db-config/lib/readers/gcp_config_reader.js index 9a644a648..6c7c9936d 100644 --- a/packages/external-db-config/lib/readers/gcp_config_reader.js +++ b/packages/external-db-config/lib/readers/gcp_config_reader.js @@ -99,13 +99,13 @@ class GcpBigQueryConfigReader { } async readConfig() { - const { DATABASE_ID, SECRET_KEY } = process.env - return { databaseId: DATABASE_ID, secretKey: SECRET_KEY } + const { PROJECT_ID, DATABASE_ID, SECRET_KEY } = process.env + return { projectId: PROJECT_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY } } validate() { return { - missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['DATABASE_ID', 'SECRET_KEY']) + missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['PROJECT_ID', 'DATABASE_ID', 'SECRET_KEY']) } } diff --git a/packages/velo-external-db/package.json b/packages/velo-external-db/package.json index 25235f1b6..a044dc56d 100644 --- a/packages/velo-external-db/package.json +++ b/packages/velo-external-db/package.json @@ -11,10 +11,10 @@ "test:firestore": "TEST_ENGINE=firestore jest --runInBand ./test", "test:mssql": "TEST_ENGINE=mssql jest --runInBand ./test", "test:mongo": "TEST_ENGINE=mongo jest --runInBand ./test", - "test:google-sheets": "TEST_ENGINE=google-sheet jest -i packages/velo-external-db/test/e2e/app_data.e2e.spec.js --runInBand -t 'Velo External DB Data REST API' ", + "test:google-sheets": "TEST_ENGINE=google-sheet jest -i packages/velo-external-db/test/e2e/app_data.e2e.spec.js --runInBand -t 'Schema API' ", "test:airtable": "TEST_ENGINE=airtable jest --runInBand ./test ", "test:dynamodb": "TEST_ENGINE=dynamodb jest --runInBand ./test ", - "test:bigquery": "TEST_ENGINE=bigquery jest --runInBand -i packages/velo-external-db/test/e2e/app_schema.e2e.spec.js -t 'Velo External DB Schema REST API' ", + "test:bigquery": "TEST_ENGINE=bigquery jest --runInBand ./test", "test:watch": "jest --detectOpenHandles --watch", "start": "node lib/app.js", "deploy": "gcloud app deploy ./app.yaml" diff --git a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js index e57f3201a..21144ae4f 100644 --- a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js +++ b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js @@ -83,15 +83,19 @@ describe('Velo External DB Data REST API', () => { }) } + + if (shouldNotRunOn(['BigQuery'], name)) { test('delete one api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) + await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) - }) + expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) + }) + } + if (shouldNotRunOn(['BigQuery'], name)) { test('bulk delete api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems(ctx.items, ctx.collectionName, authAdmin) @@ -100,31 +104,38 @@ describe('Velo External DB Data REST API', () => { expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) }) + } + if (shouldNotRunOn(['BigQuery'], name)) { test('get by id api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item], ctx.collectionName, authAdmin) expect((await axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin)).data).toEqual({ item: ctx.item }) }) + } + + if (shouldNotRunOn(['BigQuery'], name)) { + test('update api', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - test('update api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - - await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) + await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) - }) + expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) + }) + } - test('bulk update api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems(ctx.items, ctx.collectionName, authAdmin) + if (shouldNotRunOn(['BigQuery'], name)) { + test('bulk update api', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems(ctx.items, ctx.collectionName, authAdmin) - await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) + await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) - expect((await data.expectAllDataIn(ctx.collectionName, authAdmin)).items).toEqual(expect.arrayContaining(ctx.modifiedItems)) - }) + expect((await data.expectAllDataIn(ctx.collectionName, authAdmin)).items).toEqual(expect.arrayContaining(ctx.modifiedItems)) + }) + } test('count api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) diff --git a/packages/velo-external-db/test/resources/engines/bigquery_resources.js b/packages/velo-external-db/test/resources/engines/bigquery_resources.js index 4d7a7ecd5..2930f7aa5 100644 --- a/packages/velo-external-db/test/resources/engines/bigquery_resources.js +++ b/packages/velo-external-db/test/resources/engines/bigquery_resources.js @@ -1,29 +1,31 @@ const { init } = require('external-db-bigquery') const databaseId = 'testDB' +const projectId = 'corvid-managed-cfe9809c' const connection = () => { - const { connection, cleanup } = init({ databaseId }) + const { connection, cleanup } = init({ databaseId, projectId }) return { pool: connection, cleanup } } const cleanup = async() => { - const { schemaProvider } = init({ databaseId }) + const { schemaProvider } = init({ databaseId, projectId }) const tables = await schemaProvider.list() await Promise.all(tables.map(t => t.id).map( t => schemaProvider.drop(t) )) } const initEnv = async() => { - // await runImage('') } const setActive = () => { process.env.TYPE = 'bigquery' + process.env.PROJECT_ID = projectId process.env.DATABASE_ID = databaseId } const shutdownEnv = async() => { - // await stopImage('') } -module.exports = { initEnv, shutdownEnv, setActive, connection, cleanup } \ No newline at end of file +const schemaProviderTestVariables = () => ({ projectId, databaseId }) + +module.exports = { initEnv, shutdownEnv, setActive, connection, cleanup, schemaProviderTestVariables } \ No newline at end of file diff --git a/packages/velo-external-db/test/resources/provider_resources.js b/packages/velo-external-db/test/resources/provider_resources.js index dda79120d..07fe2e269 100644 --- a/packages/velo-external-db/test/resources/provider_resources.js +++ b/packages/velo-external-db/test/resources/provider_resources.js @@ -40,7 +40,7 @@ const dbInit = async(testEnv, impl) => { const { pool, cleanup } = await testEnv.connection() const driver = impl.driver() - + env.dataProvider = new impl.DataProvider(pool, driver.filterParser) env.schemaProvider = new impl.SchemaProvider(pool, testEnv.schemaProviderTestVariables?.() ) env.driver = driver diff --git a/packages/velo-external-db/test/storage/data_provider.spec.js b/packages/velo-external-db/test/storage/data_provider.spec.js index 39deb1375..e4081abba 100644 --- a/packages/velo-external-db/test/storage/data_provider.spec.js +++ b/packages/velo-external-db/test/storage/data_provider.spec.js @@ -106,32 +106,38 @@ describe('Data API', () => { expect( await env.dataProvider.find(ctx.numericCollectionName, '', '', 0, 50) ).toEqual([ctx.numberEntity]) }) - test('delete data from collection', async() => { - await givenCollectionWith(ctx.entities, ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('delete data from collection', async() => { + await givenCollectionWith(ctx.entities, ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.delete(ctx.collectionName, ctx.entities.map(e => e._id)) ).toEqual(ctx.entities.length) + expect( await env.dataProvider.delete(ctx.collectionName, ctx.entities.map(e => e._id)) ).toEqual(ctx.entities.length) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([]) - }) + expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([]) + }) + } - test('allow update for single entity', async() => { - await givenCollectionWith([ctx.entity], ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('allow update for single entity', async() => { + await givenCollectionWith([ctx.entity], ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.update(ctx.collectionName, [ctx.modifiedEntity]) ).toEqual(1) + expect( await env.dataProvider.update(ctx.collectionName, [ctx.modifiedEntity]) ).toEqual(1) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([ctx.modifiedEntity]) - }) + expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([ctx.modifiedEntity]) + }) + } - test('allow update for multiple entities', async() => { - await givenCollectionWith(ctx.entities, ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('allow update for multiple entities', async() => { + await givenCollectionWith(ctx.entities, ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).toEqual(ctx.modifiedEntities.length) + expect( await env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).toEqual(ctx.modifiedEntities.length) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual(expect.arrayContaining(ctx.modifiedEntities)) - }) + expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual(expect.arrayContaining(ctx.modifiedEntities)) + }) + } // testt('if update does not have and updatable fields, do nothing', async () => { // await givenCollectionWith([ctx.entity], ctx.collectionName) diff --git a/packages/velo-external-db/test/storage/schema_provider.spec.js b/packages/velo-external-db/test/storage/schema_provider.spec.js index 7362f349f..1bec79213 100644 --- a/packages/velo-external-db/test/storage/schema_provider.spec.js +++ b/packages/velo-external-db/test/storage/schema_provider.spec.js @@ -1,5 +1,5 @@ const { CollectionDoesNotExists, FieldAlreadyExists, CannotModifySystemField, FieldDoesNotExist } = require('velo-external-db-commons').errors -const { Uninitialized, gen } = require('test-commons') +const { Uninitialized, gen, shouldNotRunOn } = require('test-commons') const each = require('jest-each').default const Chance = require('chance') const { env, testSuits, dbTeardown } = require('../resources/provider_resources') @@ -81,11 +81,13 @@ describe('Schema API', () => { { field: '_owner', type: 'text' }], ctx.collectionName)) }) - // eslint-disable-next-line jest/expect-expect - test('create collection twice will do nothing', async() => { - await env.schemaProvider.create(ctx.collectionName, []) - await env.schemaProvider.create(ctx.collectionName, []) - }) + if (shouldNotRunOn(['BigQuery'], name)) { + // eslint-disable-next-line jest/expect-expect + test('create collection twice will do nothing', async() => { + await env.schemaProvider.create(ctx.collectionName, []) + await env.schemaProvider.create(ctx.collectionName, []) + }) + } test('add column on a non existing collection will fail', async() => { await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(CollectionDoesNotExists) @@ -115,13 +117,15 @@ describe('Schema API', () => { ] }) }) - test('add duplicate column will fail', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('add duplicate column will fail', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) + await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) - await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(FieldAlreadyExists) - }) + await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(FieldAlreadyExists) + }) + } test('add system column will fail', async() => { await env.schemaProvider.create(ctx.collectionName, []) @@ -141,20 +145,24 @@ describe('Schema API', () => { expect((await env.schemaProvider.describeCollection(ctx.collectionName)).fields).not.toHaveProperty(ctx.columnName) }) - test('drop column on a a non existing collection', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('drop column on a a non existing collection', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - await expect(env.schemaProvider.removeColumn(ctx.collectionName, ctx.columnName)).rejects.toThrow(FieldDoesNotExist) - }) + await expect(env.schemaProvider.removeColumn(ctx.collectionName, ctx.columnName)).rejects.toThrow(FieldDoesNotExist) + }) + } - test('drop system column will fail', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('drop system column will fail', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - SystemFields.map(f => f.name) - .forEach(async f => { - await expect(env.schemaProvider.removeColumn(ctx.collectionName, f)).rejects.toThrow(CannotModifySystemField) - }) - }) + SystemFields.map(f => f.name) + .forEach(async f => { + await expect(env.schemaProvider.removeColumn(ctx.collectionName, f)).rejects.toThrow(CannotModifySystemField) + }) + }) + } const ctx = { collectionName: Uninitialized, From a4bb952c955a9d4692e5929876721a75d9e30fb4 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:21:07 +0200 Subject: [PATCH 06/47] test if in yml file --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dd087a81..32282fe00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,6 +109,7 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - id: auth + if: ${{ (1 == 1) }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} From bf2f7571c67a5303dc5459cf2509d6d079f8206b Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:27:09 +0200 Subject: [PATCH 07/47] main yml refactor to support ifs --- .github/workflows/main.yml | 42 +++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32282fe00..396718f70 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,8 +93,13 @@ jobs: test_bigquery: runs-on: ubuntu-latest + strategy: + matrix: + database: ["BigQuery"] + steps: - uses: actions/checkout@v2 + - name: Use Node.js 14 uses: actions/setup-node@v2 with: @@ -108,18 +113,34 @@ jobs: */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + - name: Downloading Docker Compose V2 + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh + docker compose version + + - name: Downloading Image + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose pull --quiet ${{ matrix.database }} + + - name: Start Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose up --detach ${{ matrix.database }} + - id: auth - if: ${{ (1 == 1) }} + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - + - name: Set up Cloud SDK + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - - - name: Use gcloud CLI - run: gcloud info - + - name: Installing Dependencies run: npm install @@ -127,4 +148,11 @@ jobs: run: npm run build:dev - name: Testing - run: npm run test:bigquery + run: npm run test:${{ matrix.database }} + + - name: Stop Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose down + From abd935b7fb8660ecac0a7b29b755947f46e5f446 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:34:33 +0200 Subject: [PATCH 08/47] main yml refactor to support ifs --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 396718f70..bc2e9aeba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -114,31 +114,31 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - name: Downloading Docker Compose V2 - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh docker compose version - name: Downloading Image - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose pull --quiet ${{ matrix.database }} - name: Start Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose up --detach ${{ matrix.database }} - id: auth - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - name: Set up Cloud SDK - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - name: Installing Dependencies @@ -151,7 +151,7 @@ jobs: run: npm run test:${{ matrix.database }} - name: Stop Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose down From 0f400b5bc0e1b2c89a4fabd74444d0ddc4051e59 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:38:30 +0200 Subject: [PATCH 09/47] main yml refactor to support ifs --- .github/workflows/main.yml | 56 ++------------------------------------ 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc2e9aeba..37543e197 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: "mongo", "mongo4", "firestore", "airtable", - "dynamodb"] + "dynamodb","bigquery"] steps: - uses: actions/checkout@v2 @@ -62,57 +62,6 @@ jobs: */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - - name: Downloading Docker Compose V2 - run: | - curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh - docker compose version - - - name: Downloading Image - run: | - cd packages/velo-external-db/test/resources - docker compose pull --quiet ${{ matrix.database }} - - - name: Start Container - run: | - cd packages/velo-external-db/test/resources - docker compose up --detach ${{ matrix.database }} - - - name: Installing Dependencies - run: npm install - - - name: Install dependencies of the homemade packages. - run: npm run build:dev - - - name: Testing - run: npm run test:${{ matrix.database }} - - - name: Stop Container - run: | - cd packages/velo-external-db/test/resources - docker compose down - - test_bigquery: - runs-on: ubuntu-latest - strategy: - matrix: - database: ["BigQuery"] - - steps: - - uses: actions/checkout@v2 - - - name: Use Node.js 14 - uses: actions/setup-node@v2 - with: - node-version: 14 - - - name: Restore Dependencies - uses: actions/cache@v2 - with: - path: | - node_modules - */*/node_modules - key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - - name: Downloading Docker Compose V2 if: ${{ ( matrix.database != 'bigquery') }} run: | @@ -140,7 +89,7 @@ jobs: - name: Set up Cloud SDK if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - + - name: Installing Dependencies run: npm install @@ -155,4 +104,3 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down - From c5f701292f8403cfe89d69b9810d95db55027e9d Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:16:29 +0200 Subject: [PATCH 10/47] testing main yml --- .github/workflows/main.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3463bf1fc..d76113a50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,3 +90,40 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down + + test_bigquery: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14 + uses: actions/setup-node@v2 + with: + node-version: 14 + + - name: Restore Dependencies + uses: actions/cache@v2 + with: + path: | + node_modules + */*/node_modules + key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + + - id: auth + uses: google-github-actions/auth@v0.4.0 + with: + credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.1 + + - name: Use gcloud CLI + run: gcloud info + + - name: Installing Dependencies + run: npm install + + - name: Install dependencies of the homemade packages. + run: npm run build:dev + + - name: Testing + run: npm run test:bigquery From b2a2419fd6cef738b41d5288ac7e98842540265c Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:50:17 +0200 Subject: [PATCH 11/47] disable other tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d76113a50..4dd087a81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,7 +121,7 @@ jobs: - name: Installing Dependencies run: npm install - + - name: Install dependencies of the homemade packages. run: npm run build:dev From c66fd653420b505261c35107679ddbd6d164400e Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 15:05:12 +0200 Subject: [PATCH 12/47] print errors --- packages/external-db-bigquery/lib/sql_exception_translator.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 41bd1c54b..3e7c5bfe5 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,6 +15,7 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { + console.log(err) throw notThrowingTranslateErrorCodes(err) } From 6d153751a0882465168706e3c71ea2c1079afe6f Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 15:43:47 +0200 Subject: [PATCH 13/47] log errors --- .../lib/bigquery_schema_provider.js | 77 ++++++++++++------- 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index 07af53982..bd82f8969 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -11,55 +11,76 @@ class SchemaProvider { } async list() { - const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') - const tables = parseTableData(res[0]) + try{ + const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') + const tables = parseTableData(res[0]) + return Object.entries(tables) + .map(([collectionName, rs]) => ({ + id: collectionName, + fields: rs.map( this.translateDbTypes.bind(this) ) + })) + } catch(e) { + console.log(e) + } - return Object.entries(tables) - .map(([collectionName, rs]) => ({ - id: collectionName, - fields: rs.map( this.translateDbTypes.bind(this) ) - })) } async create(collectionName, _columns) { - const columns = _columns || [] - const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) - await this.pool.createTable(collectionName, { schema: dbColumnsSql }) - .catch(translateErrorCodes) + try{ + const columns = _columns || [] + const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) + await this.pool.createTable(collectionName, { schema: dbColumnsSql }) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } } async drop(collectionName) { - await this.pool.table(collectionName).delete() - .catch(translateErrorCodes) + try { + await this.pool.table(collectionName).delete() + .catch(translateErrorCodes) + } catch(e) { + og.error(e) + } } async addColumn(collectionName, column) { - await validateSystemFields(column.name) - - await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) - .catch(translateErrorCodes) - + try { + await validateSystemFields(column.name) + await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } } async removeColumn(collectionName, columnName) { - await validateSystemFields(columnName) + try { + await validateSystemFields(columnName) + await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } + - await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) - .catch(translateErrorCodes) } async describeCollection(collectionName) { - const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) - - if (res[0].length === 0) { - throw new CollectionDoesNotExists('Collection does not exists') + try{ + const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) + if (res[0].length === 0) { + throw new CollectionDoesNotExists('Collection does not exists') + } + return res[0].map( this.translateDbTypes.bind(this) ) + } catch(e) { + console.log(e) } - return res[0].map( this.translateDbTypes.bind(this) ) } - - + translateDbTypes(row) { row.type = this.sqlSchemaTranslator.translateType(row.type) return row From 293d0cbf97900a44b5d72ad13d22898814ac6dde Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:00:39 +0200 Subject: [PATCH 14/47] test fefactor bigquery --- .../lib/bigquery_data_provider.js | 9 +-- .../lib/bigquery_schema_provider.js | 69 ++++++++----------- .../lib/bigquery_utils.js | 2 +- .../lib/connection_provider.js | 6 +- .../lib/sql_exception_translator.js | 1 - .../sql_filter_transformer_test_support.js | 4 +- .../lib/readers/gcp_config_reader.js | 6 +- packages/velo-external-db/package.json | 4 +- .../test/e2e/app_data.e2e.spec.js | 47 ++++++++----- .../resources/engines/bigquery_resources.js | 12 ++-- .../test/resources/provider_resources.js | 2 +- .../test/storage/data_provider.spec.js | 42 ++++++----- .../test/storage/schema_provider.spec.js | 52 ++++++++------ 13 files changed, 135 insertions(+), 121 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_data_provider.js b/packages/external-db-bigquery/lib/bigquery_data_provider.js index 828734a69..dd769879e 100644 --- a/packages/external-db-bigquery/lib/bigquery_data_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_data_provider.js @@ -34,6 +34,7 @@ class DataProvider { } async insert(collectionName, items) { + // const patchedItems = items.map(patchDateTime) // const itemsArr = patchedItems.map( item => `(${Object.values(item).map(i => typeof(i) === 'number' ? i : `"${i}"`).join(', ')})` ).join(', ') // const sql = `INSERT INTO \`${collectionName}\` (${Object.keys(items[0]).join(', ')}) VALUES ${itemsArr}` @@ -41,9 +42,9 @@ class DataProvider { // const res = await this.pool.query(sql).catch( translateErrorCodes ) const table = await this.pool.table(collectionName) - const res = await table.insert(items) - - return res + await table.insert(items) + + return items.length } async update(collectionName, items) { @@ -82,7 +83,7 @@ class DataProvider { const resultSet = await this.pool.query({ query: sql, params: [...whereParameters, ...parameters] }) .catch( translateErrorCodes ) - return resultSet[0] + return resultSet[0].map( unPatchDateTime ) } } diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index bd82f8969..b47d96d6f 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -5,7 +5,9 @@ const { SystemFields, validateSystemFields, parseTableData } = require('velo-ext const escapeIdentifier = i => i class SchemaProvider { - constructor(pool) { + constructor(pool, { projectId, databaseId }) { + this.projectId = projectId + this.databaseId = databaseId this.pool = pool this.sqlSchemaTranslator = new SchemaColumnTranslator() } @@ -18,7 +20,7 @@ class SchemaProvider { .map(([collectionName, rs]) => ({ id: collectionName, fields: rs.map( this.translateDbTypes.bind(this) ) - })) + })) } catch(e) { console.log(e) } @@ -26,61 +28,46 @@ class SchemaProvider { } async create(collectionName, _columns) { - try{ - const columns = _columns || [] - const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) - await this.pool.createTable(collectionName, { schema: dbColumnsSql }) - .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } + const columns = _columns || [] + const dbColumnsSql = [...SystemFields, ...columns].map(c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) + await this.pool.createTable(collectionName, { schema: dbColumnsSql }) + .catch(translateErrorCodes) } async drop(collectionName) { - try { - await this.pool.table(collectionName).delete() - .catch(translateErrorCodes) - } catch(e) { - og.error(e) - } + await this.pool.table(collectionName).delete() + .catch(translateErrorCodes) } - async addColumn(collectionName, column) { - try { - await validateSystemFields(column.name) + async addColumn(collectionName, column) { + await validateSystemFields(column.name) + try{ await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) - .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } + } catch (err) { + if (err.message.includes('was not found')) + throw new CollectionDoesNotExists('Collection does not exists') + else + translateErrorCodes(err) + } } async removeColumn(collectionName, columnName) { - try { - await validateSystemFields(columnName) - await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) + await validateSystemFields(columnName) + await this.pool.query(`CREATE OR REPLACE TABLE ${this.projectId}.${this.databaseId}.${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } - - - } async describeCollection(collectionName) { - try{ - const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) - if (res[0].length === 0) { - throw new CollectionDoesNotExists('Collection does not exists') - } - return res[0].map( this.translateDbTypes.bind(this) ) - } catch(e) { - console.log(e) + const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM ${this.projectId}.${this.databaseId}.INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) + + if (res[0].length === 0) { + throw new CollectionDoesNotExists('Collection does not exists') } + + return res[0].map( this.translateDbTypes.bind(this) ) } - + translateDbTypes(row) { row.type = this.sqlSchemaTranslator.translateType(row.type) return row diff --git a/packages/external-db-bigquery/lib/bigquery_utils.js b/packages/external-db-bigquery/lib/bigquery_utils.js index d9b5dd3f0..8d032c376 100644 --- a/packages/external-db-bigquery/lib/bigquery_utils.js +++ b/packages/external-db-bigquery/lib/bigquery_utils.js @@ -24,7 +24,7 @@ const unPatchDateTime = (item) => { if (reISO.test(value)) { obj[key] = new Date(value) } else { - obj[key] = item[key] + obj[key] = item[key].toNumber ? item[key].toNumber() : item[key] } } diff --git a/packages/external-db-bigquery/lib/connection_provider.js b/packages/external-db-bigquery/lib/connection_provider.js index d269a42a0..c526f5272 100644 --- a/packages/external-db-bigquery/lib/connection_provider.js +++ b/packages/external-db-bigquery/lib/connection_provider.js @@ -4,14 +4,14 @@ const DatabaseOperations = require ('./bigquery_operations') const SchemaProvider = require('./bigquery_schema_provider') const DataProvider = require('./bigquery_data_provider') -const init = ( cfg ) => { +const init = ({ projectId, databaseId }) => { const bigquery = new BigQuery() - const pool = bigquery.dataset(cfg.databaseId) + const pool = bigquery.dataset(databaseId) const filterParser = new FilterParser() const databaseOperations = new DatabaseOperations(pool) const dataProvider = new DataProvider(pool, filterParser) - const schemaProvider = new SchemaProvider(pool) + const schemaProvider = new SchemaProvider(pool, { projectId, databaseId }) return { dataProvider: dataProvider, schemaProvider: schemaProvider, databaseOperations, connection: pool, cleanup: async() => {} } } diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 3e7c5bfe5..41bd1c54b 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,7 +15,6 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { - console.log(err) throw notThrowingTranslateErrorCodes(err) } diff --git a/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js b/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js index f2c667530..79a71f8de 100644 --- a/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js +++ b/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js @@ -33,11 +33,11 @@ const givenOrderByFor = (column, sort) => { const givenFilterByIdWith = (id, filter) => { when(filterParser.transform).calledWith(filter) - .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = $1`, parameters: [id], offset: 2 }) + .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = ?`, parameters: [id], offset: 2 }) } const givenAggregateQueryWith = (having, numericColumns, columnAliases, groupByColumns, filter, offest) => { - when(filterParser.parseAggregation).calledWith(having, filter, offest) + when(filterParser.parseAggregation).calledWith(having, filter) .mockReturnValue({ fieldsStatement: `${groupByColumns.map( escapeIdentifier ).join(', ')}, MAX(${escapeIdentifier(numericColumns[0].name)}) AS ${escapeIdentifier(columnAliases[0])}, SUM(${escapeIdentifier(numericColumns[1].name)}) AS ${escapeIdentifier(columnAliases[1])}`, groupByColumns: groupByColumns, diff --git a/packages/external-db-config/lib/readers/gcp_config_reader.js b/packages/external-db-config/lib/readers/gcp_config_reader.js index 9a644a648..6c7c9936d 100644 --- a/packages/external-db-config/lib/readers/gcp_config_reader.js +++ b/packages/external-db-config/lib/readers/gcp_config_reader.js @@ -99,13 +99,13 @@ class GcpBigQueryConfigReader { } async readConfig() { - const { DATABASE_ID, SECRET_KEY } = process.env - return { databaseId: DATABASE_ID, secretKey: SECRET_KEY } + const { PROJECT_ID, DATABASE_ID, SECRET_KEY } = process.env + return { projectId: PROJECT_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY } } validate() { return { - missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['DATABASE_ID', 'SECRET_KEY']) + missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['PROJECT_ID', 'DATABASE_ID', 'SECRET_KEY']) } } diff --git a/packages/velo-external-db/package.json b/packages/velo-external-db/package.json index 25235f1b6..a044dc56d 100644 --- a/packages/velo-external-db/package.json +++ b/packages/velo-external-db/package.json @@ -11,10 +11,10 @@ "test:firestore": "TEST_ENGINE=firestore jest --runInBand ./test", "test:mssql": "TEST_ENGINE=mssql jest --runInBand ./test", "test:mongo": "TEST_ENGINE=mongo jest --runInBand ./test", - "test:google-sheets": "TEST_ENGINE=google-sheet jest -i packages/velo-external-db/test/e2e/app_data.e2e.spec.js --runInBand -t 'Velo External DB Data REST API' ", + "test:google-sheets": "TEST_ENGINE=google-sheet jest -i packages/velo-external-db/test/e2e/app_data.e2e.spec.js --runInBand -t 'Schema API' ", "test:airtable": "TEST_ENGINE=airtable jest --runInBand ./test ", "test:dynamodb": "TEST_ENGINE=dynamodb jest --runInBand ./test ", - "test:bigquery": "TEST_ENGINE=bigquery jest --runInBand -i packages/velo-external-db/test/e2e/app_schema.e2e.spec.js -t 'Velo External DB Schema REST API' ", + "test:bigquery": "TEST_ENGINE=bigquery jest --runInBand ./test", "test:watch": "jest --detectOpenHandles --watch", "start": "node lib/app.js", "deploy": "gcloud app deploy ./app.yaml" diff --git a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js index e57f3201a..21144ae4f 100644 --- a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js +++ b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js @@ -83,15 +83,19 @@ describe('Velo External DB Data REST API', () => { }) } + + if (shouldNotRunOn(['BigQuery'], name)) { test('delete one api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) + await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) - }) + expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) + }) + } + if (shouldNotRunOn(['BigQuery'], name)) { test('bulk delete api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems(ctx.items, ctx.collectionName, authAdmin) @@ -100,31 +104,38 @@ describe('Velo External DB Data REST API', () => { expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) }) + } + if (shouldNotRunOn(['BigQuery'], name)) { test('get by id api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item], ctx.collectionName, authAdmin) expect((await axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin)).data).toEqual({ item: ctx.item }) }) + } + + if (shouldNotRunOn(['BigQuery'], name)) { + test('update api', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - test('update api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - - await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) + await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) - }) + expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) + }) + } - test('bulk update api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems(ctx.items, ctx.collectionName, authAdmin) + if (shouldNotRunOn(['BigQuery'], name)) { + test('bulk update api', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems(ctx.items, ctx.collectionName, authAdmin) - await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) + await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) - expect((await data.expectAllDataIn(ctx.collectionName, authAdmin)).items).toEqual(expect.arrayContaining(ctx.modifiedItems)) - }) + expect((await data.expectAllDataIn(ctx.collectionName, authAdmin)).items).toEqual(expect.arrayContaining(ctx.modifiedItems)) + }) + } test('count api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) diff --git a/packages/velo-external-db/test/resources/engines/bigquery_resources.js b/packages/velo-external-db/test/resources/engines/bigquery_resources.js index 4d7a7ecd5..2930f7aa5 100644 --- a/packages/velo-external-db/test/resources/engines/bigquery_resources.js +++ b/packages/velo-external-db/test/resources/engines/bigquery_resources.js @@ -1,29 +1,31 @@ const { init } = require('external-db-bigquery') const databaseId = 'testDB' +const projectId = 'corvid-managed-cfe9809c' const connection = () => { - const { connection, cleanup } = init({ databaseId }) + const { connection, cleanup } = init({ databaseId, projectId }) return { pool: connection, cleanup } } const cleanup = async() => { - const { schemaProvider } = init({ databaseId }) + const { schemaProvider } = init({ databaseId, projectId }) const tables = await schemaProvider.list() await Promise.all(tables.map(t => t.id).map( t => schemaProvider.drop(t) )) } const initEnv = async() => { - // await runImage('') } const setActive = () => { process.env.TYPE = 'bigquery' + process.env.PROJECT_ID = projectId process.env.DATABASE_ID = databaseId } const shutdownEnv = async() => { - // await stopImage('') } -module.exports = { initEnv, shutdownEnv, setActive, connection, cleanup } \ No newline at end of file +const schemaProviderTestVariables = () => ({ projectId, databaseId }) + +module.exports = { initEnv, shutdownEnv, setActive, connection, cleanup, schemaProviderTestVariables } \ No newline at end of file diff --git a/packages/velo-external-db/test/resources/provider_resources.js b/packages/velo-external-db/test/resources/provider_resources.js index dda79120d..07fe2e269 100644 --- a/packages/velo-external-db/test/resources/provider_resources.js +++ b/packages/velo-external-db/test/resources/provider_resources.js @@ -40,7 +40,7 @@ const dbInit = async(testEnv, impl) => { const { pool, cleanup } = await testEnv.connection() const driver = impl.driver() - + env.dataProvider = new impl.DataProvider(pool, driver.filterParser) env.schemaProvider = new impl.SchemaProvider(pool, testEnv.schemaProviderTestVariables?.() ) env.driver = driver diff --git a/packages/velo-external-db/test/storage/data_provider.spec.js b/packages/velo-external-db/test/storage/data_provider.spec.js index 39deb1375..e4081abba 100644 --- a/packages/velo-external-db/test/storage/data_provider.spec.js +++ b/packages/velo-external-db/test/storage/data_provider.spec.js @@ -106,32 +106,38 @@ describe('Data API', () => { expect( await env.dataProvider.find(ctx.numericCollectionName, '', '', 0, 50) ).toEqual([ctx.numberEntity]) }) - test('delete data from collection', async() => { - await givenCollectionWith(ctx.entities, ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('delete data from collection', async() => { + await givenCollectionWith(ctx.entities, ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.delete(ctx.collectionName, ctx.entities.map(e => e._id)) ).toEqual(ctx.entities.length) + expect( await env.dataProvider.delete(ctx.collectionName, ctx.entities.map(e => e._id)) ).toEqual(ctx.entities.length) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([]) - }) + expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([]) + }) + } - test('allow update for single entity', async() => { - await givenCollectionWith([ctx.entity], ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('allow update for single entity', async() => { + await givenCollectionWith([ctx.entity], ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.update(ctx.collectionName, [ctx.modifiedEntity]) ).toEqual(1) + expect( await env.dataProvider.update(ctx.collectionName, [ctx.modifiedEntity]) ).toEqual(1) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([ctx.modifiedEntity]) - }) + expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([ctx.modifiedEntity]) + }) + } - test('allow update for multiple entities', async() => { - await givenCollectionWith(ctx.entities, ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('allow update for multiple entities', async() => { + await givenCollectionWith(ctx.entities, ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).toEqual(ctx.modifiedEntities.length) + expect( await env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).toEqual(ctx.modifiedEntities.length) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual(expect.arrayContaining(ctx.modifiedEntities)) - }) + expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual(expect.arrayContaining(ctx.modifiedEntities)) + }) + } // testt('if update does not have and updatable fields, do nothing', async () => { // await givenCollectionWith([ctx.entity], ctx.collectionName) diff --git a/packages/velo-external-db/test/storage/schema_provider.spec.js b/packages/velo-external-db/test/storage/schema_provider.spec.js index aa0bb15f5..18ae45bc6 100644 --- a/packages/velo-external-db/test/storage/schema_provider.spec.js +++ b/packages/velo-external-db/test/storage/schema_provider.spec.js @@ -1,5 +1,5 @@ const { CollectionDoesNotExists, FieldAlreadyExists, CannotModifySystemField, FieldDoesNotExist } = require('velo-external-db-commons').errors -const { Uninitialized, gen } = require('test-commons') +const { Uninitialized, gen, shouldNotRunOn } = require('test-commons') const each = require('jest-each').default const Chance = require('chance') const { env, testSuits, dbTeardown } = require('../resources/provider_resources') @@ -75,11 +75,13 @@ describe('Schema API', () => { expect(db).toEqual(collectionWithDefaultFields()) }) - // eslint-disable-next-line jest/expect-expect - test('create collection twice will do nothing', async() => { - await env.schemaProvider.create(ctx.collectionName, []) - await env.schemaProvider.create(ctx.collectionName, []) - }) + if (shouldNotRunOn(['BigQuery'], name)) { + // eslint-disable-next-line jest/expect-expect + test('create collection twice will do nothing', async() => { + await env.schemaProvider.create(ctx.collectionName, []) + await env.schemaProvider.create(ctx.collectionName, []) + }) + } test('add column on a non existing collection will fail', async() => { await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(CollectionDoesNotExists) @@ -94,13 +96,15 @@ describe('Schema API', () => { expect(db).toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }])) }) - test('add duplicate column will fail', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('add duplicate column will fail', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) + await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) - await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(FieldAlreadyExists) - }) + await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(FieldAlreadyExists) + }) + } test('add system column will fail', async() => { await env.schemaProvider.create(ctx.collectionName, []) @@ -119,20 +123,24 @@ describe('Schema API', () => { expect((await env.schemaProvider.describeCollection(ctx.collectionName))).not.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }]) ) }) - test('drop column on a a non existing collection', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('drop column on a a non existing collection', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - await expect(env.schemaProvider.removeColumn(ctx.collectionName, ctx.columnName)).rejects.toThrow(FieldDoesNotExist) - }) + await expect(env.schemaProvider.removeColumn(ctx.collectionName, ctx.columnName)).rejects.toThrow(FieldDoesNotExist) + }) + } - test('drop system column will fail', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('drop system column will fail', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - SystemFields.map(f => f.name) - .forEach(async f => { - await expect(env.schemaProvider.removeColumn(ctx.collectionName, f)).rejects.toThrow(CannotModifySystemField) - }) - }) + SystemFields.map(f => f.name) + .forEach(async f => { + await expect(env.schemaProvider.removeColumn(ctx.collectionName, f)).rejects.toThrow(CannotModifySystemField) + }) + }) + } const ctx = { collectionName: Uninitialized, From 2aa18ee932a801d7327b672f1ce1a7b170e265da Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:21:07 +0200 Subject: [PATCH 15/47] test if in yml file --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dd087a81..32282fe00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,6 +109,7 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - id: auth + if: ${{ (1 == 1) }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} From a4c58f1ad561c71aefac6f7bf032a9babfdf61d1 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:27:09 +0200 Subject: [PATCH 16/47] main yml refactor to support ifs --- .github/workflows/main.yml | 42 +++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32282fe00..396718f70 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,8 +93,13 @@ jobs: test_bigquery: runs-on: ubuntu-latest + strategy: + matrix: + database: ["BigQuery"] + steps: - uses: actions/checkout@v2 + - name: Use Node.js 14 uses: actions/setup-node@v2 with: @@ -108,18 +113,34 @@ jobs: */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + - name: Downloading Docker Compose V2 + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh + docker compose version + + - name: Downloading Image + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose pull --quiet ${{ matrix.database }} + + - name: Start Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose up --detach ${{ matrix.database }} + - id: auth - if: ${{ (1 == 1) }} + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - + - name: Set up Cloud SDK + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - - - name: Use gcloud CLI - run: gcloud info - + - name: Installing Dependencies run: npm install @@ -127,4 +148,11 @@ jobs: run: npm run build:dev - name: Testing - run: npm run test:bigquery + run: npm run test:${{ matrix.database }} + + - name: Stop Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose down + From b2c7b5e41fbbdcdfc4b73eca9cdadef61e684cba Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:34:33 +0200 Subject: [PATCH 17/47] main yml refactor to support ifs --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 396718f70..bc2e9aeba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -114,31 +114,31 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - name: Downloading Docker Compose V2 - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh docker compose version - name: Downloading Image - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose pull --quiet ${{ matrix.database }} - name: Start Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose up --detach ${{ matrix.database }} - id: auth - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - name: Set up Cloud SDK - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - name: Installing Dependencies @@ -151,7 +151,7 @@ jobs: run: npm run test:${{ matrix.database }} - name: Stop Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose down From ff063d82b43be375181b0439170b44085b7f9581 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:38:30 +0200 Subject: [PATCH 18/47] main yml refactor to support ifs --- .github/workflows/main.yml | 56 ++------------------------------------ 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc2e9aeba..37543e197 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: "mongo", "mongo4", "firestore", "airtable", - "dynamodb"] + "dynamodb","bigquery"] steps: - uses: actions/checkout@v2 @@ -62,57 +62,6 @@ jobs: */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - - name: Downloading Docker Compose V2 - run: | - curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh - docker compose version - - - name: Downloading Image - run: | - cd packages/velo-external-db/test/resources - docker compose pull --quiet ${{ matrix.database }} - - - name: Start Container - run: | - cd packages/velo-external-db/test/resources - docker compose up --detach ${{ matrix.database }} - - - name: Installing Dependencies - run: npm install - - - name: Install dependencies of the homemade packages. - run: npm run build:dev - - - name: Testing - run: npm run test:${{ matrix.database }} - - - name: Stop Container - run: | - cd packages/velo-external-db/test/resources - docker compose down - - test_bigquery: - runs-on: ubuntu-latest - strategy: - matrix: - database: ["BigQuery"] - - steps: - - uses: actions/checkout@v2 - - - name: Use Node.js 14 - uses: actions/setup-node@v2 - with: - node-version: 14 - - - name: Restore Dependencies - uses: actions/cache@v2 - with: - path: | - node_modules - */*/node_modules - key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - - name: Downloading Docker Compose V2 if: ${{ ( matrix.database != 'bigquery') }} run: | @@ -140,7 +89,7 @@ jobs: - name: Set up Cloud SDK if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - + - name: Installing Dependencies run: npm install @@ -155,4 +104,3 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down - From aacef07de1b5057b03b5ee754ba180f9f6d9c004 Mon Sep 17 00:00:00 2001 From: Noam Almog Date: Wed, 15 Dec 2021 17:23:43 +0200 Subject: [PATCH 19/47] removed dead code --- .../lib/schema_common.spec.js | 38 ------------------- .../lib/schema_commons.js | 22 +---------- .../lib/service/schema_utils.js | 10 ----- 3 files changed, 1 insertion(+), 69 deletions(-) delete mode 100644 packages/velo-external-db-commons/lib/schema_common.spec.js delete mode 100644 packages/velo-external-db-core/lib/service/schema_utils.js diff --git a/packages/velo-external-db-commons/lib/schema_common.spec.js b/packages/velo-external-db-commons/lib/schema_common.spec.js deleted file mode 100644 index a6254efa3..000000000 --- a/packages/velo-external-db-commons/lib/schema_common.spec.js +++ /dev/null @@ -1,38 +0,0 @@ -const { fieldsWithoutSubType } = require('./schema_commons') -const { Uninitialized, gen } = require('test-commons') - -describe('Schema common', () => { - test('fieldsWithoutSubType should remove subtype from fields', () => { - const fields = { - [ctx.fieldName]: { - displayName: ctx.fieldName, - type: 'type', - subtype: 'subtype', - }, - [ctx.anotherFieldName]: { - displayName: ctx.anotherFieldName, - type: 'type', - }, - } - expect(fieldsWithoutSubType(fields)).toEqual({ - [ctx.fieldName]: { - displayName: ctx.fieldName, - type: 'type', - }, - [ctx.anotherFieldName]: { - displayName: ctx.anotherFieldName, - type: 'type', - } - }) - }) - - const ctx = { - fieldName: Uninitialized, - anotherFieldName: Uninitialized, - } - - beforeEach(() => { - ctx.fieldName = gen.randomFieldName() - ctx.anotherFieldName = gen.randomFieldName() - }) -}) \ No newline at end of file diff --git a/packages/velo-external-db-commons/lib/schema_commons.js b/packages/velo-external-db-commons/lib/schema_commons.js index b833bda29..af876dba0 100644 --- a/packages/velo-external-db-commons/lib/schema_commons.js +++ b/packages/velo-external-db-commons/lib/schema_commons.js @@ -32,7 +32,6 @@ const asWixSchema = (fields, collectionName) => { fields: fields.reduce( (o, r) => ( { ...o, [r.field]: { displayName: r.field, type: r.type, - // subtype: r.subtype, queryOperators: [ 'eq', 'lt', @@ -65,23 +64,4 @@ const parseTableData = data => data.reduce((o, r) => { return o }, {}) - - -const schemasWithoutSubtype = schemas => ( - schemas.map(({ fields, ...rest }) => ({ - ...rest, - fields: fieldsWithoutSubType(fields) - }) - ) -) - -const fieldsWithoutSubType = (fields) => { - return Object.entries(fields) - .reduce((pV, [k, v]) => { - const { subtype, ...rest } = v - return { ...pV, ...{ [k]: rest } } - }, {}) -} - - -module.exports = { SystemFields, asWixSchema, validateSystemFields, parseTableData, schemasWithoutSubtype, fieldsWithoutSubType } \ No newline at end of file +module.exports = { SystemFields, asWixSchema, validateSystemFields, parseTableData } \ No newline at end of file diff --git a/packages/velo-external-db-core/lib/service/schema_utils.js b/packages/velo-external-db-core/lib/service/schema_utils.js deleted file mode 100644 index a5196e222..000000000 --- a/packages/velo-external-db-core/lib/service/schema_utils.js +++ /dev/null @@ -1,10 +0,0 @@ - -const fieldsWithoutSubType = (fields) => { - return Object.entries(fields) - .reduce((pV, [k, v]) => { - const { subtype, ...rest } = v - return { ...pV, ...{ [k]: rest } } - }, {}) -} - -module.exports = { fieldsWithoutSubType } \ No newline at end of file From 3ef66c4a9fa9747db95f686ff8f37cfa90b1ff73 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Wed, 15 Dec 2021 17:25:31 +0200 Subject: [PATCH 20/47] minor changes --- .../lib/bigquery_data_provider.js | 7 ------- .../lib/bigquery_schema_provider.js | 19 +++++++------------ 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_data_provider.js b/packages/external-db-bigquery/lib/bigquery_data_provider.js index dd769879e..d54dc5231 100644 --- a/packages/external-db-bigquery/lib/bigquery_data_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_data_provider.js @@ -34,13 +34,6 @@ class DataProvider { } async insert(collectionName, items) { - - // const patchedItems = items.map(patchDateTime) - // const itemsArr = patchedItems.map( item => `(${Object.values(item).map(i => typeof(i) === 'number' ? i : `"${i}"`).join(', ')})` ).join(', ') - // const sql = `INSERT INTO \`${collectionName}\` (${Object.keys(items[0]).join(', ')}) VALUES ${itemsArr}` - - // const res = await this.pool.query(sql).catch( translateErrorCodes ) - const table = await this.pool.table(collectionName) await table.insert(items) diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index b47d96d6f..54c10d853 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -13,18 +13,13 @@ class SchemaProvider { } async list() { - try{ - const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') - const tables = parseTableData(res[0]) - return Object.entries(tables) - .map(([collectionName, rs]) => ({ - id: collectionName, - fields: rs.map( this.translateDbTypes.bind(this) ) - })) - } catch(e) { - console.log(e) - } - + const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') + const tables = parseTableData(res[0]) + return Object.entries(tables) + .map(([collectionName, rs]) => ({ + id: collectionName, + fields: rs.map( this.translateDbTypes.bind(this) ) + })) } async create(collectionName, _columns) { From 2a3274641147cb825ec444108977e3f0151a11bb Mon Sep 17 00:00:00 2001 From: Noam Almog Date: Thu, 16 Dec 2021 11:47:14 +0200 Subject: [PATCH 21/47] refactored data to use schemaFieldsFor instead of parsing schema on data.js --- .../velo-external-db-core/lib/service/data.js | 12 +++++----- .../lib/service/schema_information.js | 8 +++++-- .../lib/service/schema_information.spec.js | 23 ++++++++++--------- .../schema_information_test_support.js | 8 +++---- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/packages/velo-external-db-core/lib/service/data.js b/packages/velo-external-db-core/lib/service/data.js index 75b59bb30..58664244f 100644 --- a/packages/velo-external-db-core/lib/service/data.js +++ b/packages/velo-external-db-core/lib/service/data.js @@ -33,9 +33,9 @@ class DataService { } async bulkInsert(collectionName, items) { - const info = await this.schemaInformation.schemaFor(collectionName) - const prepared = items.map(i => prepareForInsert(i, info.fields)) - await this.storage.insert(collectionName, prepared.map(i => unpackDates(i)), info.fields) + const fields = await this.schemaInformation.schemaFieldsFor(collectionName) + const prepared = items.map(i => prepareForInsert(i, fields)) + await this.storage.insert(collectionName, prepared.map(i => unpackDates(i)), fields) return { items: prepared } } @@ -45,9 +45,9 @@ class DataService { } async bulkUpdate(collectionName, items) { - const info = await this.schemaInformation.schemaFor(collectionName) - const prepared = items.map(i => prepareForUpdate(i, info.fields)) - await this.storage.update(collectionName, prepared.map(i => unpackDates(i)), info.fields) + const fields = await this.schemaInformation.schemaFieldsFor(collectionName) + const prepared = items.map(i => prepareForUpdate(i, fields)) + await this.storage.update(collectionName, prepared.map(i => unpackDates(i)), fields) return { items: prepared } } diff --git a/packages/velo-external-db-core/lib/service/schema_information.js b/packages/velo-external-db-core/lib/service/schema_information.js index c7005af00..3a7993c4e 100644 --- a/packages/velo-external-db-core/lib/service/schema_information.js +++ b/packages/velo-external-db-core/lib/service/schema_information.js @@ -19,8 +19,8 @@ class CacheableSchemaInformation { } async schemaFieldsFor(collectionName) { - const schema = await this.schemaFor(collectionName) - return schema.fields + const s = await this.schemaFor(collectionName) + return s.fields } async schema() { @@ -36,6 +36,10 @@ class CacheableSchemaInformation { const schema = await this.schemaProvider.list() this.cache.set(CacheKey, schema) } + + async clear() { + this.cache.flushAll() + } } module.exports = CacheableSchemaInformation \ No newline at end of file diff --git a/packages/velo-external-db-core/lib/service/schema_information.spec.js b/packages/velo-external-db-core/lib/service/schema_information.spec.js index dee02730f..b679dd74b 100644 --- a/packages/velo-external-db-core/lib/service/schema_information.spec.js +++ b/packages/velo-external-db-core/lib/service/schema_information.spec.js @@ -17,17 +17,17 @@ describe('Schema Information Service', () => { await expect( env.schemaInformation.schemaFor(ctx.collectionName) ).rejects.toThrow(CollectionDoesNotExists) }) - // test('will automatically refresh and return schema fields for collection when queried', async() => { - // driver.givenListResult(ctx.dbs) - // - // await expect( () => env.schemaInformation.schemaFieldsFor(ctx.dbs[0].id) ).resolves.toEqual(ctx.dbs[0].fields) - // }) - // - // test('retrieve collection fields if it does not exists, throw an exception', async() => { - // driver.givenListResult([]) - // - // await expect( async () => env.schemaInformation.schemaFieldsFor(ctx.collectionName) ).rejects.toThrow(CollectionDoesNotExists) - // }) + test('will automatically refresh and return schema fields for collection when queried', async() => { + driver.givenListResult(ctx.dbs) + + await expect( env.schemaInformation.schemaFieldsFor(ctx.dbs[0].id) ).resolves.toEqual(ctx.dbs[0].fields) + }) + + test('retrieve collection fields if it does not exists, throw an exception', async() => { + driver.givenListResult([]) + + await expect(env.schemaInformation.schemaFieldsFor(ctx.collectionName)).rejects.toThrow(CollectionDoesNotExists) + }) test('force refresh will invalidate cache', async() => { driver.givenListResult([]) @@ -54,6 +54,7 @@ describe('Schema Information Service', () => { beforeEach(() => { driver.reset() + env.schemaInformation.clear() ctx.dbs = gen.randomDbs() ctx.collectionName = gen.randomCollectionName() diff --git a/packages/velo-external-db-core/test/drivers/schema_information_test_support.js b/packages/velo-external-db-core/test/drivers/schema_information_test_support.js index 137de16df..e66361dcb 100644 --- a/packages/velo-external-db-core/test/drivers/schema_information_test_support.js +++ b/packages/velo-external-db-core/test/drivers/schema_information_test_support.js @@ -2,13 +2,13 @@ const { SystemFields } = require('velo-external-db-commons') const { when } = require('jest-when') const schemaInformation = { - schemaFor: jest.fn(), + schemaFieldsFor: jest.fn(), refresh: jest.fn(), } const givenDefaultSchemaFor = collectionName => { - when(schemaInformation.schemaFor).calledWith(collectionName) - .mockResolvedValue( { id: collectionName, fields: SystemFields.map(({ name, type, subtype }) => ({ field: name, type, subtype }), {}) }) + when(schemaInformation.schemaFieldsFor).calledWith(collectionName) + .mockResolvedValue( SystemFields.map(({ name, type, subtype }) => ({ field: name, type, subtype }) ) ) } const expectSchemaRefresh = () => @@ -16,7 +16,7 @@ const expectSchemaRefresh = () => const reset = () => { - schemaInformation.schemaFor.mockClear() + schemaInformation.schemaFieldsFor.mockClear() schemaInformation.refresh.mockClear() } From ffcf35b56e6be3d99ef1aa40488da87604c82362 Mon Sep 17 00:00:00 2001 From: Noam Almog Date: Thu, 16 Dec 2021 05:42:05 -0800 Subject: [PATCH 22/47] refactor tests for schema (#101) --- .../lib/service/schema.js | 3 + .../test/drivers/schema_api_rest_matchers.js | 35 ++++++ .../drivers/schema_api_rest_test_support.js | 101 +----------------- .../test/e2e/app_schema.e2e.spec.js | 35 ++---- .../test/storage/schema_provider.spec.js | 6 +- 5 files changed, 49 insertions(+), 131 deletions(-) create mode 100644 packages/velo-external-db/test/drivers/schema_api_rest_matchers.js diff --git a/packages/velo-external-db-core/lib/service/schema.js b/packages/velo-external-db-core/lib/service/schema.js index b81d5a54f..b57995fa1 100644 --- a/packages/velo-external-db-core/lib/service/schema.js +++ b/packages/velo-external-db-core/lib/service/schema.js @@ -19,16 +19,19 @@ class SchemaService { async create(collectionName) { await this.storage.create(collectionName) await this.schemaInformation.refresh() + return {} } async addColumn(collectionName, column) { await this.storage.addColumn(collectionName, column) await this.schemaInformation.refresh() + return {} } async removeColumn(collectionName, columnName) { await this.storage.removeColumn(collectionName, columnName) await this.schemaInformation.refresh() + return {} } } diff --git a/packages/velo-external-db/test/drivers/schema_api_rest_matchers.js b/packages/velo-external-db/test/drivers/schema_api_rest_matchers.js new file mode 100644 index 000000000..0ae9d11a0 --- /dev/null +++ b/packages/velo-external-db/test/drivers/schema_api_rest_matchers.js @@ -0,0 +1,35 @@ +const { SystemFields } = require('velo-external-db-commons') + +const hasSameSchemaFieldsLike = fields => expect.objectContaining( fields.reduce((o, f) => ( { ...o, [f.name]: expect.objectContaining({ displayName: f.name, type: f.type }) } ), {}) ) + +const schemaWithDefaultFieldsFor = collectionName => expect.objectContaining({ id: collectionName, + displayName: collectionName, + fields: hasSameSchemaFieldsLike( SystemFields ) }) + +const schemaWithFields = fields => expect.objectContaining({ fields: hasSameSchemaFieldsLike( fields ) }) + +const collectionWithDefaultFieldsFor = collectionName => expect.objectContaining( { + schemas: expect.arrayContaining( [ schemaWithDefaultFieldsFor( collectionName ) ] ) +} ) + +const collectionResponseWithDefaultFieldsFor = collectionName => expect.objectContaining( { + data: collectionWithDefaultFieldsFor( collectionName ) +} ) + +const collectionResponseHasField = field => expect.objectContaining( { + data: collectionHasField( field ) +} ) + +const collectionResponseWithNoCollections = () => expect.objectContaining( { + data: toHaveNoCollections() +} ) + +const collectionHasField = field => expect.objectContaining( { + schemas: expect.arrayContaining( [ schemaWithFields( [field] ) ] ) +} ) + +const toHaveNoCollections = () => expect.objectContaining( { + schemas: [ ] +} ) + +module.exports = { collectionResponseWithDefaultFieldsFor, collectionResponseHasField, collectionResponseWithNoCollections } \ No newline at end of file diff --git a/packages/velo-external-db/test/drivers/schema_api_rest_test_support.js b/packages/velo-external-db/test/drivers/schema_api_rest_test_support.js index 5522eaf13..08c2fe287 100644 --- a/packages/velo-external-db/test/drivers/schema_api_rest_test_support.js +++ b/packages/velo-external-db/test/drivers/schema_api_rest_test_support.js @@ -2,7 +2,6 @@ const axios = require('axios').create({ baseURL: 'http://localhost:8080' }) - const givenCollection = async(name, columns, auth) => { await axios.post('/schemas/create', { collectionName: name }, auth) for (const column of columns) { @@ -10,102 +9,6 @@ const givenCollection = async(name, columns, auth) => { } } -const expectColumnInCollection = async(columnName, collectionName, auth) => { - const dbs = (await axios.post('/schemas/list', {}, auth)).data.schemas - const field = dbs.find(e => e.id === collectionName) - .fields[columnName] - return field -} - -const expectDefaultCollectionWith = (collectionName, res) => { - expect(res.data).toEqual({ schemas: [{ id: collectionName, - displayName: collectionName, - allowedOperations: [ - 'get', - 'find', - 'count', - 'update', - 'insert', - 'remove' - ], - maxPageSize: 50, - ttl: 3600, - fields: { - _id: { - displayName: '_id', - type: 'text', - queryOperators: [ - 'eq', - 'lt', - 'gt', - 'hasSome', - 'and', - 'lte', - 'gte', - 'or', - 'not', - 'ne', - 'startsWith', - 'endsWith' - ] - }, - _createdDate: { - displayName: '_createdDate', - type: 'datetime', - queryOperators: [ - 'eq', - 'lt', - 'gt', - 'hasSome', - 'and', - 'lte', - 'gte', - 'or', - 'not', - 'ne', - 'startsWith', - 'endsWith' - ] - }, - _updatedDate: { - displayName: '_updatedDate', - type: 'datetime', - queryOperators: [ - 'eq', - 'lt', - 'gt', - 'hasSome', - 'and', - 'lte', - 'gte', - 'or', - 'not', - 'ne', - 'startsWith', - 'endsWith' - ] - }, - _owner: { - displayName: '_owner', - type: 'text', - queryOperators: [ - 'eq', - 'lt', - 'gt', - 'hasSome', - 'and', - 'lte', - 'gte', - 'or', - 'not', - 'ne', - 'startsWith', - 'endsWith' - ] - }, - } - }] }) - -} +const retrieveSchemaFor = async(collectionName, auth) => axios.post('/schemas/find', { schemaIds: [collectionName] }, auth) -module.exports = { givenCollection, expectColumnInCollection, expectDefaultCollectionWith } \ No newline at end of file +module.exports = { givenCollection, retrieveSchemaFor } \ No newline at end of file diff --git a/packages/velo-external-db/test/e2e/app_schema.e2e.spec.js b/packages/velo-external-db/test/e2e/app_schema.e2e.spec.js index 2ef9821a8..dc3ea78eb 100644 --- a/packages/velo-external-db/test/e2e/app_schema.e2e.spec.js +++ b/packages/velo-external-db/test/e2e/app_schema.e2e.spec.js @@ -1,5 +1,6 @@ const { Uninitialized, gen } = require('test-commons') const schema = require('../drivers/schema_api_rest_test_support') +const matchers = require('../drivers/schema_api_rest_matchers') const { authOwner } = require('../drivers/auth_test_support') const Chance = require('chance') const each = require('jest-each').default @@ -23,23 +24,19 @@ describe('Velo External DB Schema REST API', () => { }, 20000) test('list', async() => { - expect((await axios.post('/schemas/list', {}, authOwner)).data).toEqual({ schemas: [] }) + await expect( axios.post('/schemas/list', {}, authOwner) ).resolves.toEqual( matchers.collectionResponseWithNoCollections() ) }) - // eslint-disable-next-line jest/expect-expect test('create', async() => { - await schema.givenCollection(ctx.collectionName, [], authOwner) + await axios.post('/schemas/create', { collectionName: ctx.collectionName }, authOwner) - const res = await axios.post('/schemas/list', {}, authOwner) - schema.expectDefaultCollectionWith(ctx.collectionName, res) + await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.toEqual( matchers.collectionResponseWithDefaultFieldsFor(ctx.collectionName) ) }) - // eslint-disable-next-line jest/expect-expect test('find', async() => { await schema.givenCollection(ctx.collectionName, [], authOwner) - const res = await axios.post('/schemas/find', { schemaIds: [ctx.collectionName] }, authOwner) - schema.expectDefaultCollectionWith(ctx.collectionName, res) + await expect( axios.post('/schemas/find', { schemaIds: [ctx.collectionName] }, authOwner)).resolves.toEqual( matchers.collectionResponseWithDefaultFieldsFor(ctx.collectionName) ) }) test('add column', async() => { @@ -47,23 +44,7 @@ describe('Velo External DB Schema REST API', () => { await axios.post('/schemas/column/add', { collectionName: ctx.collectionName, column: ctx.column }, authOwner) - const field = await schema.expectColumnInCollection(ctx.column.name, ctx.collectionName, authOwner) - expect(field).toEqual({ displayName: ctx.column.name, - type: 'text', - queryOperators: [ - 'eq', - 'lt', - 'gt', - 'hasSome', - 'and', - 'lte', - 'gte', - 'or', - 'not', - 'ne', - 'startsWith', - 'endsWith', - ] }) + await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.toEqual( matchers.collectionResponseHasField( ctx.column ) ) }) test('remove column', async() => { @@ -71,10 +52,8 @@ describe('Velo External DB Schema REST API', () => { await axios.post('/schemas/column/remove', { collectionName: ctx.collectionName, columnName: ctx.column.name }, authOwner) - const field = await schema.expectColumnInCollection(ctx.column.name, ctx.collectionName, authOwner) - expect(field).toBeUndefined() + await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.not.toEqual( matchers.collectionResponseHasField( ctx.column ) ) }) - }) diff --git a/packages/velo-external-db/test/storage/schema_provider.spec.js b/packages/velo-external-db/test/storage/schema_provider.spec.js index aa0bb15f5..1ab1841ee 100644 --- a/packages/velo-external-db/test/storage/schema_provider.spec.js +++ b/packages/velo-external-db/test/storage/schema_provider.spec.js @@ -66,8 +66,6 @@ describe('Schema API', () => { expect(db).toEqual(collectionWithDefaultFields()) }) - - test('retrieve collection data by collection name', async() => { await env.schemaProvider.create(ctx.collectionName) @@ -75,10 +73,10 @@ describe('Schema API', () => { expect(db).toEqual(collectionWithDefaultFields()) }) - // eslint-disable-next-line jest/expect-expect test('create collection twice will do nothing', async() => { await env.schemaProvider.create(ctx.collectionName, []) - await env.schemaProvider.create(ctx.collectionName, []) + + await expect( env.schemaProvider.create(ctx.collectionName, []) ).resolves.toBeUndefined() }) test('add column on a non existing collection will fail', async() => { From 14d56a723cdee89efcf8081a3adcce0f6e1f789b Mon Sep 17 00:00:00 2001 From: Noam Almog Date: Thu, 16 Dec 2021 06:04:13 -0800 Subject: [PATCH 23/47] refactor tests for data (#102) --- .../test/e2e/app_data.e2e.spec.js | 11 ++-- .../test/storage/data_provider.spec.js | 64 +++++++------------ .../test/storage/schema_provider.spec.js | 23 ++----- 3 files changed, 37 insertions(+), 61 deletions(-) diff --git a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js index e57f3201a..7335554d3 100644 --- a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js +++ b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js @@ -28,8 +28,11 @@ describe('Velo External DB Data REST API', () => { test('find api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin) - expect((await axios.post('/data/find', { collectionName: ctx.collectionName, filter: '', sort: [{ fieldName: ctx.column.name }], skip: 0, limit: 25 }, authAdmin)).data).toEqual({ items: [ ctx.item, ctx.anotherItem ].sort((a, b) => (a[ctx.column.name] > b[ctx.column.name]) ? 1 : -1), - totalCount: 2 }) + await expect( axios.post('/data/find', { collectionName: ctx.collectionName, filter: '', sort: [{ fieldName: ctx.column.name }], skip: 0, limit: 25 }, authAdmin) ).resolves.toEqual( + expect.objectContaining({ data: { + items: [ ctx.item, ctx.anotherItem ].sort((a, b) => (a[ctx.column.name] > b[ctx.column.name]) ? 1 : -1), + totalCount: 2 + } })) }) } @@ -38,7 +41,7 @@ describe('Velo External DB Data REST API', () => { await axios.post('/data/insert', { collectionName: ctx.collectionName, item: ctx.item }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ctx.item], totalCount: 1 }) + await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual({ items: [ctx.item], totalCount: 1 }) }) test('bulk insert api', async() => { @@ -46,7 +49,7 @@ describe('Velo External DB Data REST API', () => { await axios.post('/data/insert/bulk', { collectionName: ctx.collectionName, items: ctx.items }, authAdmin) - expect((await data.expectAllDataIn(ctx.collectionName, authAdmin)).items).toEqual(expect.arrayContaining(ctx.items)) + await expect( data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual( { items: expect.arrayContaining(ctx.items), totalCount: ctx.items.length }) }) diff --git a/packages/velo-external-db/test/storage/data_provider.spec.js b/packages/velo-external-db/test/storage/data_provider.spec.js index 39deb1375..00bd68fd8 100644 --- a/packages/velo-external-db/test/storage/data_provider.spec.js +++ b/packages/velo-external-db/test/storage/data_provider.spec.js @@ -25,9 +25,7 @@ describe('Data API', () => { env.driver.stubEmptyFilterFor(ctx.filter) env.driver.stubEmptyOrderByFor(ctx.sort) - const res = await env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit) - - expect( res ).toEqual([]) + await expect( env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual([]) }) @@ -37,9 +35,7 @@ describe('Data API', () => { env.driver.givenFilterByIdWith(ctx.entity._id, ctx.filter) env.driver.stubEmptyOrderByFor(ctx.sort) - const res = await env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit) - - expect( res ).toEqual(expect.arrayContaining([ctx.entity])) + await expect( env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual(expect.arrayContaining([ctx.entity])) }) test('search with non empty order by will return sorted data', async() => { @@ -47,9 +43,7 @@ describe('Data API', () => { env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenOrderByFor('_owner', ctx.sort) - const res = await env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit) - - expect( res ).toEqual([ctx.anotherEntity, ctx.entity].sort((a, b) => (a._owner > b._owner) ? 1 : -1)) + await expect( env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual([ctx.anotherEntity, ctx.entity].sort((a, b) => (a._owner > b._owner) ? 1 : -1)) }) test('search with empty order and filter but with limit and skip', async() => { @@ -57,9 +51,7 @@ describe('Data API', () => { env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenOrderByFor('_owner', ctx.sort) - const res = await env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, 1, 1) - - expect( res ).toEqual([ctx.anotherEntity, ctx.entity].sort((a, b) => (a._owner < b._owner) ? 1 : -1).slice(0, 1)) + await expect( env.dataProvider.find(ctx.collectionName, ctx.filter, ctx.sort, 1, 1) ).resolves.toEqual([ctx.anotherEntity, ctx.entity].sort((a, b) => (a._owner < b._owner) ? 1 : -1).slice(0, 1)) }) } @@ -67,70 +59,66 @@ describe('Data API', () => { await givenCollectionWith(ctx.entities, ctx.collectionName) env.driver.stubEmptyFilterFor(ctx.filter) - const res = await env.dataProvider.count(ctx.collectionName, ctx.filter) - - expect( res ).toEqual(ctx.entities.length) + await expect( env.dataProvider.count(ctx.collectionName, ctx.filter) ).resolves.toEqual(ctx.entities.length) }) test('count will run query with filter', async() => { await givenCollectionWith([ctx.entity, ctx.anotherEntity], ctx.collectionName) env.driver.givenFilterByIdWith(ctx.entity._id, ctx.filter) - const res = await env.dataProvider.count(ctx.collectionName, ctx.filter) - - expect( res ).toEqual(1) + await expect( env.dataProvider.count(ctx.collectionName, ctx.filter) ).resolves.toEqual(1) }) test('insert data into collection name and query all of it', async() => { env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.insert(ctx.collectionName, [ctx.entity]) ).toEqual(1) + await expect( env.dataProvider.insert(ctx.collectionName, [ctx.entity]) ).resolves.toEqual(1) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([ctx.entity]) + await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual([ctx.entity]) }) test('bulk insert data into collection name and query all of it', async() => { env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.insert(ctx.collectionName, ctx.entities) ).toEqual(ctx.entities.length) + await expect( env.dataProvider.insert(ctx.collectionName, ctx.entities) ).resolves.toEqual(ctx.entities.length) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual(expect.arrayContaining(ctx.entities)) + await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual(expect.arrayContaining(ctx.entities)) }) test('insert entity with number', async() => { await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns) env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.insert(ctx.numericCollectionName, [ctx.numberEntity], gen.fieldsArrayToFieldObj(ctx.numericColumns)) ).toEqual(1) + await expect( env.dataProvider.insert(ctx.numericCollectionName, [ctx.numberEntity], gen.fieldsArrayToFieldObj(ctx.numericColumns)) ).resolves.toEqual(1) - expect( await env.dataProvider.find(ctx.numericCollectionName, '', '', 0, 50) ).toEqual([ctx.numberEntity]) + await expect( env.dataProvider.find(ctx.numericCollectionName, '', '', 0, 50) ).resolves.toEqual([ctx.numberEntity]) }) test('delete data from collection', async() => { await givenCollectionWith(ctx.entities, ctx.collectionName) env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.delete(ctx.collectionName, ctx.entities.map(e => e._id)) ).toEqual(ctx.entities.length) + await expect( env.dataProvider.delete(ctx.collectionName, ctx.entities.map(e => e._id)) ).resolves.toEqual(ctx.entities.length) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([]) + await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual([]) }) test('allow update for single entity', async() => { await givenCollectionWith([ctx.entity], ctx.collectionName) env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.update(ctx.collectionName, [ctx.modifiedEntity]) ).toEqual(1) + await expect( env.dataProvider.update(ctx.collectionName, [ctx.modifiedEntity]) ).resolves.toEqual(1) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([ctx.modifiedEntity]) + await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual([ctx.modifiedEntity]) }) test('allow update for multiple entities', async() => { await givenCollectionWith(ctx.entities, ctx.collectionName) env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).toEqual(ctx.modifiedEntities.length) + await expect( env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).resolves.toEqual(ctx.modifiedEntities.length) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual(expect.arrayContaining(ctx.modifiedEntities)) + await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual(expect.arrayContaining(ctx.modifiedEntities)) }) // testt('if update does not have and updatable fields, do nothing', async () => { @@ -148,7 +136,7 @@ describe('Data API', () => { await env.dataProvider.truncate(ctx.collectionName) - expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual([]) + await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual([]) }) if (shouldNotRunOn(['Firestore', 'Airtable', 'DynamoDb'], name)) { @@ -159,9 +147,7 @@ describe('Data API', () => { env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1) - const res = await env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) - - expect( res ).toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, + await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, { _id: ctx.anotherNumberEntity._id, [ctx.aliasColumns[0]]: ctx.anotherNumberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.anotherNumberEntity[ctx.numericColumns[1].name] } ])) }) @@ -173,10 +159,8 @@ describe('Data API', () => { env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1) - const res = await env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) - - expect( res ).toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, - { _id: ctx.anotherNumberEntity._id, [ctx.aliasColumns[0]]: ctx.anotherNumberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.anotherNumberEntity[ctx.numericColumns[1].name] } + await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, + { _id: ctx.anotherNumberEntity._id, [ctx.aliasColumns[0]]: ctx.anotherNumberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.anotherNumberEntity[ctx.numericColumns[1].name] } ])) }) @@ -187,9 +171,7 @@ describe('Data API', () => { env.driver.givenFilterByIdWith(ctx.numberEntity._id, ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 2) - const res = await env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) - - expect( res ).toEqual([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }]) + await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }]) }) } diff --git a/packages/velo-external-db/test/storage/schema_provider.spec.js b/packages/velo-external-db/test/storage/schema_provider.spec.js index 1ab1841ee..6421d3315 100644 --- a/packages/velo-external-db/test/storage/schema_provider.spec.js +++ b/packages/velo-external-db/test/storage/schema_provider.spec.js @@ -20,18 +20,14 @@ describe('Schema API', () => { }, 20000) test('list of empty db will result with an empty array', async() => { - const db = await env.schemaProvider.list() - - expect(db).toEqual([]) + await expect( env.schemaProvider.list() ).resolves.toEqual([]) }) test('list db will result with a list of wix databases', async() => { await env.schemaProvider.create(ctx.collectionName) await env.schemaProvider.create(ctx.anotherCollectionName) - const dbs = await env.schemaProvider.list() - - expect(dbs).toEqual(expect.arrayContaining([ + await expect( env.schemaProvider.list() ).resolves.toEqual(expect.arrayContaining([ expect.objectContaining({ id: ctx.collectionName, fields: collectionWithDefaultFields() @@ -46,9 +42,7 @@ describe('Schema API', () => { test('create collection with default columns', async() => { await env.schemaProvider.create(ctx.collectionName) - const db = await env.schemaProvider.describeCollection(ctx.collectionName) - - expect(db).toEqual(collectionWithDefaultFields()) + await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields()) }) test('drop collection', async() => { @@ -62,15 +56,13 @@ describe('Schema API', () => { test('collection name and variables are case sensitive', async() => { await env.schemaProvider.create(ctx.collectionName.toUpperCase()) - const db = await env.schemaProvider.describeCollection(ctx.collectionName.toUpperCase()) - expect(db).toEqual(collectionWithDefaultFields()) + await expect( env.schemaProvider.describeCollection(ctx.collectionName.toUpperCase()) ).resolves.toEqual(collectionWithDefaultFields()) }) test('retrieve collection data by collection name', async() => { await env.schemaProvider.create(ctx.collectionName) - const db = await env.schemaProvider.describeCollection(ctx.collectionName) - expect(db).toEqual(collectionWithDefaultFields()) + await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields()) }) test('create collection twice will do nothing', async() => { @@ -87,9 +79,8 @@ describe('Schema API', () => { await env.schemaProvider.create(ctx.collectionName, []) await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) - const db = await env.schemaProvider.describeCollection(ctx.collectionName) - expect(db).toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }])) + await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }])) }) test('add duplicate column will fail', async() => { @@ -114,7 +105,7 @@ describe('Schema API', () => { await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) await env.schemaProvider.removeColumn(ctx.collectionName, ctx.columnName) - expect((await env.schemaProvider.describeCollection(ctx.collectionName))).not.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }]) ) + await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.not.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }]) ) }) test('drop column on a a non existing collection', async() => { From 8a492fbea505e0629b21b6c0c8140936280c929b Mon Sep 17 00:00:00 2001 From: Noam Almog Date: Thu, 16 Dec 2021 16:35:19 +0200 Subject: [PATCH 24/47] deps --- packages/external-db-airtable/package.json | 4 ++-- packages/external-db-bigquery/package.json | 2 +- packages/external-db-config/package.json | 4 ++-- packages/external-db-dynamodb/package.json | 4 ++-- packages/external-db-firestore/package.json | 4 ++-- packages/external-db-google-sheets/package.json | 2 +- packages/external-db-mongo/package.json | 6 +++--- packages/external-db-mssql/package.json | 4 ++-- packages/external-db-mysql/package.json | 4 ++-- packages/external-db-postgres/package.json | 4 ++-- packages/external-db-spanner/package.json | 4 ++-- packages/velo-external-db-core/package.json | 4 ++-- packages/velo-external-db/package.json | 4 ++-- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/external-db-airtable/package.json b/packages/external-db-airtable/package.json index 13613d0dd..ff5620b37 100644 --- a/packages/external-db-airtable/package.json +++ b/packages/external-db-airtable/package.json @@ -33,9 +33,9 @@ }, "devDependencies": { "chance": "^1.1.8", - "jest": "^27.4.4", + "jest": "^27.4.5", "test-commons": "^0.0.0", "ts-jest": "^27.1.1", - "jest-when": "^3.4.2" + "jest-when": "^3.5.0" } } diff --git a/packages/external-db-bigquery/package.json b/packages/external-db-bigquery/package.json index 7947c326a..38a43b466 100644 --- a/packages/external-db-bigquery/package.json +++ b/packages/external-db-bigquery/package.json @@ -26,7 +26,7 @@ "devDependencies": { "chance": "^1.1.8", "jest-each": "^27.4.2", - "jest-when": "^3.4.2", + "jest-when": "^3.5.0", "test-commons": "^0.0.0" } } diff --git a/packages/external-db-config/package.json b/packages/external-db-config/package.json index 64b2daa29..b24e86191 100644 --- a/packages/external-db-config/package.json +++ b/packages/external-db-config/package.json @@ -25,9 +25,9 @@ "devDependencies": { "aws-sdk-client-mock": "^0.5.6", "chance": "^1.1.8", - "jest": "^27.4.4", + "jest": "^27.4.5", "jest-each": "^27.4.2", - "jest-when": "^3.4.2", + "jest-when": "^3.5.0", "ts-jest": "^27.1.1", "test-commons": "^0.0.0" }, diff --git a/packages/external-db-dynamodb/package.json b/packages/external-db-dynamodb/package.json index 3f1a0423a..a47931406 100644 --- a/packages/external-db-dynamodb/package.json +++ b/packages/external-db-dynamodb/package.json @@ -33,8 +33,8 @@ }, "devDependencies": { "chance": "^1.1.8", - "jest": "^27.4.4", - "jest-when": "^3.4.2", + "jest": "^27.4.5", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1" } diff --git a/packages/external-db-firestore/package.json b/packages/external-db-firestore/package.json index c2e574f0c..cc7a4efcc 100644 --- a/packages/external-db-firestore/package.json +++ b/packages/external-db-firestore/package.json @@ -30,8 +30,8 @@ "velo-external-db-commons": "^0.0.0" }, "devDependencies": { - "jest": "^27.4.4", - "jest-when": "^3.4.2", + "jest": "^27.4.5", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1" } diff --git a/packages/external-db-google-sheets/package.json b/packages/external-db-google-sheets/package.json index 69c458c08..ee7e2c98e 100644 --- a/packages/external-db-google-sheets/package.json +++ b/packages/external-db-google-sheets/package.json @@ -22,7 +22,7 @@ "velo-external-db-commons": "^0.0.0" }, "devDependencies": { - "jest": "^27.4.4", + "jest": "^27.4.5", "test-commons": "^0.0.0", "ts-jest": "^27.1.1" } diff --git a/packages/external-db-mongo/package.json b/packages/external-db-mongo/package.json index 8eb1e80ee..41aa4f0ce 100644 --- a/packages/external-db-mongo/package.json +++ b/packages/external-db-mongo/package.json @@ -26,12 +26,12 @@ "url": "https://github.com/wix-private/noam-external-db-poc/issues" }, "dependencies": { - "mongodb": "^4.2.1", + "mongodb": "^4.2.2", "velo-external-db-commons": "^0.0.0" }, "devDependencies": { - "jest": "^27.4.4", - "jest-when": "^3.4.2", + "jest": "^27.4.5", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1", "chance": "^1.1.8" diff --git a/packages/external-db-mssql/package.json b/packages/external-db-mssql/package.json index 5ef5be14e..a175f67eb 100644 --- a/packages/external-db-mssql/package.json +++ b/packages/external-db-mssql/package.json @@ -31,8 +31,8 @@ "velo-external-db-commons": "^0.0.0" }, "devDependencies": { - "jest": "^27.4.4", - "jest-when": "^3.4.2", + "jest": "^27.4.5", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1", "chance": "^1.1.8" diff --git a/packages/external-db-mysql/package.json b/packages/external-db-mysql/package.json index 7e9a46de6..6f40648e7 100644 --- a/packages/external-db-mysql/package.json +++ b/packages/external-db-mysql/package.json @@ -30,8 +30,8 @@ "velo-external-db-commons": "^0.0.0" }, "devDependencies": { - "jest": "^27.4.4", - "jest-when": "^3.4.2", + "jest": "^27.4.5", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1", "chance": "^1.1.8" diff --git a/packages/external-db-postgres/package.json b/packages/external-db-postgres/package.json index ed9feaf45..32b710c93 100644 --- a/packages/external-db-postgres/package.json +++ b/packages/external-db-postgres/package.json @@ -31,8 +31,8 @@ "velo-external-db-commons": "^0.0.0" }, "devDependencies": { - "jest": "^27.4.4", - "jest-when": "^3.4.2", + "jest": "^27.4.5", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1", "chance": "^1.1.8" diff --git a/packages/external-db-spanner/package.json b/packages/external-db-spanner/package.json index 77736e436..51e6888b0 100644 --- a/packages/external-db-spanner/package.json +++ b/packages/external-db-spanner/package.json @@ -31,9 +31,9 @@ }, "devDependencies": { "chance": "^1.1.8", - "jest": "^27.4.4", + "jest": "^27.4.5", "jest-each": "^27.4.2", - "jest-when": "^3.4.2", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1" }, diff --git a/packages/velo-external-db-core/package.json b/packages/velo-external-db-core/package.json index 23667eed1..71c68eb9c 100644 --- a/packages/velo-external-db-core/package.json +++ b/packages/velo-external-db-core/package.json @@ -30,9 +30,9 @@ }, "devDependencies": { "chance": "^1.1.8", - "jest": "^27.4.4", + "jest": "^27.4.5", "jest-each": "^27.4.2", - "jest-when": "^3.4.2", + "jest-when": "^3.5.0", "rewire": "^5.0.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1", diff --git a/packages/velo-external-db/package.json b/packages/velo-external-db/package.json index 25235f1b6..2eeacfadb 100644 --- a/packages/velo-external-db/package.json +++ b/packages/velo-external-db/package.json @@ -45,9 +45,9 @@ "axios": "^0.24.0", "chance": "^1.1.8", "docker-compose": "^0.23.14", - "jest": "^27.4.4", + "jest": "^27.4.5", "jest-each": "^27.4.2", - "jest-when": "^3.4.2", + "jest-when": "^3.5.0", "test-commons": "^0.0.0", "ts-jest": "^27.1.1" } From faa21d22da4a58356fbca4735205194cda4ff1e3 Mon Sep 17 00:00:00 2001 From: Noam Almog Date: Thu, 16 Dec 2021 16:48:16 +0200 Subject: [PATCH 25/47] refactor data async tests --- .../test/drivers/schema_api_rest_matchers.js | 16 +++++-------- .../test/e2e/app_data.e2e.spec.js | 23 ++++++++++--------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/packages/velo-external-db/test/drivers/schema_api_rest_matchers.js b/packages/velo-external-db/test/drivers/schema_api_rest_matchers.js index 0ae9d11a0..b75c4aee1 100644 --- a/packages/velo-external-db/test/drivers/schema_api_rest_matchers.js +++ b/packages/velo-external-db/test/drivers/schema_api_rest_matchers.js @@ -1,5 +1,7 @@ const { SystemFields } = require('velo-external-db-commons') +const responseWith = (matcher) => expect.objectContaining( { data: matcher } ) + const hasSameSchemaFieldsLike = fields => expect.objectContaining( fields.reduce((o, f) => ( { ...o, [f.name]: expect.objectContaining({ displayName: f.name, type: f.type }) } ), {}) ) const schemaWithDefaultFieldsFor = collectionName => expect.objectContaining({ id: collectionName, @@ -12,17 +14,11 @@ const collectionWithDefaultFieldsFor = collectionName => expect.objectContaining schemas: expect.arrayContaining( [ schemaWithDefaultFieldsFor( collectionName ) ] ) } ) -const collectionResponseWithDefaultFieldsFor = collectionName => expect.objectContaining( { - data: collectionWithDefaultFieldsFor( collectionName ) -} ) +const collectionResponseWithDefaultFieldsFor = collectionName => responseWith( collectionWithDefaultFieldsFor( collectionName ) ) -const collectionResponseHasField = field => expect.objectContaining( { - data: collectionHasField( field ) -} ) +const collectionResponseHasField = field => responseWith(collectionHasField( field )) -const collectionResponseWithNoCollections = () => expect.objectContaining( { - data: toHaveNoCollections() -} ) +const collectionResponseWithNoCollections = () => responseWith(toHaveNoCollections()) const collectionHasField = field => expect.objectContaining( { schemas: expect.arrayContaining( [ schemaWithFields( [field] ) ] ) @@ -32,4 +28,4 @@ const toHaveNoCollections = () => expect.objectContaining( { schemas: [ ] } ) -module.exports = { collectionResponseWithDefaultFieldsFor, collectionResponseHasField, collectionResponseWithNoCollections } \ No newline at end of file +module.exports = { collectionResponseWithDefaultFieldsFor, collectionResponseHasField, collectionResponseWithNoCollections, responseWith } \ No newline at end of file diff --git a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js index 7335554d3..88128d102 100644 --- a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js +++ b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js @@ -1,6 +1,7 @@ const { Uninitialized, gen, shouldNotRunOn } = require('test-commons') const schema = require('../drivers/schema_api_rest_test_support') const data = require('../drivers/data_api_rest_test_support') +const matchers = require('../drivers/schema_api_rest_matchers') const { authAdmin, authOwner } = require('../drivers/auth_test_support') const Chance = require('chance') const each = require('jest-each').default @@ -59,8 +60,8 @@ describe('Velo External DB Data REST API', () => { await schema.givenCollection(ctx.collectionName, ctx.numberColumns, authOwner) await data.givenItems([ctx.numberItem, ctx.anotherNumberItem], ctx.collectionName, authAdmin) - expect((await axios.post('/data/aggregate', - { + await expect( axios.post('/data/aggregate', + { collectionName: ctx.collectionName, filter: { _id: { $eq: ctx.numberItem._id } }, processingStep: { @@ -81,8 +82,8 @@ describe('Velo External DB Data REST API', () => { { mySum: { $gt: 0 } } ], }, - }, authAdmin)).data).toEqual({ items: [ { _id: ctx.numberItem._id, _owner: ctx.numberItem._owner, myAvg: ctx.numberItem[ctx.numberColumns[0].name], mySum: ctx.numberItem[ctx.numberColumns[1].name] } ], - totalCount: 0 }) + }, authAdmin) ).resolves.toEqual(matchers.responseWith({ items: [ { _id: ctx.numberItem._id, _owner: ctx.numberItem._owner, myAvg: ctx.numberItem[ctx.numberColumns[0].name], mySum: ctx.numberItem[ctx.numberColumns[1].name] } ], + totalCount: 0 })) }) } @@ -92,7 +93,7 @@ describe('Velo External DB Data REST API', () => { await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) + await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 }) }) test('bulk delete api', async() => { @@ -101,14 +102,14 @@ describe('Velo External DB Data REST API', () => { await axios.post('/data/remove/bulk', { collectionName: ctx.collectionName, itemIds: ctx.items.map(i => i._id) }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) + await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 }) }) test('get by id api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - expect((await axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin)).data).toEqual({ item: ctx.item }) + await expect( axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) ).resolves.toEqual(matchers.responseWith({ item: ctx.item })) }) test('update api', async() => { @@ -117,7 +118,7 @@ describe('Velo External DB Data REST API', () => { await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) + await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) }) test('bulk update api', async() => { @@ -126,14 +127,14 @@ describe('Velo External DB Data REST API', () => { await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) - expect((await data.expectAllDataIn(ctx.collectionName, authAdmin)).items).toEqual(expect.arrayContaining(ctx.modifiedItems)) + await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual( { items: expect.arrayContaining(ctx.modifiedItems), totalCount: ctx.modifiedItems.length }) }) test('count api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin) - expect((await axios.post('/data/count', { collectionName: ctx.collectionName, filter: '' }, authAdmin)).data).toEqual({ totalCount: 2 }) + await expect( axios.post('/data/count', { collectionName: ctx.collectionName, filter: '' }, authAdmin) ).resolves.toEqual(matchers.responseWith( { totalCount: 2 } )) }) test('truncate api', async() => { @@ -142,7 +143,7 @@ describe('Velo External DB Data REST API', () => { await axios.post('/data/truncate', { collectionName: ctx.collectionName }, authAdmin) - expect(await data.expectAllDataIn(ctx.collectionName, authAdmin)).toEqual({ items: [ ], totalCount: 0 }) + await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual({ items: [ ], totalCount: 0 }) }) }) From 1975dcc957d2419cc4a356dec4785372280ac387 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:16:29 +0200 Subject: [PATCH 26/47] testing main yml --- .github/workflows/main.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3463bf1fc..d76113a50 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,3 +90,40 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down + + test_bigquery: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14 + uses: actions/setup-node@v2 + with: + node-version: 14 + + - name: Restore Dependencies + uses: actions/cache@v2 + with: + path: | + node_modules + */*/node_modules + key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + + - id: auth + uses: google-github-actions/auth@v0.4.0 + with: + credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.1 + + - name: Use gcloud CLI + run: gcloud info + + - name: Installing Dependencies + run: npm install + + - name: Install dependencies of the homemade packages. + run: npm run build:dev + + - name: Testing + run: npm run test:bigquery From 6dc74ab9c64ac571d37a8d42678c8ac780bac874 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:50:17 +0200 Subject: [PATCH 27/47] disable other tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d76113a50..4dd087a81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,7 +121,7 @@ jobs: - name: Installing Dependencies run: npm install - + - name: Install dependencies of the homemade packages. run: npm run build:dev From b57979314a066c4f5f3501363f28fecdf79be6e1 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 15:05:12 +0200 Subject: [PATCH 28/47] print errors --- packages/external-db-bigquery/lib/sql_exception_translator.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 41bd1c54b..3e7c5bfe5 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,6 +15,7 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { + console.log(err) throw notThrowingTranslateErrorCodes(err) } From b9874d1c3376d997258d86256b81579940df2fc0 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 15:43:47 +0200 Subject: [PATCH 29/47] log errors --- .../lib/bigquery_schema_provider.js | 77 ++++++++++++------- 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index 07af53982..bd82f8969 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -11,55 +11,76 @@ class SchemaProvider { } async list() { - const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') - const tables = parseTableData(res[0]) + try{ + const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') + const tables = parseTableData(res[0]) + return Object.entries(tables) + .map(([collectionName, rs]) => ({ + id: collectionName, + fields: rs.map( this.translateDbTypes.bind(this) ) + })) + } catch(e) { + console.log(e) + } - return Object.entries(tables) - .map(([collectionName, rs]) => ({ - id: collectionName, - fields: rs.map( this.translateDbTypes.bind(this) ) - })) } async create(collectionName, _columns) { - const columns = _columns || [] - const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) - await this.pool.createTable(collectionName, { schema: dbColumnsSql }) - .catch(translateErrorCodes) + try{ + const columns = _columns || [] + const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) + await this.pool.createTable(collectionName, { schema: dbColumnsSql }) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } } async drop(collectionName) { - await this.pool.table(collectionName).delete() - .catch(translateErrorCodes) + try { + await this.pool.table(collectionName).delete() + .catch(translateErrorCodes) + } catch(e) { + og.error(e) + } } async addColumn(collectionName, column) { - await validateSystemFields(column.name) - - await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) - .catch(translateErrorCodes) - + try { + await validateSystemFields(column.name) + await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } } async removeColumn(collectionName, columnName) { - await validateSystemFields(columnName) + try { + await validateSystemFields(columnName) + await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) + .catch(translateErrorCodes) + } catch(e) { + console.log(e) + } + - await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) - .catch(translateErrorCodes) } async describeCollection(collectionName) { - const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) - - if (res[0].length === 0) { - throw new CollectionDoesNotExists('Collection does not exists') + try{ + const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) + if (res[0].length === 0) { + throw new CollectionDoesNotExists('Collection does not exists') + } + return res[0].map( this.translateDbTypes.bind(this) ) + } catch(e) { + console.log(e) } - return res[0].map( this.translateDbTypes.bind(this) ) } - - + translateDbTypes(row) { row.type = this.sqlSchemaTranslator.translateType(row.type) return row From 330800f56c1e5b570b1ed1af603ac58814805a92 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:00:39 +0200 Subject: [PATCH 30/47] test fefactor bigquery --- .../lib/bigquery_data_provider.js | 9 +-- .../lib/bigquery_schema_provider.js | 69 ++++++++----------- .../lib/bigquery_utils.js | 2 +- .../lib/connection_provider.js | 6 +- .../lib/sql_exception_translator.js | 1 - .../sql_filter_transformer_test_support.js | 4 +- .../lib/readers/gcp_config_reader.js | 6 +- packages/velo-external-db/package.json | 4 +- .../test/e2e/app_data.e2e.spec.js | 39 +++++++---- .../resources/engines/bigquery_resources.js | 12 ++-- .../test/resources/provider_resources.js | 2 +- .../test/storage/data_provider.spec.js | 32 +++++---- .../test/storage/schema_provider.spec.js | 50 ++++++++------ 13 files changed, 125 insertions(+), 111 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_data_provider.js b/packages/external-db-bigquery/lib/bigquery_data_provider.js index 828734a69..dd769879e 100644 --- a/packages/external-db-bigquery/lib/bigquery_data_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_data_provider.js @@ -34,6 +34,7 @@ class DataProvider { } async insert(collectionName, items) { + // const patchedItems = items.map(patchDateTime) // const itemsArr = patchedItems.map( item => `(${Object.values(item).map(i => typeof(i) === 'number' ? i : `"${i}"`).join(', ')})` ).join(', ') // const sql = `INSERT INTO \`${collectionName}\` (${Object.keys(items[0]).join(', ')}) VALUES ${itemsArr}` @@ -41,9 +42,9 @@ class DataProvider { // const res = await this.pool.query(sql).catch( translateErrorCodes ) const table = await this.pool.table(collectionName) - const res = await table.insert(items) - - return res + await table.insert(items) + + return items.length } async update(collectionName, items) { @@ -82,7 +83,7 @@ class DataProvider { const resultSet = await this.pool.query({ query: sql, params: [...whereParameters, ...parameters] }) .catch( translateErrorCodes ) - return resultSet[0] + return resultSet[0].map( unPatchDateTime ) } } diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index bd82f8969..b47d96d6f 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -5,7 +5,9 @@ const { SystemFields, validateSystemFields, parseTableData } = require('velo-ext const escapeIdentifier = i => i class SchemaProvider { - constructor(pool) { + constructor(pool, { projectId, databaseId }) { + this.projectId = projectId + this.databaseId = databaseId this.pool = pool this.sqlSchemaTranslator = new SchemaColumnTranslator() } @@ -18,7 +20,7 @@ class SchemaProvider { .map(([collectionName, rs]) => ({ id: collectionName, fields: rs.map( this.translateDbTypes.bind(this) ) - })) + })) } catch(e) { console.log(e) } @@ -26,61 +28,46 @@ class SchemaProvider { } async create(collectionName, _columns) { - try{ - const columns = _columns || [] - const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) - await this.pool.createTable(collectionName, { schema: dbColumnsSql }) - .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } + const columns = _columns || [] + const dbColumnsSql = [...SystemFields, ...columns].map(c => this.sqlSchemaTranslator.columnToDbColumnSql(c)) + await this.pool.createTable(collectionName, { schema: dbColumnsSql }) + .catch(translateErrorCodes) } async drop(collectionName) { - try { - await this.pool.table(collectionName).delete() - .catch(translateErrorCodes) - } catch(e) { - og.error(e) - } + await this.pool.table(collectionName).delete() + .catch(translateErrorCodes) } - async addColumn(collectionName, column) { - try { - await validateSystemFields(column.name) + async addColumn(collectionName, column) { + await validateSystemFields(column.name) + try{ await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) - .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } + } catch (err) { + if (err.message.includes('was not found')) + throw new CollectionDoesNotExists('Collection does not exists') + else + translateErrorCodes(err) + } } async removeColumn(collectionName, columnName) { - try { - await validateSystemFields(columnName) - await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) + await validateSystemFields(columnName) + await this.pool.query(`CREATE OR REPLACE TABLE ${this.projectId}.${this.databaseId}.${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM ${escapeIdentifier(collectionName)}`) .catch(translateErrorCodes) - } catch(e) { - console.log(e) - } - - - } async describeCollection(collectionName) { - try{ - const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) - if (res[0].length === 0) { - throw new CollectionDoesNotExists('Collection does not exists') - } - return res[0].map( this.translateDbTypes.bind(this) ) - } catch(e) { - console.log(e) + const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM ${this.projectId}.${this.databaseId}.INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`) + + if (res[0].length === 0) { + throw new CollectionDoesNotExists('Collection does not exists') } + + return res[0].map( this.translateDbTypes.bind(this) ) } - + translateDbTypes(row) { row.type = this.sqlSchemaTranslator.translateType(row.type) return row diff --git a/packages/external-db-bigquery/lib/bigquery_utils.js b/packages/external-db-bigquery/lib/bigquery_utils.js index d9b5dd3f0..8d032c376 100644 --- a/packages/external-db-bigquery/lib/bigquery_utils.js +++ b/packages/external-db-bigquery/lib/bigquery_utils.js @@ -24,7 +24,7 @@ const unPatchDateTime = (item) => { if (reISO.test(value)) { obj[key] = new Date(value) } else { - obj[key] = item[key] + obj[key] = item[key].toNumber ? item[key].toNumber() : item[key] } } diff --git a/packages/external-db-bigquery/lib/connection_provider.js b/packages/external-db-bigquery/lib/connection_provider.js index d269a42a0..c526f5272 100644 --- a/packages/external-db-bigquery/lib/connection_provider.js +++ b/packages/external-db-bigquery/lib/connection_provider.js @@ -4,14 +4,14 @@ const DatabaseOperations = require ('./bigquery_operations') const SchemaProvider = require('./bigquery_schema_provider') const DataProvider = require('./bigquery_data_provider') -const init = ( cfg ) => { +const init = ({ projectId, databaseId }) => { const bigquery = new BigQuery() - const pool = bigquery.dataset(cfg.databaseId) + const pool = bigquery.dataset(databaseId) const filterParser = new FilterParser() const databaseOperations = new DatabaseOperations(pool) const dataProvider = new DataProvider(pool, filterParser) - const schemaProvider = new SchemaProvider(pool) + const schemaProvider = new SchemaProvider(pool, { projectId, databaseId }) return { dataProvider: dataProvider, schemaProvider: schemaProvider, databaseOperations, connection: pool, cleanup: async() => {} } } diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 3e7c5bfe5..41bd1c54b 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,7 +15,6 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { - console.log(err) throw notThrowingTranslateErrorCodes(err) } diff --git a/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js b/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js index f2c667530..79a71f8de 100644 --- a/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js +++ b/packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js @@ -33,11 +33,11 @@ const givenOrderByFor = (column, sort) => { const givenFilterByIdWith = (id, filter) => { when(filterParser.transform).calledWith(filter) - .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = $1`, parameters: [id], offset: 2 }) + .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = ?`, parameters: [id], offset: 2 }) } const givenAggregateQueryWith = (having, numericColumns, columnAliases, groupByColumns, filter, offest) => { - when(filterParser.parseAggregation).calledWith(having, filter, offest) + when(filterParser.parseAggregation).calledWith(having, filter) .mockReturnValue({ fieldsStatement: `${groupByColumns.map( escapeIdentifier ).join(', ')}, MAX(${escapeIdentifier(numericColumns[0].name)}) AS ${escapeIdentifier(columnAliases[0])}, SUM(${escapeIdentifier(numericColumns[1].name)}) AS ${escapeIdentifier(columnAliases[1])}`, groupByColumns: groupByColumns, diff --git a/packages/external-db-config/lib/readers/gcp_config_reader.js b/packages/external-db-config/lib/readers/gcp_config_reader.js index 9a644a648..6c7c9936d 100644 --- a/packages/external-db-config/lib/readers/gcp_config_reader.js +++ b/packages/external-db-config/lib/readers/gcp_config_reader.js @@ -99,13 +99,13 @@ class GcpBigQueryConfigReader { } async readConfig() { - const { DATABASE_ID, SECRET_KEY } = process.env - return { databaseId: DATABASE_ID, secretKey: SECRET_KEY } + const { PROJECT_ID, DATABASE_ID, SECRET_KEY } = process.env + return { projectId: PROJECT_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY } } validate() { return { - missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['DATABASE_ID', 'SECRET_KEY']) + missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['PROJECT_ID', 'DATABASE_ID', 'SECRET_KEY']) } } diff --git a/packages/velo-external-db/package.json b/packages/velo-external-db/package.json index 2eeacfadb..0dbb2d908 100644 --- a/packages/velo-external-db/package.json +++ b/packages/velo-external-db/package.json @@ -11,10 +11,10 @@ "test:firestore": "TEST_ENGINE=firestore jest --runInBand ./test", "test:mssql": "TEST_ENGINE=mssql jest --runInBand ./test", "test:mongo": "TEST_ENGINE=mongo jest --runInBand ./test", - "test:google-sheets": "TEST_ENGINE=google-sheet jest -i packages/velo-external-db/test/e2e/app_data.e2e.spec.js --runInBand -t 'Velo External DB Data REST API' ", + "test:google-sheets": "TEST_ENGINE=google-sheet jest -i packages/velo-external-db/test/e2e/app_data.e2e.spec.js --runInBand -t 'Schema API' ", "test:airtable": "TEST_ENGINE=airtable jest --runInBand ./test ", "test:dynamodb": "TEST_ENGINE=dynamodb jest --runInBand ./test ", - "test:bigquery": "TEST_ENGINE=bigquery jest --runInBand -i packages/velo-external-db/test/e2e/app_schema.e2e.spec.js -t 'Velo External DB Schema REST API' ", + "test:bigquery": "TEST_ENGINE=bigquery jest --runInBand ./test", "test:watch": "jest --detectOpenHandles --watch", "start": "node lib/app.js", "deploy": "gcloud app deploy ./app.yaml" diff --git a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js index 88128d102..1cd017b6f 100644 --- a/packages/velo-external-db/test/e2e/app_data.e2e.spec.js +++ b/packages/velo-external-db/test/e2e/app_data.e2e.spec.js @@ -87,15 +87,19 @@ describe('Velo External DB Data REST API', () => { }) } + + if (shouldNotRunOn(['BigQuery'], name)) { test('delete one api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) + await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 }) }) + } + if (shouldNotRunOn(['BigQuery'], name)) { test('bulk delete api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems(ctx.items, ctx.collectionName, authAdmin) @@ -104,31 +108,38 @@ describe('Velo External DB Data REST API', () => { await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 }) }) + } + if (shouldNotRunOn(['BigQuery'], name)) { test('get by id api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item], ctx.collectionName, authAdmin) await expect( axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) ).resolves.toEqual(matchers.responseWith({ item: ctx.item })) }) + } + + if (shouldNotRunOn(['BigQuery'], name)) { + test('update api', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - test('update api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - - await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) + await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) - }) + }) + } - test('bulk update api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems(ctx.items, ctx.collectionName, authAdmin) + if (shouldNotRunOn(['BigQuery'], name)) { + test('bulk update api', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems(ctx.items, ctx.collectionName, authAdmin) - await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) + await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual( { items: expect.arrayContaining(ctx.modifiedItems), totalCount: ctx.modifiedItems.length }) - }) + }) + } test('count api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) diff --git a/packages/velo-external-db/test/resources/engines/bigquery_resources.js b/packages/velo-external-db/test/resources/engines/bigquery_resources.js index 4d7a7ecd5..2930f7aa5 100644 --- a/packages/velo-external-db/test/resources/engines/bigquery_resources.js +++ b/packages/velo-external-db/test/resources/engines/bigquery_resources.js @@ -1,29 +1,31 @@ const { init } = require('external-db-bigquery') const databaseId = 'testDB' +const projectId = 'corvid-managed-cfe9809c' const connection = () => { - const { connection, cleanup } = init({ databaseId }) + const { connection, cleanup } = init({ databaseId, projectId }) return { pool: connection, cleanup } } const cleanup = async() => { - const { schemaProvider } = init({ databaseId }) + const { schemaProvider } = init({ databaseId, projectId }) const tables = await schemaProvider.list() await Promise.all(tables.map(t => t.id).map( t => schemaProvider.drop(t) )) } const initEnv = async() => { - // await runImage('') } const setActive = () => { process.env.TYPE = 'bigquery' + process.env.PROJECT_ID = projectId process.env.DATABASE_ID = databaseId } const shutdownEnv = async() => { - // await stopImage('') } -module.exports = { initEnv, shutdownEnv, setActive, connection, cleanup } \ No newline at end of file +const schemaProviderTestVariables = () => ({ projectId, databaseId }) + +module.exports = { initEnv, shutdownEnv, setActive, connection, cleanup, schemaProviderTestVariables } \ No newline at end of file diff --git a/packages/velo-external-db/test/resources/provider_resources.js b/packages/velo-external-db/test/resources/provider_resources.js index dda79120d..07fe2e269 100644 --- a/packages/velo-external-db/test/resources/provider_resources.js +++ b/packages/velo-external-db/test/resources/provider_resources.js @@ -40,7 +40,7 @@ const dbInit = async(testEnv, impl) => { const { pool, cleanup } = await testEnv.connection() const driver = impl.driver() - + env.dataProvider = new impl.DataProvider(pool, driver.filterParser) env.schemaProvider = new impl.SchemaProvider(pool, testEnv.schemaProviderTestVariables?.() ) env.driver = driver diff --git a/packages/velo-external-db/test/storage/data_provider.spec.js b/packages/velo-external-db/test/storage/data_provider.spec.js index 00bd68fd8..0ab6e483a 100644 --- a/packages/velo-external-db/test/storage/data_provider.spec.js +++ b/packages/velo-external-db/test/storage/data_provider.spec.js @@ -94,32 +94,38 @@ describe('Data API', () => { await expect( env.dataProvider.find(ctx.numericCollectionName, '', '', 0, 50) ).resolves.toEqual([ctx.numberEntity]) }) - test('delete data from collection', async() => { - await givenCollectionWith(ctx.entities, ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('delete data from collection', async() => { + await givenCollectionWith(ctx.entities, ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') await expect( env.dataProvider.delete(ctx.collectionName, ctx.entities.map(e => e._id)) ).resolves.toEqual(ctx.entities.length) await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual([]) - }) + }) + } - test('allow update for single entity', async() => { - await givenCollectionWith([ctx.entity], ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('allow update for single entity', async() => { + await givenCollectionWith([ctx.entity], ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') await expect( env.dataProvider.update(ctx.collectionName, [ctx.modifiedEntity]) ).resolves.toEqual(1) await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual([ctx.modifiedEntity]) }) + } - test('allow update for multiple entities', async() => { - await givenCollectionWith(ctx.entities, ctx.collectionName) - env.driver.stubEmptyFilterAndSortFor('', '') + if (shouldNotRunOn(['BigQuery'], name)) { + test('allow update for multiple entities', async() => { + await givenCollectionWith(ctx.entities, ctx.collectionName) + env.driver.stubEmptyFilterAndSortFor('', '') - await expect( env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).resolves.toEqual(ctx.modifiedEntities.length) + expect( await env.dataProvider.update(ctx.collectionName, ctx.modifiedEntities) ).toEqual(ctx.modifiedEntities.length) - await expect( env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).resolves.toEqual(expect.arrayContaining(ctx.modifiedEntities)) - }) + expect( await env.dataProvider.find(ctx.collectionName, '', '', 0, 50) ).toEqual(expect.arrayContaining(ctx.modifiedEntities)) + }) + } // testt('if update does not have and updatable fields, do nothing', async () => { // await givenCollectionWith([ctx.entity], ctx.collectionName) diff --git a/packages/velo-external-db/test/storage/schema_provider.spec.js b/packages/velo-external-db/test/storage/schema_provider.spec.js index 6421d3315..cabce89c9 100644 --- a/packages/velo-external-db/test/storage/schema_provider.spec.js +++ b/packages/velo-external-db/test/storage/schema_provider.spec.js @@ -1,5 +1,5 @@ const { CollectionDoesNotExists, FieldAlreadyExists, CannotModifySystemField, FieldDoesNotExist } = require('velo-external-db-commons').errors -const { Uninitialized, gen } = require('test-commons') +const { Uninitialized, gen, shouldNotRunOn } = require('test-commons') const each = require('jest-each').default const Chance = require('chance') const { env, testSuits, dbTeardown } = require('../resources/provider_resources') @@ -65,11 +65,13 @@ describe('Schema API', () => { await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields()) }) - test('create collection twice will do nothing', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + // if (shouldNotRunOn(['BigQuery'], name)) { + test('create collection twice will do nothing', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - await expect( env.schemaProvider.create(ctx.collectionName, []) ).resolves.toBeUndefined() - }) + await expect( env.schemaProvider.create(ctx.collectionName, []) ).resolves.toBeUndefined() + }) + // } test('add column on a non existing collection will fail', async() => { await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(CollectionDoesNotExists) @@ -83,13 +85,15 @@ describe('Schema API', () => { await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }])) }) - test('add duplicate column will fail', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('add duplicate column will fail', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) + await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) - await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(FieldAlreadyExists) - }) + await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(FieldAlreadyExists) + }) + } test('add system column will fail', async() => { await env.schemaProvider.create(ctx.collectionName, []) @@ -108,20 +112,24 @@ describe('Schema API', () => { await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.not.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName, type: 'datetime' }]) ) }) - test('drop column on a a non existing collection', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('drop column on a a non existing collection', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - await expect(env.schemaProvider.removeColumn(ctx.collectionName, ctx.columnName)).rejects.toThrow(FieldDoesNotExist) - }) + await expect(env.schemaProvider.removeColumn(ctx.collectionName, ctx.columnName)).rejects.toThrow(FieldDoesNotExist) + }) + } - test('drop system column will fail', async() => { - await env.schemaProvider.create(ctx.collectionName, []) + if (shouldNotRunOn(['BigQuery'], name)) { + test('drop system column will fail', async() => { + await env.schemaProvider.create(ctx.collectionName, []) - SystemFields.map(f => f.name) - .forEach(async f => { - await expect(env.schemaProvider.removeColumn(ctx.collectionName, f)).rejects.toThrow(CannotModifySystemField) - }) - }) + SystemFields.map(f => f.name) + .forEach(async f => { + await expect(env.schemaProvider.removeColumn(ctx.collectionName, f)).rejects.toThrow(CannotModifySystemField) + }) + }) + } const ctx = { collectionName: Uninitialized, From 9dad99957667a7b838a21d7e88ef382e10bbedaf Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:21:07 +0200 Subject: [PATCH 31/47] test if in yml file --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dd087a81..32282fe00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,6 +109,7 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - id: auth + if: ${{ (1 == 1) }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} From 8ace2dfdeffde1566ab081289b69a781ddb93ce3 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:27:09 +0200 Subject: [PATCH 32/47] main yml refactor to support ifs --- .github/workflows/main.yml | 42 +++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32282fe00..396718f70 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,8 +93,13 @@ jobs: test_bigquery: runs-on: ubuntu-latest + strategy: + matrix: + database: ["BigQuery"] + steps: - uses: actions/checkout@v2 + - name: Use Node.js 14 uses: actions/setup-node@v2 with: @@ -108,18 +113,34 @@ jobs: */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + - name: Downloading Docker Compose V2 + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh + docker compose version + + - name: Downloading Image + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose pull --quiet ${{ matrix.database }} + + - name: Start Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose up --detach ${{ matrix.database }} + - id: auth - if: ${{ (1 == 1) }} + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - + - name: Set up Cloud SDK + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - - - name: Use gcloud CLI - run: gcloud info - + - name: Installing Dependencies run: npm install @@ -127,4 +148,11 @@ jobs: run: npm run build:dev - name: Testing - run: npm run test:bigquery + run: npm run test:${{ matrix.database }} + + - name: Stop Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose down + From 8c05edc8ebcda757858d02fedcb8f2468275b8ff Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:34:33 +0200 Subject: [PATCH 33/47] main yml refactor to support ifs --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 396718f70..bc2e9aeba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -114,31 +114,31 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - name: Downloading Docker Compose V2 - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh docker compose version - name: Downloading Image - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose pull --quiet ${{ matrix.database }} - name: Start Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose up --detach ${{ matrix.database }} - id: auth - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - name: Set up Cloud SDK - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - name: Installing Dependencies @@ -151,7 +151,7 @@ jobs: run: npm run test:${{ matrix.database }} - name: Stop Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose down From c894cd23dd50e532e24d1ac0d77eb771e6698e4e Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:38:30 +0200 Subject: [PATCH 34/47] main yml refactor to support ifs --- .github/workflows/main.yml | 56 ++------------------------------------ 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc2e9aeba..37543e197 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: "mongo", "mongo4", "firestore", "airtable", - "dynamodb"] + "dynamodb","bigquery"] steps: - uses: actions/checkout@v2 @@ -62,57 +62,6 @@ jobs: */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - - name: Downloading Docker Compose V2 - run: | - curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh - docker compose version - - - name: Downloading Image - run: | - cd packages/velo-external-db/test/resources - docker compose pull --quiet ${{ matrix.database }} - - - name: Start Container - run: | - cd packages/velo-external-db/test/resources - docker compose up --detach ${{ matrix.database }} - - - name: Installing Dependencies - run: npm install - - - name: Install dependencies of the homemade packages. - run: npm run build:dev - - - name: Testing - run: npm run test:${{ matrix.database }} - - - name: Stop Container - run: | - cd packages/velo-external-db/test/resources - docker compose down - - test_bigquery: - runs-on: ubuntu-latest - strategy: - matrix: - database: ["BigQuery"] - - steps: - - uses: actions/checkout@v2 - - - name: Use Node.js 14 - uses: actions/setup-node@v2 - with: - node-version: 14 - - - name: Restore Dependencies - uses: actions/cache@v2 - with: - path: | - node_modules - */*/node_modules - key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - - name: Downloading Docker Compose V2 if: ${{ ( matrix.database != 'bigquery') }} run: | @@ -140,7 +89,7 @@ jobs: - name: Set up Cloud SDK if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - + - name: Installing Dependencies run: npm install @@ -155,4 +104,3 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down - From 192e96185b42a63bebb1996f3e5d201033b618e4 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Wed, 15 Dec 2021 17:25:31 +0200 Subject: [PATCH 35/47] minor changes --- .../lib/bigquery_data_provider.js | 7 ------- .../lib/bigquery_schema_provider.js | 19 +++++++------------ 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/packages/external-db-bigquery/lib/bigquery_data_provider.js b/packages/external-db-bigquery/lib/bigquery_data_provider.js index dd769879e..d54dc5231 100644 --- a/packages/external-db-bigquery/lib/bigquery_data_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_data_provider.js @@ -34,13 +34,6 @@ class DataProvider { } async insert(collectionName, items) { - - // const patchedItems = items.map(patchDateTime) - // const itemsArr = patchedItems.map( item => `(${Object.values(item).map(i => typeof(i) === 'number' ? i : `"${i}"`).join(', ')})` ).join(', ') - // const sql = `INSERT INTO \`${collectionName}\` (${Object.keys(items[0]).join(', ')}) VALUES ${itemsArr}` - - // const res = await this.pool.query(sql).catch( translateErrorCodes ) - const table = await this.pool.table(collectionName) await table.insert(items) diff --git a/packages/external-db-bigquery/lib/bigquery_schema_provider.js b/packages/external-db-bigquery/lib/bigquery_schema_provider.js index b47d96d6f..54c10d853 100644 --- a/packages/external-db-bigquery/lib/bigquery_schema_provider.js +++ b/packages/external-db-bigquery/lib/bigquery_schema_provider.js @@ -13,18 +13,13 @@ class SchemaProvider { } async list() { - try{ - const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') - const tables = parseTableData(res[0]) - return Object.entries(tables) - .map(([collectionName, rs]) => ({ - id: collectionName, - fields: rs.map( this.translateDbTypes.bind(this) ) - })) - } catch(e) { - console.log(e) - } - + const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS') + const tables = parseTableData(res[0]) + return Object.entries(tables) + .map(([collectionName, rs]) => ({ + id: collectionName, + fields: rs.map( this.translateDbTypes.bind(this) ) + })) } async create(collectionName, _columns) { From a6abcb695b3d5c3ec6209660bf9ff0b095c31771 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:16:29 +0200 Subject: [PATCH 36/47] testing main yml --- .github/workflows/main.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37543e197..224794e51 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,3 +104,40 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down + + test_bigquery: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14 + uses: actions/setup-node@v2 + with: + node-version: 14 + + - name: Restore Dependencies + uses: actions/cache@v2 + with: + path: | + node_modules + */*/node_modules + key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + + - id: auth + uses: google-github-actions/auth@v0.4.0 + with: + credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.1 + + - name: Use gcloud CLI + run: gcloud info + + - name: Installing Dependencies + run: npm install + + - name: Install dependencies of the homemade packages. + run: npm run build:dev + + - name: Testing + run: npm run test:bigquery From b6f147600be48b55d2a99cd15d2c1d4750e58cd6 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 14:50:17 +0200 Subject: [PATCH 37/47] disable other tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 224794e51..23ae429b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -135,7 +135,7 @@ jobs: - name: Installing Dependencies run: npm install - + - name: Install dependencies of the homemade packages. run: npm run build:dev From b0d72ebccc6137ad62fc75d83e9a640e9d8b542e Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 5 Dec 2021 15:05:12 +0200 Subject: [PATCH 38/47] print errors --- packages/external-db-bigquery/lib/sql_exception_translator.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 41bd1c54b..3e7c5bfe5 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,6 +15,7 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { + console.log(err) throw notThrowingTranslateErrorCodes(err) } From 6d761c0d4aeac41491abd119b690d3a5e1c436f8 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:00:39 +0200 Subject: [PATCH 39/47] test fefactor bigquery --- packages/external-db-bigquery/lib/sql_exception_translator.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/external-db-bigquery/lib/sql_exception_translator.js b/packages/external-db-bigquery/lib/sql_exception_translator.js index 3e7c5bfe5..41bd1c54b 100644 --- a/packages/external-db-bigquery/lib/sql_exception_translator.js +++ b/packages/external-db-bigquery/lib/sql_exception_translator.js @@ -15,7 +15,6 @@ const notThrowingTranslateErrorCodes = err => { } const translateErrorCodes = err => { - console.log(err) throw notThrowingTranslateErrorCodes(err) } From ef3b141c62efe774f1ef051254a56c768b471416 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:21:07 +0200 Subject: [PATCH 40/47] test if in yml file --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 23ae429b6..ee71bdbc7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -123,6 +123,7 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - id: auth + if: ${{ (1 == 1) }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} From ef2045923784e5fb2d245a68ac3d648b6265f714 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:27:09 +0200 Subject: [PATCH 41/47] main yml refactor to support ifs --- .github/workflows/main.yml | 42 +++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee71bdbc7..ba53d41c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -107,8 +107,13 @@ jobs: test_bigquery: runs-on: ubuntu-latest + strategy: + matrix: + database: ["BigQuery"] + steps: - uses: actions/checkout@v2 + - name: Use Node.js 14 uses: actions/setup-node@v2 with: @@ -122,18 +127,34 @@ jobs: */*/node_modules key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + - name: Downloading Docker Compose V2 + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh + docker compose version + + - name: Downloading Image + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose pull --quiet ${{ matrix.database }} + + - name: Start Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose up --detach ${{ matrix.database }} + - id: auth - if: ${{ (1 == 1) }} + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - + - name: Set up Cloud SDK + if: ${{ ( matrix.database == 'BigQuery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - - - name: Use gcloud CLI - run: gcloud info - + - name: Installing Dependencies run: npm install @@ -141,4 +162,11 @@ jobs: run: npm run build:dev - name: Testing - run: npm run test:bigquery + run: npm run test:${{ matrix.database }} + + - name: Stop Container + if: ${{ ( matrix.database != 'BigQuery') }} + run: | + cd packages/velo-external-db/test/resources + docker compose down + From 96198a38ad55286ef9278429974a21bfcc42ba44 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:34:33 +0200 Subject: [PATCH 42/47] main yml refactor to support ifs --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba53d41c3..d42b9c1b9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -128,31 +128,31 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - name: Downloading Docker Compose V2 - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh docker compose version - name: Downloading Image - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose pull --quiet ${{ matrix.database }} - name: Start Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose up --detach ${{ matrix.database }} - id: auth - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/auth@v0.4.0 with: credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - name: Set up Cloud SDK - if: ${{ ( matrix.database == 'BigQuery') }} + if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - name: Installing Dependencies @@ -165,7 +165,7 @@ jobs: run: npm run test:${{ matrix.database }} - name: Stop Container - if: ${{ ( matrix.database != 'BigQuery') }} + if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose down From 9b028204d6d5b89309ab42ca3d23e6b875d43844 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Mon, 6 Dec 2021 17:38:30 +0200 Subject: [PATCH 43/47] main yml refactor to support ifs --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d42b9c1b9..1be6dd467 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,7 +154,7 @@ jobs: - name: Set up Cloud SDK if: ${{ ( matrix.database == 'bigquery') }} uses: google-github-actions/setup-gcloud@v0.2.1 - + - name: Installing Dependencies run: npm install @@ -169,4 +169,3 @@ jobs: run: | cd packages/velo-external-db/test/resources docker compose down - From 2abbdf73948e7f7e5aa21b0581a01b993ceb9c89 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 19 Dec 2021 12:10:37 +0200 Subject: [PATCH 44/47] disable test only for BigQuery --- .../velo-external-db/test/storage/schema_provider.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/velo-external-db/test/storage/schema_provider.spec.js b/packages/velo-external-db/test/storage/schema_provider.spec.js index cabce89c9..cf2476a6f 100644 --- a/packages/velo-external-db/test/storage/schema_provider.spec.js +++ b/packages/velo-external-db/test/storage/schema_provider.spec.js @@ -65,13 +65,13 @@ describe('Schema API', () => { await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields()) }) - // if (shouldNotRunOn(['BigQuery'], name)) { + if (shouldNotRunOn(['BigQuery'], name)) { test('create collection twice will do nothing', async() => { await env.schemaProvider.create(ctx.collectionName, []) await expect( env.schemaProvider.create(ctx.collectionName, []) ).resolves.toBeUndefined() }) - // } + } test('add column on a non existing collection will fail', async() => { await expect(env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })).rejects.toThrow(CollectionDoesNotExists) From d1bd8d49a24167390c8d14a6d9352c6ad4ba5899 Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 19 Dec 2021 12:16:59 +0200 Subject: [PATCH 45/47] nop --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1be6dd467..aa943b8c7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,7 +109,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - database: ["BigQuery"] + database: ["bigquery"] steps: - uses: actions/checkout@v2 From b8101261a29c82bf2eae3bb1df6bd80a075c59ba Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 19 Dec 2021 12:29:01 +0200 Subject: [PATCH 46/47] nop --- .github/workflows/main.yml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1be6dd467..08027bf56 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: "mongo", "mongo4", "firestore", "airtable", - "dynamodb","bigquery"] + "dynamodb"] steps: - uses: actions/checkout@v2 @@ -63,33 +63,20 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - name: Downloading Docker Compose V2 - if: ${{ ( matrix.database != 'bigquery') }} run: | curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh docker compose version - name: Downloading Image - if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose pull --quiet ${{ matrix.database }} - name: Start Container - if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose up --detach ${{ matrix.database }} - - id: auth - if: ${{ ( matrix.database == 'bigquery') }} - uses: google-github-actions/auth@v0.4.0 - with: - credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - - - name: Set up Cloud SDK - if: ${{ ( matrix.database == 'bigquery') }} - uses: google-github-actions/setup-gcloud@v0.2.1 - - name: Installing Dependencies run: npm install @@ -100,7 +87,6 @@ jobs: run: npm run test:${{ matrix.database }} - name: Stop Container - if: ${{ ( matrix.database != 'bigquery') }} run: | cd packages/velo-external-db/test/resources docker compose down @@ -109,7 +95,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - database: ["BigQuery"] + database: ["bigquery"] steps: - uses: actions/checkout@v2 From ee0514f87790d89c766810b19cb36b972eb988aa Mon Sep 17 00:00:00 2001 From: Max Polsky Date: Sun, 19 Dec 2021 12:34:39 +0200 Subject: [PATCH 47/47] disable bigquery tests for now --- .github/workflows/main.yml | 128 ++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 08027bf56..c60e7bec3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,67 +91,67 @@ jobs: cd packages/velo-external-db/test/resources docker compose down - test_bigquery: - runs-on: ubuntu-latest - strategy: - matrix: - database: ["bigquery"] - - steps: - - uses: actions/checkout@v2 - - - name: Use Node.js 14 - uses: actions/setup-node@v2 - with: - node-version: 14 - - - name: Restore Dependencies - uses: actions/cache@v2 - with: - path: | - node_modules - */*/node_modules - key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} - - - name: Downloading Docker Compose V2 - if: ${{ ( matrix.database != 'bigquery') }} - run: | - curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh - docker compose version - - - name: Downloading Image - if: ${{ ( matrix.database != 'bigquery') }} - run: | - cd packages/velo-external-db/test/resources - docker compose pull --quiet ${{ matrix.database }} - - - name: Start Container - if: ${{ ( matrix.database != 'bigquery') }} - run: | - cd packages/velo-external-db/test/resources - docker compose up --detach ${{ matrix.database }} - - - id: auth - if: ${{ ( matrix.database == 'bigquery') }} - uses: google-github-actions/auth@v0.4.0 - with: - credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} - - - name: Set up Cloud SDK - if: ${{ ( matrix.database == 'bigquery') }} - uses: google-github-actions/setup-gcloud@v0.2.1 - - - name: Installing Dependencies - run: npm install - - - name: Install dependencies of the homemade packages. - run: npm run build:dev - - - name: Testing - run: npm run test:${{ matrix.database }} - - - name: Stop Container - if: ${{ ( matrix.database != 'bigquery') }} - run: | - cd packages/velo-external-db/test/resources - docker compose down + # test_bigquery: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # database: ["bigquery"] + + # steps: + # - uses: actions/checkout@v2 + + # - name: Use Node.js 14 + # uses: actions/setup-node@v2 + # with: + # node-version: 14 + + # - name: Restore Dependencies + # uses: actions/cache@v2 + # with: + # path: | + # node_modules + # */*/node_modules + # key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} + + # - name: Downloading Docker Compose V2 + # if: ${{ ( matrix.database != 'bigquery') }} + # run: | + # curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh + # docker compose version + + # - name: Downloading Image + # if: ${{ ( matrix.database != 'bigquery') }} + # run: | + # cd packages/velo-external-db/test/resources + # docker compose pull --quiet ${{ matrix.database }} + + # - name: Start Container + # if: ${{ ( matrix.database != 'bigquery') }} + # run: | + # cd packages/velo-external-db/test/resources + # docker compose up --detach ${{ matrix.database }} + + # - id: auth + # if: ${{ ( matrix.database == 'bigquery') }} + # uses: google-github-actions/auth@v0.4.0 + # with: + # credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }} + + # - name: Set up Cloud SDK + # if: ${{ ( matrix.database == 'bigquery') }} + # uses: google-github-actions/setup-gcloud@v0.2.1 + + # - name: Installing Dependencies + # run: npm install + + # - name: Install dependencies of the homemade packages. + # run: npm run build:dev + + # - name: Testing + # run: npm run test:${{ matrix.database }} + + # - name: Stop Container + # if: ${{ ( matrix.database != 'bigquery') }} + # run: | + # cd packages/velo-external-db/test/resources + # docker compose down