BigQuery tests

.github/workflows/main.yml

@@ -90,3 +90,68 @@ jobs:
         run: |
           cd packages/velo-external-db/test/resources
           docker compose down
+
+  # test_bigquery:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       database: ["bigquery"]
+
+  #   steps:
+  #     - uses: actions/checkout@v2
+
+  #     - name: Use Node.js 14
+  #       uses: actions/setup-node@v2
+  #       with:
+  #         node-version: 14
+
+  #     - name: Restore Dependencies
+  #       uses: actions/cache@v2
+  #       with:
+  #         path: |
+  #           node_modules
+  #           */*/node_modules
+  #         key: ${{ runner.os }}-${{ hashFiles('**/package.json') }}
+
+  #     - name: Downloading Docker Compose V2
+  #       if: ${{ ( matrix.database != 'bigquery') }}
+  #       run: |
+  #         curl -sfL https://raw.githubusercontent.com/docker/compose-cli/main/scripts/install/install_linux.sh | sh
+  #         docker compose version
+
+  #     - name: Downloading Image
+  #       if: ${{ ( matrix.database != 'bigquery') }}
+  #       run: |
+  #         cd packages/velo-external-db/test/resources
+  #         docker compose pull --quiet ${{ matrix.database }}
+
+  #     - name: Start Container
+  #       if: ${{ ( matrix.database != 'bigquery') }}
+  #       run: |
+  #         cd packages/velo-external-db/test/resources
+  #         docker compose up --detach ${{ matrix.database }}
+
+  #     - id: auth
+  #       if: ${{ ( matrix.database == 'bigquery') }}
+  #       uses: google-github-actions/[email protected]
+  #       with:
+  #         credentials_json: ${{ secrets.ACTIONS_GCP_CREDENTIALS }}
+
+  #     - name: Set up Cloud SDK
+  #       if: ${{ ( matrix.database == 'bigquery') }}
+  #       uses: google-github-actions/[email protected]
+
+  #     - name: Installing Dependencies
+  #       run: npm install
+
+  #     - name: Install dependencies of the homemade packages.
+  #       run: npm run build:dev
+
+  #     - name: Testing
+  #       run: npm run test:${{ matrix.database }}
+
+  #     - name: Stop Container
+  #       if: ${{ ( matrix.database != 'bigquery') }}
+  #       run: |
+  #         cd packages/velo-external-db/test/resources
+  #         docker compose down

packages/external-db-airtable/package.json

@@ -33,9 +33,9 @@
   },
   "devDependencies": {
     "chance": "^1.1.8",
-    "jest": "^27.4.4",
+    "jest": "^27.4.5",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1",
-    "jest-when": "^3.4.2"
+    "jest-when": "^3.5.0"
   }
 }

packages/external-db-bigquery/lib/bigquery_data_provider.js

@@ -34,16 +34,10 @@ class DataProvider {
     }
 
     async insert(collectionName, items) {
-        // const patchedItems = items.map(patchDateTime)
-        // const itemsArr = patchedItems.map( item => `(${Object.values(item).map(i => typeof(i) === 'number' ? i : `"${i}"`).join(', ')})` ).join(', ')
-        // const sql = `INSERT INTO \`${collectionName}\` (${Object.keys(items[0]).join(', ')}) VALUES ${itemsArr}`
-
-        // const res = await this.pool.query(sql).catch( translateErrorCodes )
-
         const table = await this.pool.table(collectionName)
-        const res = await table.insert(items)
-    
-        return res
+        await table.insert(items)
+
+        return items.length
     }
 
     async update(collectionName, items) {
@@ -82,7 +76,7 @@ class DataProvider {
         const resultSet = await this.pool.query({ query: sql, params: [...whereParameters, ...parameters] })
                                     .catch( translateErrorCodes )
 
-        return resultSet[0]
+        return resultSet[0].map( unPatchDateTime )
     }
 }
 

packages/external-db-bigquery/lib/bigquery_schema_provider.js

@@ -5,61 +5,64 @@ const { SystemFields, validateSystemFields, parseTableData } = require('velo-ext
 
 const escapeIdentifier = i => i
 class SchemaProvider {
-    constructor(pool) {
+    constructor(pool, { projectId, databaseId }) {
+        this.projectId = projectId
+        this.databaseId = databaseId
         this.pool = pool
         this.sqlSchemaTranslator = new SchemaColumnTranslator()
     }
 
     async list() {
-        const res = await this.pool.query('SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS')
+        const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM ${this.projectId}.${this.databaseId}.INFORMATION_SCHEMA.COLUMNS`)
         const tables = parseTableData(res[0])
-
         return Object.entries(tables)
-                     .map(([collectionName, rs]) => ({
-                         id: collectionName,
-                         fields: rs.map( this.translateDbTypes.bind(this) )
-                     }))
+                        .map(([collectionName, rs]) => ({
+                            id: collectionName,
+                            fields: rs.map( this.translateDbTypes.bind(this) )
+                        }))
     }
 
     async create(collectionName, _columns) {
         const columns = _columns || []
-        const dbColumnsSql = [...SystemFields, ...columns].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c))
+        const dbColumnsSql = [...SystemFields, ...columns].map(c => this.sqlSchemaTranslator.columnToDbColumnSql(c))
         await this.pool.createTable(collectionName, { schema: dbColumnsSql })
-                  .catch(translateErrorCodes)         
+                       .catch(translateErrorCodes)
     }
 
     async drop(collectionName) {
         await this.pool.table(collectionName).delete()
-                  .catch(translateErrorCodes)  
+            .catch(translateErrorCodes)
     }
 
 
-    async addColumn(collectionName, column) {
+    async addColumn(collectionName, column) {   
         await validateSystemFields(column.name)
-
-        await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`)
-                  .catch(translateErrorCodes)
-
+        try{
+            await this.pool.query(`ALTER TABLE ${escapeIdentifier(collectionName)} ADD COLUMN ${escapeIdentifier(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`)
+        } catch (err) {
+            if (err.message.includes('was not found')) 
+                throw new CollectionDoesNotExists('Collection does not exists')
+            else 
+                translateErrorCodes(err)
+        }
     }
 
     async removeColumn(collectionName, columnName) {
         await validateSystemFields(columnName)
-
-        await this.pool.query(`CREATE OR REPLACE TABLE ${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM  ${escapeIdentifier(collectionName)}`)
-                         .catch(translateErrorCodes)
-
+        await this.pool.query(`CREATE OR REPLACE TABLE ${this.projectId}.${this.databaseId}.${escapeIdentifier(collectionName)} AS SELECT * EXCEPT (${escapeIdentifier(columnName)}) FROM  ${escapeIdentifier(collectionName)}`)
+            .catch(translateErrorCodes)
     }
 
     async describeCollection(collectionName) {
-        const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`)
+        const res = await this.pool.query(`SELECT table_name, column_name AS field, data_type as type, FROM ${this.projectId}.${this.databaseId}.INFORMATION_SCHEMA.COLUMNS WHERE table_name="${escapeIdentifier(collectionName)}"`)
 
         if (res[0].length === 0) {
             throw new CollectionDoesNotExists('Collection does not exists')
         }
+
         return res[0].map( this.translateDbTypes.bind(this) )
     }
 
-
     translateDbTypes(row) {
         row.type = this.sqlSchemaTranslator.translateType(row.type)
         return row

packages/external-db-bigquery/lib/bigquery_utils.js

@@ -24,7 +24,7 @@ const unPatchDateTime = (item) => {
         if (reISO.test(value)) {
             obj[key] = new Date(value)
         } else {
-            obj[key] = item[key]
+            obj[key] = item[key].toNumber ? item[key].toNumber() : item[key]
         }
     }
 

packages/external-db-bigquery/lib/connection_provider.js

@@ -4,14 +4,14 @@ const DatabaseOperations = require ('./bigquery_operations')
 const SchemaProvider = require('./bigquery_schema_provider')
 const DataProvider = require('./bigquery_data_provider')
 
-const init = ( cfg ) => {    
+const init = ({ projectId, databaseId }) => {
     const bigquery = new BigQuery()
-    const pool = bigquery.dataset(cfg.databaseId)
+    const pool = bigquery.dataset(databaseId)
 
     const filterParser = new FilterParser()
     const databaseOperations = new DatabaseOperations(pool)
     const dataProvider = new DataProvider(pool, filterParser)
-    const schemaProvider = new SchemaProvider(pool)
+    const schemaProvider = new SchemaProvider(pool, { projectId, databaseId })
 
     return { dataProvider: dataProvider, schemaProvider: schemaProvider, databaseOperations, connection: pool,  cleanup: async() => {} }
 }

packages/external-db-bigquery/package.json

@@ -26,7 +26,7 @@
   "devDependencies": {
     "chance": "^1.1.8",
     "jest-each": "^27.4.2",
-    "jest-when": "^3.4.2",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0"
   }
 }

packages/external-db-bigquery/tests/drivers/sql_filter_transformer_test_support.js

@@ -33,11 +33,11 @@ const givenOrderByFor = (column, sort) => {
 
 const givenFilterByIdWith = (id, filter) => {
     when(filterParser.transform).calledWith(filter)
-                                .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = $1`, parameters: [id], offset: 2 })
+                                .mockReturnValue({ filterExpr: `WHERE ${escapeIdentifier('_id')} = ?`, parameters: [id], offset: 2 })
 }
 
 const givenAggregateQueryWith = (having, numericColumns, columnAliases, groupByColumns, filter, offest) => {
-    when(filterParser.parseAggregation).calledWith(having, filter, offest)
+    when(filterParser.parseAggregation).calledWith(having, filter)
                                        .mockReturnValue({
                                            fieldsStatement: `${groupByColumns.map( escapeIdentifier ).join(', ')}, MAX(${escapeIdentifier(numericColumns[0].name)}) AS ${escapeIdentifier(columnAliases[0])}, SUM(${escapeIdentifier(numericColumns[1].name)}) AS ${escapeIdentifier(columnAliases[1])}`,
                                            groupByColumns: groupByColumns,

packages/external-db-config/lib/readers/gcp_config_reader.js

@@ -99,13 +99,13 @@ class GcpBigQueryConfigReader {
   }
 
   async readConfig() {
-    const { DATABASE_ID, SECRET_KEY } = process.env
-    return { databaseId: DATABASE_ID, secretKey: SECRET_KEY }
+    const { PROJECT_ID, DATABASE_ID, SECRET_KEY } = process.env
+    return { projectId: PROJECT_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY }
   }
 
   validate() {
     return {
-      missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['DATABASE_ID', 'SECRET_KEY'])
+      missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['PROJECT_ID', 'DATABASE_ID', 'SECRET_KEY'])
     }
   }
 

packages/external-db-config/package.json

@@ -25,9 +25,9 @@
 	"devDependencies": {
 		"aws-sdk-client-mock": "^0.5.6",
 		"chance": "^1.1.8",
-		"jest": "^27.4.4",
+		"jest": "^27.4.5",
 		"jest-each": "^27.4.2",
-		"jest-when": "^3.4.2",
+		"jest-when": "^3.5.0",
 		"ts-jest": "^27.1.1",
 		"test-commons": "^0.0.0"
 	},

packages/external-db-dynamodb/package.json

@@ -33,8 +33,8 @@
   },
   "devDependencies": {
     "chance": "^1.1.8",
-    "jest": "^27.4.4",
-    "jest-when": "^3.4.2",
+    "jest": "^27.4.5",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1"
   }

packages/external-db-firestore/package.json

@@ -30,8 +30,8 @@
     "velo-external-db-commons": "^0.0.0"
   },
   "devDependencies": {
-    "jest": "^27.4.4",
-    "jest-when": "^3.4.2",
+    "jest": "^27.4.5",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1"
   }

packages/external-db-google-sheets/package.json

@@ -22,7 +22,7 @@
     "velo-external-db-commons": "^0.0.0"
   },
   "devDependencies": {
-    "jest": "^27.4.4",
+    "jest": "^27.4.5",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1"
   }

packages/external-db-mongo/package.json

@@ -26,12 +26,12 @@
     "url": "https://github.com/wix-private/noam-external-db-poc/issues"
   },
   "dependencies": {
-    "mongodb": "^4.2.1",
+    "mongodb": "^4.2.2",
     "velo-external-db-commons": "^0.0.0"
   },
   "devDependencies": {
-    "jest": "^27.4.4",
-    "jest-when": "^3.4.2",
+    "jest": "^27.4.5",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1",
     "chance": "^1.1.8"

packages/external-db-mssql/package.json

@@ -31,8 +31,8 @@
     "velo-external-db-commons": "^0.0.0"
   },
   "devDependencies": {
-    "jest": "^27.4.4",
-    "jest-when": "^3.4.2",
+    "jest": "^27.4.5",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1",
     "chance": "^1.1.8"

packages/external-db-mysql/package.json

@@ -30,8 +30,8 @@
     "velo-external-db-commons": "^0.0.0"
   },
   "devDependencies": {
-    "jest": "^27.4.4",
-    "jest-when": "^3.4.2",
+    "jest": "^27.4.5",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1",
     "chance": "^1.1.8"

packages/external-db-postgres/package.json

@@ -31,8 +31,8 @@
     "velo-external-db-commons": "^0.0.0"
   },
   "devDependencies": {
-    "jest": "^27.4.4",
-    "jest-when": "^3.4.2",
+    "jest": "^27.4.5",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1",
     "chance": "^1.1.8"

packages/external-db-spanner/package.json

@@ -31,9 +31,9 @@
   },
   "devDependencies": {
     "chance": "^1.1.8",
-    "jest": "^27.4.4",
+    "jest": "^27.4.5",
     "jest-each": "^27.4.2",
-    "jest-when": "^3.4.2",
+    "jest-when": "^3.5.0",
     "test-commons": "^0.0.0",
     "ts-jest": "^27.1.1"
   },