diff --git a/packages/external-db-spanner/lib/spanner_data_provider.js b/packages/external-db-spanner/lib/spanner_data_provider.js index 0e3a96fa7..e9b274306 100644 --- a/packages/external-db-spanner/lib/spanner_data_provider.js +++ b/packages/external-db-spanner/lib/spanner_data_provider.js @@ -1,4 +1,4 @@ -const { recordSetToObj, escapeId, patchFieldName, unpatchFieldName } = require('./spanner_utils') +const { recordSetToObj, escapeId, patchFieldName, unpatchFieldName, patchFloat, extractFloatFields } = require('./spanner_utils') class DataProvider { constructor(database, filterParser) { @@ -37,10 +37,14 @@ class DataProvider { return objs[0].num } - async insert(collectionName, items) { - await this.database.table(collectionName) - .insert(items.map(this.asDBEntity.bind(this))) - return items.length + async insert(collectionName, items, _fields) { + const floatFields = extractFloatFields(_fields || []) + await this.database.table(collectionName) + .insert( + (items.map(item => patchFloat(item, floatFields))) + .map(this.asDBEntity.bind(this)) + ) + return items.length } asDBEntity(item) { @@ -68,9 +72,13 @@ class DataProvider { }.bind(this), {}) } - async update(collectionName, items) { + async update(collectionName, items, _fields) { + const floatFields = extractFloatFields(_fields || []) await this.database.table(collectionName) - .update(items.map( this.asDBEntity.bind(this) )) + .update( + (items.map(item => patchFloat(item, floatFields))) + .map(this.asDBEntity.bind(this)) + ) return items.length } diff --git a/packages/external-db-spanner/lib/spanner_utils.js b/packages/external-db-spanner/lib/spanner_utils.js index 7aa8e192c..675860a66 100644 --- a/packages/external-db-spanner/lib/spanner_utils.js +++ b/packages/external-db-spanner/lib/spanner_utils.js @@ -1,5 +1,6 @@ const { escapeId } = require('sqlstring') const { InvalidQuery } = require('velo-external-db-commons').errors +const { Spanner } = require ('@google-cloud/spanner') const recordSetToObj = (rows) => rows.map(row => row.toJSON()) @@ -27,4 +28,20 @@ const validateLiteral = l => { const escapeFieldId = f => escapeId(patchFieldName(f)) -module.exports = { recordSetToObj, escapeId, patchFieldName, unpatchFieldName, testLiteral, validateLiteral, escapeFieldId } + +const patchFloat = (item, floatFields) => { + return Object.keys(item).reduce((pV, key) => ( + { ...pV, ...{ [key]: floatFields.includes(key) ? Spanner.float(item[key]) : item[key] } } + ), {}) +} +const extractFloatFields = fields => ( parseFields(fields).filter(f => isFloatSubtype(f.subtype)).map(f => f.name) ) + +const isFloatSubtype = (subtype) => (['float', 'double', 'decimal'].includes(subtype)) + +const parseFields = fields => ( + Object.entries(fields).map(([name, v]) => ({ name, type: v.type, subtype: v.subtype })) +) + +module.exports = { recordSetToObj, escapeId, patchFieldName, unpatchFieldName, + testLiteral, validateLiteral, escapeFieldId, + patchFloat, extractFloatFields } diff --git a/packages/external-db-spanner/lib/spanner_utils.spec.js b/packages/external-db-spanner/lib/spanner_utils.spec.js new file mode 100644 index 000000000..eeefc5252 --- /dev/null +++ b/packages/external-db-spanner/lib/spanner_utils.spec.js @@ -0,0 +1,59 @@ +const { patchFloat, extractFloatFields } = require('./spanner_utils') +const { Spanner } = require ('@google-cloud/spanner') +const { Chance } = require('chance') +const { Uninitialized } = require('test-commons') + +const chance = Chance() + +describe('Spanner utils', () => { + test('patchFloat will wrap with Spanner.float float columns data', () => { + const item = { + floatColumn: ctx.integer, + doubleColumn: ctx.integer, + intColumn: ctx.integer, + field: ctx.text, + } + + const fields = { + floatColumn: { + displayName: 'floatColumn', + type: 'number', + subtype: 'float', + }, + doubleColumn: { + displayName: 'doubleColumn', + type: 'number', + subtype: 'double', + }, + intColumn: { + displayName: 'intColumn', + type: 'number', + subtype: 'int', + }, + field: { + displayName: 'field', + type: 'text', + subtype: 'string' + }, + } + + expect(patchFloat(item, extractFloatFields(fields))).toEqual( + { + floatColumn: Spanner.float(ctx.integer), + doubleColumn: Spanner.float(ctx.integer), + intColumn: ctx.integer, + field: ctx.text, + } + ) + }) + + const ctx = { + integer: Uninitialized, + text: Uninitialized + } + + beforeEach(() => { + ctx.integer = chance.integer() + ctx.text = chance.word() + }) +}) \ No newline at end of file diff --git a/packages/velo-external-db-core/lib/service/data.js b/packages/velo-external-db-core/lib/service/data.js index a36f8c696..1b757a83f 100644 --- a/packages/velo-external-db-core/lib/service/data.js +++ b/packages/velo-external-db-core/lib/service/data.js @@ -35,7 +35,7 @@ class DataService { async bulkInsert(collectionName, items) { const info = await this.schemaInformation.schemaFor(collectionName) const prepared = items.map(i => prepareForInsert(i, Object.entries(info.fields).map(([k, v]) => ({ name: k, ...v })))) - await this.storage.insert(collectionName, prepared.map(i => unpackDates(i))) + await this.storage.insert(collectionName, prepared.map(i => unpackDates(i)), info.fields) return { items: prepared } } @@ -47,7 +47,7 @@ class DataService { async bulkUpdate(collectionName, items) { const info = await this.schemaInformation.schemaFor(collectionName) const prepared = items.map(i => prepareForUpdate(i, Object.entries(info.fields).map(([k, v]) => ({ name: k, ...v })))) - await this.storage.update(collectionName, prepared.map(i => unpackDates(i))) + await this.storage.update(collectionName, prepared.map(i => unpackDates(i)), info.fields) return { items: prepared } } diff --git a/packages/velo-external-db/test/storage/data_provider.spec.js b/packages/velo-external-db/test/storage/data_provider.spec.js index d84b4ae08..39deb1375 100644 --- a/packages/velo-external-db/test/storage/data_provider.spec.js +++ b/packages/velo-external-db/test/storage/data_provider.spec.js @@ -17,8 +17,8 @@ describe('Data API', () => { }, 20000) - const givenCollectionWith = async(entities, forCollection) => { - await env.dataProvider.insert(forCollection, entities) + const givenCollectionWith = async(entities, forCollection, fields) => { + await env.dataProvider.insert(forCollection, entities, fields) } test('search with empty filter and order by and no data', async() => { @@ -101,7 +101,7 @@ describe('Data API', () => { await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns) env.driver.stubEmptyFilterAndSortFor('', '') - expect( await env.dataProvider.insert(ctx.numericCollectionName, [ctx.numberEntity]) ).toEqual(1) + expect( await env.dataProvider.insert(ctx.numericCollectionName, [ctx.numberEntity], gen.fieldsArrayToFieldObj(ctx.numericColumns)) ).toEqual(1) expect( await env.dataProvider.find(ctx.numericCollectionName, '', '', 0, 50) ).toEqual([ctx.numberEntity]) }) @@ -154,7 +154,7 @@ describe('Data API', () => { if (shouldNotRunOn(['Firestore', 'Airtable', 'DynamoDb'], name)) { test('aggregate api without filter', async() => { await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns) - await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName) + await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName, gen.fieldsArrayToFieldObj(ctx.numericColumns)) env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1) @@ -168,7 +168,7 @@ describe('Data API', () => { test('aggregate api without having', async() => { await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns) - await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName) + await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName, gen.fieldsArrayToFieldObj(ctx.numericColumns)) env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1) @@ -182,7 +182,7 @@ describe('Data API', () => { test('aggregate api with filter', async() => { await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns) - await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName) + await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName, gen.fieldsArrayToFieldObj(ctx.numericColumns)) env.driver.givenFilterByIdWith(ctx.numberEntity._id, ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 2)