Skip to content

Commit

Permalink
fix-spanner float (#92)
Browse files Browse the repository at this point in the history 
* insert/update as spanner.Float when needed

* refactor patchFloat
  • Loading branch information
@Idokah
Idokah authored Dec 13, 2021
1 parent 67e13a4 commit 17c6572
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 16 deletions.
22 changes: 15 additions & 7 deletions packages/external-db-spanner/lib/spanner_data_provider.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const { recordSetToObj, escapeId, patchFieldName, unpatchFieldName } = require('./spanner_utils')
const { recordSetToObj, escapeId, patchFieldName, unpatchFieldName, patchFloat, extractFloatFields } = require('./spanner_utils')

class DataProvider {
constructor(database, filterParser) {
Expand Down Expand Up @@ -37,10 +37,14 @@ class DataProvider {
return objs[0].num
}

async insert(collectionName, items) {
await this.database.table(collectionName)
.insert(items.map(this.asDBEntity.bind(this)))
return items.length
async insert(collectionName, items, _fields) {
const floatFields = extractFloatFields(_fields || [])
await this.database.table(collectionName)
.insert(
(items.map(item => patchFloat(item, floatFields)))
.map(this.asDBEntity.bind(this))
)
return items.length
}

asDBEntity(item) {
Expand Down Expand Up @@ -68,9 +72,13 @@ class DataProvider {
}.bind(this), {})
}

async update(collectionName, items) {
async update(collectionName, items, _fields) {
const floatFields = extractFloatFields(_fields || [])
await this.database.table(collectionName)
.update(items.map( this.asDBEntity.bind(this) ))
.update(
(items.map(item => patchFloat(item, floatFields)))
.map(this.asDBEntity.bind(this))
)
return items.length
}

Expand Down
19 changes: 18 additions & 1 deletion packages/external-db-spanner/lib/spanner_utils.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const { escapeId } = require('sqlstring')
const { InvalidQuery } = require('velo-external-db-commons').errors
const { Spanner } = require ('@google-cloud/spanner')

const recordSetToObj = (rows) => rows.map(row => row.toJSON())

Expand Down Expand Up @@ -27,4 +28,20 @@ const validateLiteral = l => {

const escapeFieldId = f => escapeId(patchFieldName(f))

module.exports = { recordSetToObj, escapeId, patchFieldName, unpatchFieldName, testLiteral, validateLiteral, escapeFieldId }

const patchFloat = (item, floatFields) => {
return Object.keys(item).reduce((pV, key) => (
{ ...pV, ...{ [key]: floatFields.includes(key) ? Spanner.float(item[key]) : item[key] } }
), {})
}
const extractFloatFields = fields => ( parseFields(fields).filter(f => isFloatSubtype(f.subtype)).map(f => f.name) )

const isFloatSubtype = (subtype) => (['float', 'double', 'decimal'].includes(subtype))

const parseFields = fields => (
Object.entries(fields).map(([name, v]) => ({ name, type: v.type, subtype: v.subtype }))
)

module.exports = { recordSetToObj, escapeId, patchFieldName, unpatchFieldName,
testLiteral, validateLiteral, escapeFieldId,
patchFloat, extractFloatFields }
59 changes: 59 additions & 0 deletions packages/external-db-spanner/lib/spanner_utils.spec.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
const { patchFloat, extractFloatFields } = require('./spanner_utils')
const { Spanner } = require ('@google-cloud/spanner')
const { Chance } = require('chance')
const { Uninitialized } = require('test-commons')

const chance = Chance()

describe('Spanner utils', () => {
test('patchFloat will wrap with Spanner.float float columns data', () => {
const item = {
floatColumn: ctx.integer,
doubleColumn: ctx.integer,
intColumn: ctx.integer,
field: ctx.text,
}

const fields = {
floatColumn: {
displayName: 'floatColumn',
type: 'number',
subtype: 'float',
},
doubleColumn: {
displayName: 'doubleColumn',
type: 'number',
subtype: 'double',
},
intColumn: {
displayName: 'intColumn',
type: 'number',
subtype: 'int',
},
field: {
displayName: 'field',
type: 'text',
subtype: 'string'
},
}

expect(patchFloat(item, extractFloatFields(fields))).toEqual(
{
floatColumn: Spanner.float(ctx.integer),
doubleColumn: Spanner.float(ctx.integer),
intColumn: ctx.integer,
field: ctx.text,
}
)
})

const ctx = {
integer: Uninitialized,
text: Uninitialized
}

beforeEach(() => {
ctx.integer = chance.integer()
ctx.text = chance.word()
})
})
4 changes: 2 additions & 2 deletions packages/velo-external-db-core/lib/service/data.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class DataService {
async bulkInsert(collectionName, items) {
const info = await this.schemaInformation.schemaFor(collectionName)
const prepared = items.map(i => prepareForInsert(i, Object.entries(info.fields).map(([k, v]) => ({ name: k, ...v }))))
await this.storage.insert(collectionName, prepared.map(i => unpackDates(i)))
await this.storage.insert(collectionName, prepared.map(i => unpackDates(i)), info.fields)
return { items: prepared }
}

Expand All @@ -47,7 +47,7 @@ class DataService {
async bulkUpdate(collectionName, items) {
const info = await this.schemaInformation.schemaFor(collectionName)
const prepared = items.map(i => prepareForUpdate(i, Object.entries(info.fields).map(([k, v]) => ({ name: k, ...v }))))
await this.storage.update(collectionName, prepared.map(i => unpackDates(i)))
await this.storage.update(collectionName, prepared.map(i => unpackDates(i)), info.fields)
return { items: prepared }
}

Expand Down
12 changes: 6 additions & 6 deletions packages/velo-external-db/test/storage/data_provider.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ describe('Data API', () => {
}, 20000)


const givenCollectionWith = async(entities, forCollection) => {
await env.dataProvider.insert(forCollection, entities)
const givenCollectionWith = async(entities, forCollection, fields) => {
await env.dataProvider.insert(forCollection, entities, fields)
}

test('search with empty filter and order by and no data', async() => {
Expand Down Expand Up @@ -101,7 +101,7 @@ describe('Data API', () => {
await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns)
env.driver.stubEmptyFilterAndSortFor('', '')

expect( await env.dataProvider.insert(ctx.numericCollectionName, [ctx.numberEntity]) ).toEqual(1)
expect( await env.dataProvider.insert(ctx.numericCollectionName, [ctx.numberEntity], gen.fieldsArrayToFieldObj(ctx.numericColumns)) ).toEqual(1)

expect( await env.dataProvider.find(ctx.numericCollectionName, '', '', 0, 50) ).toEqual([ctx.numberEntity])
})
Expand Down Expand Up @@ -154,7 +154,7 @@ describe('Data API', () => {
if (shouldNotRunOn(['Firestore', 'Airtable', 'DynamoDb'], name)) {
test('aggregate api without filter', async() => {
await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns)
await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName)
await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName, gen.fieldsArrayToFieldObj(ctx.numericColumns))

env.driver.stubEmptyFilterFor(ctx.filter)
env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1)
Expand All @@ -168,7 +168,7 @@ describe('Data API', () => {

test('aggregate api without having', async() => {
await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns)
await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName)
await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName, gen.fieldsArrayToFieldObj(ctx.numericColumns))

env.driver.stubEmptyFilterFor(ctx.filter)
env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1)
Expand All @@ -182,7 +182,7 @@ describe('Data API', () => {

test('aggregate api with filter', async() => {
await env.schemaProvider.create(ctx.numericCollectionName, ctx.numericColumns)
await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName)
await givenCollectionWith([ctx.numberEntity, ctx.anotherNumberEntity], ctx.numericCollectionName, gen.fieldsArrayToFieldObj(ctx.numericColumns))

env.driver.givenFilterByIdWith(ctx.numberEntity._id, ctx.filter)
env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 2)
Expand Down

0 comments on commit 17c6572

Please sign in to comment.