From 74bcb3e347888d7c6953cb7209caf783abf93c2a Mon Sep 17 00:00:00 2001 From: Stefan Probst Date: Sun, 30 Dec 2018 21:53:30 +0100 Subject: [PATCH 01/20] Add serialization with v8 --- packages/gatsby/src/redux/index.js | 54 +++++------------------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index b503211d55a4b..755d1323cfc92 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -2,7 +2,7 @@ const Redux = require(`redux`) const _ = require(`lodash`) const fs = require(`fs`) const mitt = require(`mitt`) -const stringify = require(`json-stringify-safe`) +const v8 = require(`v8`) // Create event emitter for actions const emitter = mitt() @@ -10,43 +10,17 @@ const emitter = mitt() // Reducers const reducers = require(`./reducers`) -const objectToMap = obj => { - let map = new Map() - Object.keys(obj).forEach(key => { - map.set(key, obj[key]) - }) - return map -} +const readFileSync = file => v8.deserialize(fs.readFileSync(file)) -const mapToObject = map => { - const obj = {} - for (let [key, value] of map) { - obj[key] = value - } - return obj -} +const writeFileSync = (file, contents) => + fs.writeFileSync(file, v8.serialize(contents)) + +const file = `${process.cwd()}/.cache/redux-state.json` -// Read from cache the old node data. +// Read old node data from cache. let initialState = {} try { - const file = fs.readFileSync(`${process.cwd()}/.cache/redux-state.json`) - // Apparently the file mocking in node-tracking-test.js - // can override the file reading replacing the mocked string with - // an already parsed object. - if (Buffer.isBuffer(file) || typeof file === `string`) { - initialState = JSON.parse(file) - } - if (initialState.staticQueryComponents) { - initialState.staticQueryComponents = objectToMap( - initialState.staticQueryComponents - ) - } - if (initialState.components) { - initialState.components = objectToMap(initialState.components) - } - if (initialState.nodes) { - initialState.nodes = objectToMap(initialState.nodes) - } + initialState = readFileSync(file) } catch (e) { // ignore errors. } @@ -74,17 +48,7 @@ function saveState() { `staticQueryComponents`, ]) - pickedState.staticQueryComponents = mapToObject( - pickedState.staticQueryComponents - ) - pickedState.components = mapToObject(pickedState.components) - pickedState.nodes = pickedState.nodes ? mapToObject(pickedState.nodes) : [] - const stringified = stringify(pickedState, null, 2) - fs.writeFile( - `${process.cwd()}/.cache/redux-state.json`, - stringified, - () => {} - ) + writeFileSync(file, pickedState) } exports.saveState = saveState From af7bdb14a07eaf4ee73fa69a8e589b9386b561aa Mon Sep 17 00:00:00 2001 From: Stefan Probst Date: Sun, 30 Dec 2018 22:35:50 +0100 Subject: [PATCH 02/20] Change filename --- packages/gatsby/src/redux/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index 755d1323cfc92..1d13771158abc 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -15,7 +15,7 @@ const readFileSync = file => v8.deserialize(fs.readFileSync(file)) const writeFileSync = (file, contents) => fs.writeFileSync(file, v8.serialize(contents)) -const file = `${process.cwd()}/.cache/redux-state.json` +const file = `${process.cwd()}/.cache/redux.state` // Read old node data from cache. let initialState = {} From 04938eb6e51722b13086c8a1bcc443c0e440df8d Mon Sep 17 00:00:00 2001 From: Stefan Probst Date: Wed, 2 Jan 2019 23:55:26 +0100 Subject: [PATCH 03/20] Add back json fallback --- packages/gatsby/src/redux/index.js | 41 ++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index 1d13771158abc..824f56ce1b077 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -3,6 +3,7 @@ const _ = require(`lodash`) const fs = require(`fs`) const mitt = require(`mitt`) const v8 = require(`v8`) +const stringify = require(`json-stringify-safe`) // Create event emitter for actions const emitter = mitt() @@ -10,10 +11,46 @@ const emitter = mitt() // Reducers const reducers = require(`./reducers`) -const readFileSync = file => v8.deserialize(fs.readFileSync(file)) +const objectToMap = obj => { + const map = new Map() + Object.keys(obj).forEach(key => { + map.set(key, obj[key]) + }) + return map +} + +const mapToObject = map => { + const obj = {} + for (let [key, value] of map) { + obj[key] = value + } + return obj +} + +const jsonStringify = contents => { + contents.staticQueryComponents = mapToObject(contents.staticQueryComponents) + contents.components = mapToObject(contents.components) + contents.nodes = mapToObject(contents.nodes) + return stringify(contents, null, 2) +} + +const jsonParse = buffer => { + const parsed = JSON.parse(buffer.toString(`utf8`)) + parsed.staticQueryComponents = objectToMap(parsed.staticQueryComponents) + parsed.components = objectToMap(parsed.components) + parsed.nodes = objectToMap(parsed.nodes) + return parsed +} + +const useV8 = Boolean(v8.serialize) +const [serialize, deserialize] = useV8 + ? [v8.serialize, v8.deserialize] + : [jsonStringify, jsonParse] + +const readFileSync = file => deserialize(fs.readFileSync(file)) const writeFileSync = (file, contents) => - fs.writeFileSync(file, v8.serialize(contents)) + fs.writeFileSync(file, serialize(contents)) const file = `${process.cwd()}/.cache/redux.state` From 7c60f6c4ce0846c85d4e1cbab2d9c5b0368e4286 Mon Sep 17 00:00:00 2001 From: Stefan Probst Date: Wed, 16 Jan 2019 00:38:57 +0100 Subject: [PATCH 04/20] Don't use non-serializeable function in pluginOptions --- examples/using-gatsby-image/gatsby-config.js | 2 +- .../plugins/gatsby-source-remote-images/gatsby-node.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/using-gatsby-image/gatsby-config.js b/examples/using-gatsby-image/gatsby-config.js index 3324d3e993035..d7b4e8e710d5a 100644 --- a/examples/using-gatsby-image/gatsby-config.js +++ b/examples/using-gatsby-image/gatsby-config.js @@ -20,7 +20,7 @@ module.exports = { { resolve: `gatsby-source-remote-images`, options: { - filter: node => node.internal.type === `UnsplashImagesYaml`, + typeName: `UnsplashImagesYaml`, }, }, ], diff --git a/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js b/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js index 5b5a86be28eed..59f72f2626065 100644 --- a/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js +++ b/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js @@ -2,9 +2,9 @@ const { createRemoteFileNode } = require(`gatsby-source-filesystem`) exports.onCreateNode = async ( { actions: { createNode }, node, createContentDigest, store, cache }, - { filter, nodeName = `localFile` } + { typeName, nodeName = `localFile` } ) => { - if (filter(node)) { + if (node.internal.type === typeName) { const fileNode = await createRemoteFileNode({ url: node.url, store, From 2cc2036dec37ed6ea922bee1aec36c01e82031d7 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 21 Mar 2019 20:03:01 +0100 Subject: [PATCH 05/20] revert plugin changes --- examples/using-gatsby-image/gatsby-config.js | 2 +- .../plugins/gatsby-source-remote-images/gatsby-node.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/using-gatsby-image/gatsby-config.js b/examples/using-gatsby-image/gatsby-config.js index d7b4e8e710d5a..3324d3e993035 100644 --- a/examples/using-gatsby-image/gatsby-config.js +++ b/examples/using-gatsby-image/gatsby-config.js @@ -20,7 +20,7 @@ module.exports = { { resolve: `gatsby-source-remote-images`, options: { - typeName: `UnsplashImagesYaml`, + filter: node => node.internal.type === `UnsplashImagesYaml`, }, }, ], diff --git a/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js b/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js index 59f72f2626065..5b5a86be28eed 100644 --- a/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js +++ b/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js @@ -2,9 +2,9 @@ const { createRemoteFileNode } = require(`gatsby-source-filesystem`) exports.onCreateNode = async ( { actions: { createNode }, node, createContentDigest, store, cache }, - { typeName, nodeName = `localFile` } + { filter, nodeName = `localFile` } ) => { - if (node.internal.type === typeName) { + if (filter(node)) { const fileNode = await createRemoteFileNode({ url: node.url, store, From b87c8dfb04cd8ffeb05631a585cc03b96976fbf3 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 21 Mar 2019 20:03:39 +0100 Subject: [PATCH 06/20] remove pluginOptions from `sitePlugin` node - this cause trouble with inferring anyway --- .../src/internal-plugins/internal-data-bridge/gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js index 7f134cebffd7c..22d1da5e40188 100644 --- a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js +++ b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js @@ -61,7 +61,7 @@ exports.sourceNodes = ({ createContentDigest, actions, store }) => { }, }) - flattenedPlugins.forEach(plugin => { + flattenedPlugins.forEach(({ pluginOptions, ...plugin }) => { plugin.pluginFilepath = plugin.resolve createNode({ ...plugin, From 62a991ce7b60340255b2af0c33c5f96ee1572bf7 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 21 Mar 2019 20:16:15 +0100 Subject: [PATCH 07/20] keep `redux-state.json` path for json-stringify --- packages/gatsby/src/redux/index.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index 4357586181d21..42f029769ca4a 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -43,16 +43,15 @@ const jsonParse = buffer => { } const useV8 = Boolean(v8.serialize) -const [serialize, deserialize] = useV8 - ? [v8.serialize, v8.deserialize] - : [jsonStringify, jsonParse] +const [serialize, deserialize, file] = useV8 + ? [v8.serialize, v8.deserialize, `${process.cwd()}/.cache/redux.state`] + : [jsonStringify, jsonParse, `${process.cwd()}/.cache/redux-state.json`] const readFileSync = file => deserialize(fs.readFileSync(file)) -const writeFileSync = (file, contents) => +const writeFileSync = (file, contents) => { fs.writeFileSync(file, serialize(contents)) - -const file = `${process.cwd()}/.cache/redux.state` +} // Read old node data from cache. let initialState = {} From 82099dfbe7ff468fabfc542b6bd52fde24bec758 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sat, 30 Mar 2019 19:47:07 +0100 Subject: [PATCH 08/20] sanitize node data --- packages/gatsby/src/db/node-tracking.js | 26 +++++++++++++++++++++---- packages/gatsby/src/redux/actions.js | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/gatsby/src/db/node-tracking.js b/packages/gatsby/src/db/node-tracking.js index b344a6266136c..569f972d696ae 100644 --- a/packages/gatsby/src/db/node-tracking.js +++ b/packages/gatsby/src/db/node-tracking.js @@ -16,10 +16,28 @@ const getRootNodeId = node => rootNodeMap.get(node) * @param {(Object|Array)} data Inline object or array * @param {string} nodeId Id of node that contains data passed in first parameter */ -const addRootNodeToInlineObject = (data, nodeId) => { +const addRootNodeToInlineObject = (data, nodeId, sanitize, parent, key) => { if (_.isPlainObject(data) || _.isArray(data)) { - _.each(data, o => addRootNodeToInlineObject(o, nodeId)) + _.each(data, (o, key) => { + addRootNodeToInlineObject(o, nodeId, sanitize, key, data) + }) rootNodeMap.set(data, nodeId) + + // arrays and plain objects are supported - no need to to sanitize + return + } + + if (sanitize) { + const type = typeof data + // supported types + const isSupported = + type === `number` || + type === `string` || + type === `boolean` || + data instanceof Date + if (!isSupported) { + delete parent[key] + } } } @@ -28,13 +46,13 @@ const addRootNodeToInlineObject = (data, nodeId) => { * and that Node object. * @param {Node} node Root Node */ -const trackInlineObjectsInRootNode = node => { +const trackInlineObjectsInRootNode = (node, sanitize = false) => { _.each(node, (v, k) => { // Ignore the node internal object. if (k === `internal`) { return } - addRootNodeToInlineObject(v, node.id) + addRootNodeToInlineObject(v, node.id, sanitize, node, k) }) return node diff --git a/packages/gatsby/src/redux/actions.js b/packages/gatsby/src/redux/actions.js index 5db90337089fb..45f1486d00a37 100644 --- a/packages/gatsby/src/redux/actions.js +++ b/packages/gatsby/src/redux/actions.js @@ -591,7 +591,7 @@ actions.createNode = ( ) } - trackInlineObjectsInRootNode(node) + trackInlineObjectsInRootNode(node, true) const oldNode = getNode(node.id) From db6740f834a8db98ba209273a2eb9167d91a8bd8 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sat, 30 Mar 2019 20:14:20 +0100 Subject: [PATCH 09/20] Revert "remove pluginOptions from `sitePlugin` node - this cause trouble with inferring anyway" This reverts commit b87c8dfb04cd8ffeb05631a585cc03b96976fbf3. --- .../src/internal-plugins/internal-data-bridge/gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js index 22d1da5e40188..7f134cebffd7c 100644 --- a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js +++ b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js @@ -61,7 +61,7 @@ exports.sourceNodes = ({ createContentDigest, actions, store }) => { }, }) - flattenedPlugins.forEach(({ pluginOptions, ...plugin }) => { + flattenedPlugins.forEach(plugin => { plugin.pluginFilepath = plugin.resolve createNode({ ...plugin, From 6d3acc53ae5bfd32b8acdc15e11b90076fd7ff00 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 5 Apr 2019 20:10:41 +0200 Subject: [PATCH 10/20] sanitize nodes --- .../__snapshots__/node-tracking-test.js.snap | 22 ++++++ .../src/db/__tests__/node-tracking-test.js | 72 ++++++++++++++++++- packages/gatsby/src/db/node-tracking.js | 62 +++++++++++----- packages/gatsby/src/redux/actions.js | 2 +- 4 files changed, 138 insertions(+), 20 deletions(-) create mode 100644 packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap diff --git a/packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap b/packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap new file mode 100644 index 0000000000000..e397a70cb812c --- /dev/null +++ b/packages/gatsby/src/db/__tests__/__snapshots__/node-tracking-test.js.snap @@ -0,0 +1,22 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`track root nodes Node sanitization Remove not supported fields / values 1`] = ` +Object { + "children": Array [], + "id": "id1", + "inlineArray": Array [ + 1, + 2, + 3, + ], + "inlineObject": Object { + "field": "fieldOfFirstNode", + }, + "internal": Object { + "contentDigest": "digest1", + "owner": "test", + "type": "Test", + }, + "parent": null, +} +`; diff --git a/packages/gatsby/src/db/__tests__/node-tracking-test.js b/packages/gatsby/src/db/__tests__/node-tracking-test.js index c627eabc499fc..8143f5f1ca10c 100644 --- a/packages/gatsby/src/db/__tests__/node-tracking-test.js +++ b/packages/gatsby/src/db/__tests__/node-tracking-test.js @@ -3,7 +3,11 @@ const { boundActionCreators: { createNode }, } = require(`../../redux/actions`) const { getNode } = require(`../../db/nodes`) -const { findRootNodeAncestor, trackDbNodes } = require(`../node-tracking`) +const { + findRootNodeAncestor, + trackDbNodes, + trackInlineObjectsInRootNode, +} = require(`../node-tracking`) const { run: runQuery } = require(`../nodes-query`) require(`./fixtures/ensure-loki`)() @@ -136,4 +140,70 @@ describe(`track root nodes`, () => { expect(findRootNodeAncestor(result[0].inlineObject)).toEqual(result[0]) }) }) + + describe(`Node sanitization`, () => { + let testNode + beforeEach(() => { + testNode = { + id: `id1`, + parent: null, + children: [], + unsupported: () => {}, + inlineObject: { + field: `fieldOfFirstNode`, + re: /re/, + }, + inlineArray: [1, 2, 3, Symbol(`test`)], + internal: { + type: `Test`, + contentDigest: `digest1`, + owner: `test`, + }, + } + }) + + it(`Remove not supported fields / values`, () => { + const result = trackInlineObjectsInRootNode(testNode, true) + expect(result).toMatchSnapshot() + expect(result.unsupported).not.toBeDefined() + expect(result.inlineObject.re).not.toBeDefined() + expect(result.inlineArray[3]).not.toBeDefined() + }) + + it(`Doesn't mutate original`, () => { + trackInlineObjectsInRootNode(testNode, true) + expect(testNode.unsupported).toBeDefined() + expect(testNode.inlineObject.re).toBeDefined() + expect(testNode.inlineArray[3]).toBeDefined() + }) + + it(`Create copy of node if it has to remove anything`, () => { + const result = trackInlineObjectsInRootNode(testNode, true) + expect(result).not.toBe(testNode) + }) + + it(`Doesn't create clones if it doesn't have to`, () => { + const testNodeWithoutUnserializableData = { + id: `id1`, + parent: null, + children: [], + inlineObject: { + field: `fieldOfFirstNode`, + }, + inlineArray: [1, 2, 3], + internal: { + type: `Test`, + contentDigest: `digest1`, + owner: `test`, + }, + } + + const result = trackInlineObjectsInRootNode( + testNodeWithoutUnserializableData, + true + ) + // should be same instance + expect(result).toBe(testNodeWithoutUnserializableData) + }) + }) }) diff --git a/packages/gatsby/src/db/node-tracking.js b/packages/gatsby/src/db/node-tracking.js index 569f972d696ae..cabfa277fc57e 100644 --- a/packages/gatsby/src/db/node-tracking.js +++ b/packages/gatsby/src/db/node-tracking.js @@ -15,30 +15,65 @@ const getRootNodeId = node => rootNodeMap.get(node) * @see trackInlineObjectsInRootNode * @param {(Object|Array)} data Inline object or array * @param {string} nodeId Id of node that contains data passed in first parameter + * @param {boolean} sanitize Wether to strip objects of unuspported and not serializable fields + * @param {string} [ignore] Fieldname that doesn't need to be tracked and sanitized + * */ -const addRootNodeToInlineObject = (data, nodeId, sanitize, parent, key) => { - if (_.isPlainObject(data) || _.isArray(data)) { +const addRootNodeToInlineObject = (data, nodeId, sanitize, ignore = null) => { + const isPlainObject = _.isPlainObject(data) + + if (isPlainObject || _.isArray(data)) { + let returnData = data + if (sanitize) { + returnData = isPlainObject ? {} : [] + } + let anyFieldChanged = false _.each(data, (o, key) => { - addRootNodeToInlineObject(o, nodeId, sanitize, key, data) + if (ignore == key) { + returnData[key] = o + return + } + returnData[key] = addRootNodeToInlineObject( + o, + nodeId, + sanitize, + null, + key + ) + + if (returnData[key] !== o) { + anyFieldChanged = true + } }) + + if (anyFieldChanged) { + if (isPlainObject) { + data = _.pickBy(returnData, p => p !== undefined) + } else { + data = returnData.filter(p => p !== undefined) + } + } + rootNodeMap.set(data, nodeId) // arrays and plain objects are supported - no need to to sanitize - return + return data } - if (sanitize) { + if (sanitize && data !== null) { const type = typeof data - // supported types const isSupported = type === `number` || type === `string` || type === `boolean` || data instanceof Date + if (!isSupported) { - delete parent[key] + return undefined } } + // either supported or not sanitizing + return data } /** @@ -46,17 +81,8 @@ const addRootNodeToInlineObject = (data, nodeId, sanitize, parent, key) => { * and that Node object. * @param {Node} node Root Node */ -const trackInlineObjectsInRootNode = (node, sanitize = false) => { - _.each(node, (v, k) => { - // Ignore the node internal object. - if (k === `internal`) { - return - } - addRootNodeToInlineObject(v, node.id, sanitize, node, k) - }) - - return node -} +const trackInlineObjectsInRootNode = (node, sanitize = false) => + addRootNodeToInlineObject(node, node.id, sanitize, `internal`) exports.trackInlineObjectsInRootNode = trackInlineObjectsInRootNode /** diff --git a/packages/gatsby/src/redux/actions.js b/packages/gatsby/src/redux/actions.js index 45f1486d00a37..cd235cb7a0986 100644 --- a/packages/gatsby/src/redux/actions.js +++ b/packages/gatsby/src/redux/actions.js @@ -591,7 +591,7 @@ actions.createNode = ( ) } - trackInlineObjectsInRootNode(node, true) + node = trackInlineObjectsInRootNode(node, true) const oldNode = getNode(node.id) From 35c84e9480bd1f6727c062c807176603a35c6a35 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 5 Apr 2019 21:14:04 +0200 Subject: [PATCH 11/20] split redux persistance into separate file and adjust tests --- packages/gatsby/src/redux/__tests__/index.js | 13 ++--- packages/gatsby/src/redux/index.js | 51 ++------------------ packages/gatsby/src/redux/persist.js | 47 ++++++++++++++++++ 3 files changed, 56 insertions(+), 55 deletions(-) create mode 100644 packages/gatsby/src/redux/persist.js diff --git a/packages/gatsby/src/redux/__tests__/index.js b/packages/gatsby/src/redux/__tests__/index.js index 9581cf4780001..5c461a7dbdd6d 100644 --- a/packages/gatsby/src/redux/__tests__/index.js +++ b/packages/gatsby/src/redux/__tests__/index.js @@ -1,11 +1,11 @@ const path = require(`path`) -const fs = require(`fs-extra`) const { saveState, store } = require(`../index`) +const { writeToCache } = require(`../persist`) const { actions: { createPage }, } = require(`../actions`) -jest.mock(`fs-extra`) +jest.mock(`../persist`) describe(`redux db`, () => { beforeEach(() => { @@ -24,15 +24,12 @@ describe(`redux db`, () => { ) ) - fs.writeFile.mockClear() + writeToCache.mockClear() }) it(`should write cache to disk`, async () => { await saveState() - expect(fs.writeFile).toBeCalledWith( - expect.stringContaining(`.cache/redux-state.json`), - expect.stringContaining(`my-sweet-new-page.js`) - ) + expect(writeToCache).toBeCalled() }) it(`does not write to the cache when DANGEROUSLY_DISABLE_OOM is set`, async () => { @@ -40,7 +37,7 @@ describe(`redux db`, () => { await saveState() - expect(fs.writeFile).not.toBeCalled() + expect(writeToCache).not.toBeCalled() delete process.env.DANGEROUSLY_DISABLE_OOM }) diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index c92722f55dec6..ef0dade16de01 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -1,62 +1,19 @@ const Redux = require(`redux`) const _ = require(`lodash`) -const fs = require(`fs-extra`) + const mitt = require(`mitt`) -const v8 = require(`v8`) -const stringify = require(`json-stringify-safe`) // Create event emitter for actions const emitter = mitt() // Reducers const reducers = require(`./reducers`) - -const objectToMap = obj => { - const map = new Map() - Object.keys(obj).forEach(key => { - map.set(key, obj[key]) - }) - return map -} - -const mapToObject = map => { - const obj = {} - for (let [key, value] of map) { - obj[key] = value - } - return obj -} - -const jsonStringify = contents => { - contents.staticQueryComponents = mapToObject(contents.staticQueryComponents) - contents.components = mapToObject(contents.components) - contents.nodes = mapToObject(contents.nodes) - return stringify(contents, null, 2) -} - -const jsonParse = buffer => { - const parsed = JSON.parse(buffer.toString(`utf8`)) - parsed.staticQueryComponents = objectToMap(parsed.staticQueryComponents) - parsed.components = objectToMap(parsed.components) - parsed.nodes = objectToMap(parsed.nodes) - return parsed -} - -const useV8 = Boolean(v8.serialize) -const [serialize, deserialize, file] = useV8 - ? [v8.serialize, v8.deserialize, `${process.cwd()}/.cache/redux.state`] - : [jsonStringify, jsonParse, `${process.cwd()}/.cache/redux-state.json`] - -const readFileSync = file => deserialize(fs.readFileSync(file)) - -const writeFileSync = (file, contents) => { - fs.writeFileSync(file, serialize(contents)) -} +const { writeToCache, readFromCache } = require(`./persist`) // Read old node data from cache. let initialState = {} try { - initialState = readFileSync(file) + initialState = readFromCache() if (initialState.nodes) { // re-create nodesByType initialState.nodesByType = new Map() @@ -99,7 +56,7 @@ function saveState() { `staticQueryComponents`, ]) - writeFileSync(file, pickedState) + return writeToCache(pickedState) } exports.saveState = saveState diff --git a/packages/gatsby/src/redux/persist.js b/packages/gatsby/src/redux/persist.js new file mode 100644 index 0000000000000..fcbe48836f42c --- /dev/null +++ b/packages/gatsby/src/redux/persist.js @@ -0,0 +1,47 @@ +const v8 = require(`v8`) +const fs = require(`fs-extra`) +const stringify = require(`json-stringify-safe`) + +const objectToMap = obj => { + const map = new Map() + Object.keys(obj).forEach(key => { + map.set(key, obj[key]) + }) + return map +} + +const mapToObject = map => { + const obj = {} + for (let [key, value] of map) { + obj[key] = value + } + return obj +} + +const jsonStringify = contents => { + contents.staticQueryComponents = mapToObject(contents.staticQueryComponents) + contents.components = mapToObject(contents.components) + contents.nodes = mapToObject(contents.nodes) + return stringify(contents, null, 2) +} + +const jsonParse = buffer => { + const parsed = JSON.parse(buffer.toString(`utf8`)) + parsed.staticQueryComponents = objectToMap(parsed.staticQueryComponents) + parsed.components = objectToMap(parsed.components) + parsed.nodes = objectToMap(parsed.nodes) + return parsed +} + +const useV8 = Boolean(v8.serialize) +const [serialize, deserialize, file] = useV8 + ? [v8.serialize, v8.deserialize, `${process.cwd()}/.cache/redux.state`] + : [jsonStringify, jsonParse, `${process.cwd()}/.cache/redux-state.json`] + +const readFromCache = () => deserialize(fs.readFileSync(file)) + +const writeToCache = contents => { + fs.writeFileSync(file, serialize(contents)) +} + +module.exports = { readFromCache, writeToCache } From fba9b53ea2dc4764cb0c79cbeab9b85a5da2b2b2 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 5 Apr 2019 21:47:25 +0200 Subject: [PATCH 12/20] some more persisting tests confidence --- .../__tests__/__snapshots__/index.js.snap | 27 +++++++++++++ packages/gatsby/src/redux/__tests__/index.js | 39 +++++++++++++++++-- packages/gatsby/src/redux/index.js | 34 +++++++++------- 3 files changed, 83 insertions(+), 17 deletions(-) create mode 100644 packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap new file mode 100644 index 0000000000000..869fcec154b9c --- /dev/null +++ b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap @@ -0,0 +1,27 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`redux db should write cache to disk 1`] = ` +Object { + "componentDataDependencies": Object { + "connections": Object {}, + "nodes": Object {}, + }, + "components": Map { + "/src/templates/my-sweet-new-page.js" => Object { + "componentPath": "/src/templates/my-sweet-new-page.js", + "isInBootstrap": true, + "pages": Array [ + "/my-sweet-new-page/", + ], + "query": "", + }, + }, + "jsonDataPaths": Object {}, + "nodes": Map {}, + "nodesByType": Map {}, + "staticQueryComponents": Map {}, + "status": Object { + "plugins": Object {}, + }, +} +`; diff --git a/packages/gatsby/src/redux/__tests__/index.js b/packages/gatsby/src/redux/__tests__/index.js index 5c461a7dbdd6d..dec05b9b7055a 100644 --- a/packages/gatsby/src/redux/__tests__/index.js +++ b/packages/gatsby/src/redux/__tests__/index.js @@ -1,13 +1,26 @@ const path = require(`path`) -const { saveState, store } = require(`../index`) -const { writeToCache } = require(`../persist`) +const _ = require(`lodash`) +jest.mock(`fs-extra`) +const writeToCache = jest.spyOn(require(`../persist`), `writeToCache`) +const { saveState, store, readState } = require(`../index`) + const { actions: { createPage }, } = require(`../actions`) -jest.mock(`../persist`) +const mockWrittenContent = new Map() +jest.mock(`fs-extra`, () => { + return { + writeFileSync: jest.fn((file, content) => + mockWrittenContent.set(file, content) + ), + readFileSync: jest.fn(file => mockWrittenContent.get(file)), + } +}) describe(`redux db`, () => { + const initialComponentsState = _.cloneDeep(store.getState().components) + beforeEach(() => { store.dispatch( createPage( @@ -25,11 +38,31 @@ describe(`redux db`, () => { ) writeToCache.mockClear() + mockWrittenContent.clear() + }) + + it(`expect components state to be empty initially`, () => { + expect(initialComponentsState).toEqual(new Map()) }) + it(`should write cache to disk`, async () => { await saveState() expect(writeToCache).toBeCalled() + + // reset state in memory + store.dispatch({ + type: `DELETE_CACHE`, + }) + // make sure store in memory is empty + expect(store.getState().components).toEqual(initialComponentsState) + + // read data that was previously cached + const data = readState() + + // make sure data was read and is not the same as our clean redux state + expect(data.components).not.toEqual(initialComponentsState) + expect(data).toMatchSnapshot() }) it(`does not write to the cache when DANGEROUSLY_DISABLE_OOM is set`, async () => { diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index ef0dade16de01..f5e90a24115e8 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -12,22 +12,28 @@ const { writeToCache, readFromCache } = require(`./persist`) // Read old node data from cache. let initialState = {} -try { - initialState = readFromCache() - if (initialState.nodes) { - // re-create nodesByType - initialState.nodesByType = new Map() - initialState.nodes.forEach(node => { - const { type } = node.internal - if (!initialState.nodesByType.has(type)) { - initialState.nodesByType.set(type, new Map()) - } - initialState.nodesByType.get(type).set(node.id, node) - }) +const readState = () => { + try { + initialState = readFromCache() + if (initialState.nodes) { + // re-create nodesByType + initialState.nodesByType = new Map() + initialState.nodes.forEach(node => { + const { type } = node.internal + if (!initialState.nodesByType.has(type)) { + initialState.nodesByType.set(type, new Map()) + } + initialState.nodesByType.get(type).set(node.id, node) + }) + } + } catch (e) { + // ignore errors. + initialState = {} } -} catch (e) { - // ignore errors. + return initialState } +readState() +exports.readState = readState const store = Redux.createStore( Redux.combineReducers({ ...reducers }), From 45f654c293350d41b30957b7be44e7047e4f9784 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 5 Apr 2019 22:13:08 +0200 Subject: [PATCH 13/20] don't track itself --- packages/gatsby/src/db/node-tracking.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/gatsby/src/db/node-tracking.js b/packages/gatsby/src/db/node-tracking.js index cabfa277fc57e..d93db5d603669 100644 --- a/packages/gatsby/src/db/node-tracking.js +++ b/packages/gatsby/src/db/node-tracking.js @@ -19,7 +19,7 @@ const getRootNodeId = node => rootNodeMap.get(node) * @param {string} [ignore] Fieldname that doesn't need to be tracked and sanitized * */ -const addRootNodeToInlineObject = (data, nodeId, sanitize, ignore = null) => { +const addRootNodeToInlineObject = (data, nodeId, sanitize, isNode = false) => { const isPlainObject = _.isPlainObject(data) if (isPlainObject || _.isArray(data)) { @@ -29,17 +29,11 @@ const addRootNodeToInlineObject = (data, nodeId, sanitize, ignore = null) => { } let anyFieldChanged = false _.each(data, (o, key) => { - if (ignore == key) { + if (isNode && key === `internal`) { returnData[key] = o return } - returnData[key] = addRootNodeToInlineObject( - o, - nodeId, - sanitize, - null, - key - ) + returnData[key] = addRootNodeToInlineObject(o, nodeId, sanitize) if (returnData[key] !== o) { anyFieldChanged = true @@ -54,7 +48,10 @@ const addRootNodeToInlineObject = (data, nodeId, sanitize, ignore = null) => { } } - rootNodeMap.set(data, nodeId) + // don't need to track node itself + if (!isNode) { + rootNodeMap.set(data, nodeId) + } // arrays and plain objects are supported - no need to to sanitize return data @@ -82,7 +79,7 @@ const addRootNodeToInlineObject = (data, nodeId, sanitize, ignore = null) => { * @param {Node} node Root Node */ const trackInlineObjectsInRootNode = (node, sanitize = false) => - addRootNodeToInlineObject(node, node.id, sanitize, `internal`) + addRootNodeToInlineObject(node, node.id, sanitize, true) exports.trackInlineObjectsInRootNode = trackInlineObjectsInRootNode /** From 1c20610ee44778e6058fe89f103698e7db59e6b0 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 5 Apr 2019 22:53:04 +0200 Subject: [PATCH 14/20] will this work on windows? --- .../gatsby/src/redux/__tests__/__snapshots__/index.js.snap | 4 ++-- packages/gatsby/src/redux/__tests__/index.js | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap index 869fcec154b9c..ba105b6c2f4ac 100644 --- a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap +++ b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap @@ -7,8 +7,8 @@ Object { "nodes": Object {}, }, "components": Map { - "/src/templates/my-sweet-new-page.js" => Object { - "componentPath": "/src/templates/my-sweet-new-page.js", + "/Users/username/dev/site/src/templates/my-sweet-new-page.js" => Object { + "componentPath": "/Users/username/dev/site/src/templates/my-sweet-new-page.js", "isInBootstrap": true, "pages": Array [ "/my-sweet-new-page/", diff --git a/packages/gatsby/src/redux/__tests__/index.js b/packages/gatsby/src/redux/__tests__/index.js index dec05b9b7055a..08a0dcc5cef6d 100644 --- a/packages/gatsby/src/redux/__tests__/index.js +++ b/packages/gatsby/src/redux/__tests__/index.js @@ -1,6 +1,6 @@ -const path = require(`path`) +// const path = require(`path`) const _ = require(`lodash`) -jest.mock(`fs-extra`) + const writeToCache = jest.spyOn(require(`../persist`), `writeToCache`) const { saveState, store, readState } = require(`../index`) @@ -26,7 +26,8 @@ describe(`redux db`, () => { createPage( { path: `/my-sweet-new-page/`, - component: path.resolve(`./src/templates/my-sweet-new-page.js`), + // seems like jest serializer doesn't play nice with Maps on Windows + component: `/Users/username/dev/site/src/templates/my-sweet-new-page.js`, // The context is passed as props to the component as well // as into the component's GraphQL query. context: { From 0d9bf437307dba741dad3040d8c43c3fb9fea73e Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sat, 6 Apr 2019 00:50:47 +0200 Subject: [PATCH 15/20] oh loki --- .../gatsby/src/redux/__tests__/__snapshots__/index.js.snap | 2 -- packages/gatsby/src/redux/__tests__/index.js | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap index ba105b6c2f4ac..14658aae15f86 100644 --- a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap +++ b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap @@ -17,8 +17,6 @@ Object { }, }, "jsonDataPaths": Object {}, - "nodes": Map {}, - "nodesByType": Map {}, "staticQueryComponents": Map {}, "status": Object { "plugins": Object {}, diff --git a/packages/gatsby/src/redux/__tests__/index.js b/packages/gatsby/src/redux/__tests__/index.js index 08a0dcc5cef6d..407b4fc7f3162 100644 --- a/packages/gatsby/src/redux/__tests__/index.js +++ b/packages/gatsby/src/redux/__tests__/index.js @@ -63,7 +63,9 @@ describe(`redux db`, () => { // make sure data was read and is not the same as our clean redux state expect(data.components).not.toEqual(initialComponentsState) - expect(data).toMatchSnapshot() + + // yuck - loki and redux will have different shape of redux state (nodes and nodesByType) + expect(_.omit(data, [`nodes`, `nodesByType`])).toMatchSnapshot() }) it(`does not write to the cache when DANGEROUSLY_DISABLE_OOM is set`, async () => { From bb739bd3094f46c50359ead9cd31f749b7bcccd3 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sun, 7 Apr 2019 02:32:17 +0200 Subject: [PATCH 16/20] handle case when nodes state is null (loki) and v8.serialize is not available --- packages/gatsby/src/redux/__tests__/index.js | 1 - packages/gatsby/src/redux/persist.js | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/redux/__tests__/index.js b/packages/gatsby/src/redux/__tests__/index.js index 407b4fc7f3162..7c14ff137b613 100644 --- a/packages/gatsby/src/redux/__tests__/index.js +++ b/packages/gatsby/src/redux/__tests__/index.js @@ -1,4 +1,3 @@ -// const path = require(`path`) const _ = require(`lodash`) const writeToCache = jest.spyOn(require(`../persist`), `writeToCache`) diff --git a/packages/gatsby/src/redux/persist.js b/packages/gatsby/src/redux/persist.js index fcbe48836f42c..2436056955c05 100644 --- a/packages/gatsby/src/redux/persist.js +++ b/packages/gatsby/src/redux/persist.js @@ -21,7 +21,7 @@ const mapToObject = map => { const jsonStringify = contents => { contents.staticQueryComponents = mapToObject(contents.staticQueryComponents) contents.components = mapToObject(contents.components) - contents.nodes = mapToObject(contents.nodes) + contents.nodes = contents.nodes ? mapToObject(contents.nodes) : {} return stringify(contents, null, 2) } @@ -29,7 +29,7 @@ const jsonParse = buffer => { const parsed = JSON.parse(buffer.toString(`utf8`)) parsed.staticQueryComponents = objectToMap(parsed.staticQueryComponents) parsed.components = objectToMap(parsed.components) - parsed.nodes = objectToMap(parsed.nodes) + parsed.nodes = objectToMap(parsed.nodes || {}) return parsed } From 1905f2ee5207bf8662fe908cdf8639d89ff9bcbf Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 11 Apr 2019 13:33:17 +0200 Subject: [PATCH 17/20] refactor into omitUndefined --- packages/gatsby/src/db/node-tracking.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/gatsby/src/db/node-tracking.js b/packages/gatsby/src/db/node-tracking.js index d93db5d603669..c981515fe0f03 100644 --- a/packages/gatsby/src/db/node-tracking.js +++ b/packages/gatsby/src/db/node-tracking.js @@ -9,6 +9,19 @@ const rootNodeMap = new WeakMap() const getRootNodeId = node => rootNodeMap.get(node) +/** + * @param {Object} data + * @returns {Object} data without undefined values + */ +const omitUndefined = data => { + const isPlainObject = _.isPlainObject(data) + if (isPlainObject) { + return _.pickBy(data, p => p !== undefined) + } + + return data.filter(p => p !== undefined) +} + /** * Add link between passed data and Node. This function shouldn't be used * directly. Use higher level `trackInlineObjectsInRootNode` @@ -41,11 +54,7 @@ const addRootNodeToInlineObject = (data, nodeId, sanitize, isNode = false) => { }) if (anyFieldChanged) { - if (isPlainObject) { - data = _.pickBy(returnData, p => p !== undefined) - } else { - data = returnData.filter(p => p !== undefined) - } + data = omitUndefined(returnData) } // don't need to track node itself From 843497eb3d19dca879935c6cdbe48c6eb2b38ad3 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 11 Apr 2019 13:35:55 +0200 Subject: [PATCH 18/20] refactor into isTypeSupported --- packages/gatsby/src/db/node-tracking.js | 28 +++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/gatsby/src/db/node-tracking.js b/packages/gatsby/src/db/node-tracking.js index c981515fe0f03..62a913c5d5d70 100644 --- a/packages/gatsby/src/db/node-tracking.js +++ b/packages/gatsby/src/db/node-tracking.js @@ -22,6 +22,21 @@ const omitUndefined = data => { return data.filter(p => p !== undefined) } +/** + * @param {*} data + * @return {boolean} + */ +const isTypeSupported = data => { + const type = typeof data + const isSupported = + type === `number` || + type === `string` || + type === `boolean` || + data instanceof Date + + return isSupported +} + /** * Add link between passed data and Node. This function shouldn't be used * directly. Use higher level `trackInlineObjectsInRootNode` @@ -66,17 +81,8 @@ const addRootNodeToInlineObject = (data, nodeId, sanitize, isNode = false) => { return data } - if (sanitize && data !== null) { - const type = typeof data - const isSupported = - type === `number` || - type === `string` || - type === `boolean` || - data instanceof Date - - if (!isSupported) { - return undefined - } + if (sanitize && !isTypeSupported(data)) { + return undefined } // either supported or not sanitizing return data From 95dc28fa7fa065b17535f78cd056abff04fec949 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 11 Apr 2019 13:43:28 +0200 Subject: [PATCH 19/20] don't strip `null` from nodes --- packages/gatsby/src/db/node-tracking.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/gatsby/src/db/node-tracking.js b/packages/gatsby/src/db/node-tracking.js index 62a913c5d5d70..79b583f608bce 100644 --- a/packages/gatsby/src/db/node-tracking.js +++ b/packages/gatsby/src/db/node-tracking.js @@ -27,6 +27,10 @@ const omitUndefined = data => { * @return {boolean} */ const isTypeSupported = data => { + if (data === null) { + return true + } + const type = typeof data const isSupported = type === `number` || From 0641283662bbb60ba9c77041f3d9621bc4629118 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 11 Apr 2019 17:21:48 +0200 Subject: [PATCH 20/20] readCache can be pure, thanks @wardpeet --- packages/gatsby/src/redux/index.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index f5e90a24115e8..9f598a0171ef3 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -11,33 +11,32 @@ const reducers = require(`./reducers`) const { writeToCache, readFromCache } = require(`./persist`) // Read old node data from cache. -let initialState = {} const readState = () => { try { - initialState = readFromCache() - if (initialState.nodes) { + const state = readFromCache() + if (state.nodes) { // re-create nodesByType - initialState.nodesByType = new Map() - initialState.nodes.forEach(node => { + state.nodesByType = new Map() + state.nodes.forEach(node => { const { type } = node.internal - if (!initialState.nodesByType.has(type)) { - initialState.nodesByType.set(type, new Map()) + if (!state.nodesByType.has(type)) { + state.nodesByType.set(type, new Map()) } - initialState.nodesByType.get(type).set(node.id, node) + state.nodesByType.get(type).set(node.id, node) }) } + return state } catch (e) { // ignore errors. - initialState = {} } - return initialState + return {} } -readState() + exports.readState = readState const store = Redux.createStore( Redux.combineReducers({ ...reducers }), - initialState, + readState(), Redux.applyMiddleware(function multi({ dispatch }) { return next => action => Array.isArray(action)