tests: isolate tests to tmp directory (gatsbyjs#30646)

When doing e2e-tests or integration tests, isolate the projects to tmp so it can't take any modules from the monorepo.

Author: wardpeet

.circleci/config.yml

@@ -150,7 +150,7 @@ commands:
         default: false
       trigger_pattern:
         type: string
-        default: "packages/*|.circleci/*"
+        default: "packages/*|.circleci/*|scripts/e2e-test.sh"
       test_path:
         type: string
       test_command:
@@ -168,6 +168,9 @@ commands:
             - run: ./scripts/assert-changed-files.sh "<< parameters.trigger_pattern >>|<< parameters.test_path >>/*"
 
       - <<: *attach_to_bootstrap
+      - run:
+          name: Install gatsby-dev
+          command: yarn global add gatsby-dev-cli
       - run:
           name: Run tests (using defaults)
           command: ./scripts/e2e-test.sh "<< parameters.test_path >>" "<< parameters.test_command >>"
@@ -179,7 +182,7 @@ jobs:
     executor: node
     steps:
       - checkout
-      - run: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|yarn.lock"
+      - run: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|scripts/e2e-test.sh|yarn.lock"
       - <<: *restore_cache
       - <<: *install_node_modules
       - <<: *check_lockfile
@@ -264,7 +267,7 @@ jobs:
     steps:
       - e2e-test:
           test_path: integration-tests/gatsby-cli
-          trigger_pattern: packages/gatsby-cli/*|packages/gatsby/*|integration-tests/gatsby-cli/*
+          trigger_pattern: packages/gatsby-cli/*|packages/gatsby/*|integration-tests/gatsby-cli/*|scripts/e2e-test.sh
 
   integration_tests_structured_logging:
     executor: node
@@ -377,8 +380,8 @@ jobs:
       CYPRESS_RECORD_KEY: 3fb49000-4143-4bd8-9ab4-219389060910
     steps:
       - e2e-test:
-          test_path: e2e-tests/themes/development-runtime
-          test_command: yarn test
+          test_path: e2e-tests/themes
+          test_command: cd development-runtime; yarn test
 
   themes_e2e_tests_production_runtime:
     <<: *e2e-executor
@@ -387,8 +390,8 @@ jobs:
       CYPRESS_RECORD_KEY: e4e7b3b8-e1e7-4a74-a0c9-9ac76585236b
     steps:
       - e2e-test:
-          test_path: e2e-tests/themes/production-runtime
-          test_command: yarn test
+          test_path: e2e-tests/themes
+          test_command: cd production-runtime; yarn test
 
   mdx_e2e_tests:
     <<: *e2e-executor
@@ -490,7 +493,7 @@ jobs:
     steps:
       - checkout
       - run:
-          command: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|yarn.lock"
+          command: ./scripts/assert-changed-files.sh "packages/*|(e2e|integration)-tests/*|.circleci/*|scripts/e2e-test.sh|yarn.lock"
           shell: bash.exe
       # Restoring cache takes as long as installing node modules, so skipping
       # - restore_cache:

e2e-tests/contentful/package.json

@@ -13,6 +13,7 @@
     "gatsby-plugin-sharp": "^3.1.2",
     "gatsby-source-contentful": "^5.1.1",
     "gatsby-transformer-remark": "^3.2.0",
+    "gatsby-transformer-sharp": "^3.3.0",
     "gatsby-transformer-sqip": "3.1.1",
     "modern-normalize": "^1.0.0",
     "prop-types": "^15.7.2",

e2e-tests/gatsby-static-image/package.json

@@ -8,6 +8,7 @@
     "gatsby": "^2.0.118",
     "gatsby-plugin-image": "^0.0.2",
     "gatsby-plugin-sharp": "^2.0.20",
+    "gatsby-source-filesystem": "^3.3.0",
     "react": "^16.9.0",
     "react-dom": "^16.9.0"
   },
@@ -36,4 +37,4 @@
     "type": "git",
     "url": "https://github.com/gatsbyjs/gatsby-starter-default"
   }
-}
+}

e2e-tests/production-runtime/package.json

@@ -7,6 +7,7 @@
     "babel-plugin-search-and-replace": "^1.1.0",
     "cypress": "^6.5.0",
     "gatsby": "^3.0.0-next.6",
+    "gatsby-cypress": "^1.3.0",
     "gatsby-plugin-image": "^1.0.0-next.5",
     "gatsby-plugin-less": "^5.1.0-next.2",
     "gatsby-plugin-manifest": "^3.0.0-next.0",
@@ -15,6 +16,7 @@
     "gatsby-plugin-sass": "^4.1.0-next.2",
     "gatsby-plugin-sharp": "^3.0.0-next.5",
     "gatsby-plugin-stylus": "^3.1.0-next.2",
+    "gatsby-source-filesystem": "^3.3.0",
     "gatsby-seo": "^0.1.0",
     "glob": "^7.1.3",
     "react": "^16.9.0",

integration-tests/artifacts/__tests__/index.js

@@ -9,7 +9,7 @@ jest.setTimeout(100000)
 
 const publicDir = path.join(process.cwd(), `public`)
 
-const gatsbyBin = path.join(`node_modules`, `.bin`, `gatsby`)
+const gatsbyBin = path.join(`node_modules`, `gatsby`, `cli.js`)
 
 const manifest = {}
 const filesToRevert = {}
@@ -20,8 +20,8 @@ function runGatsbyWithRunTestSetup(runNumber = 1) {
   return function beforeAllImpl() {
     exitCode = `reset`
     return new Promise(resolve => {
-      const gatsbyProcess = spawn(gatsbyBin, [`build`], {
-        stdio: [`inherit`, `inherit`, `inherit`, `inherit`],
+      const gatsbyProcess = spawn(process.execPath, [gatsbyBin, `build`], {
+        stdio: [`inherit`, `inherit`, `inherit`],
         env: {
           ...process.env,
           NODE_ENV: `production`,
@@ -358,8 +358,8 @@ function assertExitCode(runNumber) {
 beforeAll(done => {
   fs.removeSync(path.join(__dirname, `__debug__`))
 
-  const gatsbyCleanProcess = spawn(gatsbyBin, [`clean`], {
-    stdio: [`inherit`, `inherit`, `inherit`, `inherit`],
+  const gatsbyCleanProcess = spawn(process.execPath, [gatsbyBin, `clean`], {
+    stdio: [`inherit`, `inherit`, `inherit`],
     env: {
       ...process.env,
       NODE_ENV: `production`,

integration-tests/artifacts/package.json

@@ -10,7 +10,7 @@
     "develop": "gatsby develop",
     "serve": "gatsby serve",
     "clean": "gatsby clean",
-    "test": "jest --config=jest.config.js --runInBand"
+    "test": "jest --runInBand"
   },
   "dependencies": {
     "gatsby": "^3.0.0-next.0",
@@ -19,8 +19,7 @@
   },
   "devDependencies": {
     "fs-extra": "^9.0.0",
-    "jest": "^24.0.0",
-    "jest-cli": "^24.0.0"
+    "jest": "^24.0.0"
   },
   "repository": {
     "type": "git",

integration-tests/cache-resilience/__tests__/cache.js

@@ -1,6 +1,7 @@
 const fs = require(`fs-extra`)
 const { spawnSync } = require(`child_process`)
 const path = require(`path`)
+const os = require(`os`)
 const _ = require(`lodash`)
 const {
   ON_PRE_BOOTSTRAP_FILE_PATH,
@@ -30,13 +31,7 @@ const getDiskCacheSnapshotSubStateByPlugins = (state, pluginNamesArray) =>
 
 jest.setTimeout(100000)
 
-const gatsbyBin = path.join(
-  `node_modules`,
-  `gatsby`,
-  `dist`,
-  `bin`,
-  `gatsby.js`
-)
+const gatsbyBin = path.join(`node_modules`, `gatsby`, `cli.js`)
 
 const { compareState } = require(`../utils/nodes-diff`)
 
@@ -380,6 +375,10 @@ describe(`Some plugins changed between gatsby runs`, () => {
 
   describe(`Query results`, () => {
     const getQueryResultTestArgs = scenarioName => {
+      if (os.platform() === "win32") {
+        scenarioName = scenarioName.replace("/", "\\")
+      }
+
       const result = {
         dataFirstRun: states.queryResults.firstRun[scenarioName].data,
         dataSecondRun: states.queryResults.secondRun[scenarioName].data,

integration-tests/cache-resilience/package.json

@@ -7,7 +7,7 @@
     "develop": "gatsby develop",
     "prebuild": "node utils/prebuild.js",
     "build": "gatsby build",
-    "test": "jest --verbose",
+    "test": "jest",
     "serve": "gatsby serve",
     "use-setup": "node utils/select-configuration",
     "inspect-state": "node utils/inspect-state"
@@ -22,9 +22,9 @@
     "fs-extra": "^9.0.1",
     "glob": "^7.1.6",
     "jest": "^24.0.0",
-    "jest-cli": "^24.0.0",
+    "jest-serializer-path": "^0.1.15",
     "lodash": "^4.17.20",
     "slash": "^3.0.0",
     "snapshot-diff": "^0.6.1"
   }
-}
+}

integration-tests/functions/jest-transformer.js

@@ -0,0 +1,4 @@
+const babelPreset = require(`babel-preset-gatsby-package`)()
+module.exports = require(`babel-jest`).createTransformer({
+  ...babelPreset,
+})

integration-tests/functions/jest.config.js

@@ -8,6 +8,6 @@ module.exports = {
   ],
   watchPathIgnorePatterns: ["src/api", ".cache"],
   transform: {
-    "^.+\\.[jt]sx?$": `<rootDir>../../jest-transformer.js`,
+    "^.+\\.[jt]sx?$": `./jest-transformer.js`,
   },
 }

integration-tests/functions/package.json

@@ -4,20 +4,23 @@
   "private": true,
   "description": "functions",
   "author": "Kyle Mathews",
-  "keywords": ["gatsby"],
+  "keywords": [
+    "gatsby"
+  ],
   "scripts": {
     "build": "gatsby build",
     "develop": "gatsby develop",
     "serve": "gatsby serve",
-    "start-servers-and-test": "start-server-and-test servers '8000|9000' test:jest",
+    "start-servers-and-test": "start-server-and-test servers \"8000|9000\" test:jest",
     "servers": "run-p develop serve",
     "test:jest": "jest",
     "test": "npm-run-all -s build start-servers-and-test"
   },
   "devDependencies": {
+    "babel-jest": "^24.0.0",
+    "babel-preset-gatsby-package": "^1.3.0",
     "fs-extra": "^9.0.0",
     "jest": "^24.0.0",
-    "jest-cli": "^24.0.0",
     "npm-run-all": "4.1.5",
     "start-server-and-test": "^1.11.3"
   },
@@ -27,4 +30,4 @@
     "react": "^17.0.1",
     "react-dom": "^17.0.1"
   }
-}
+}

integration-tests/gatsby-cli/jest-transformer.js

@@ -0,0 +1,4 @@
+const babelPreset = require(`babel-preset-gatsby-package`)()
+module.exports = require(`babel-jest`).createTransformer({
+  ...babelPreset,
+})

integration-tests/gatsby-cli/jest.boot.js

@@ -1,18 +1,6 @@
-const fs = require(`fs`)
-const path = require(`path`)
-const os = require(`os`)
-const baseConfig = require(`../jest.config.js`)
-
-// install global gatsby-cli to tmp dir to simulate sandbox
-const GLOBAL_GATSBY_CLI_LOCATION = (process.env.GLOBAL_GATSBY_CLI_LOCATION = fs.mkdtempSync(
-  path.join(os.tmpdir(), `gatsby-cli-`)
-))
-
 module.exports = {
-  ...baseConfig,
-  globalSetup: "<rootDir>/integration-tests/gatsby-cli/jest.boot.js",
-  rootDir: `../../`,
-  globals: {
-    GLOBAL_GATSBY_CLI_LOCATION,
+  testPathIgnorePatterns: [`/node_modules/`, `__tests__/fixtures`, `.cache`],
+  transform: {
+    "^.+\\.[jt]sx?$": `./jest-transformer.js`,
   },
 }

integration-tests/gatsby-cli/jest.config.js

@@ -6,13 +6,20 @@
   },
   "license": "MIT",
   "scripts": {
-    "test": "jest --config=./jest.config.js gatsby-cli/"
+    "test": "jest"
   },
   "devDependencies": {
+    "babel-jest": "^24.0.0",
+    "babel-preset-gatsby-package": "^1.3.0",
     "del-cli": "^3.0.1",
     "execa": "^4.0.1",
     "fs-extra": "^9.0.0",
-    "jest": "^24.0.0"
+    "gatsby": "^3.3.0",
+    "gatsby-cli": "^3.3.0",
+    "gatsby-core-utils": "^2.3.0",
+    "jest": "^24.0.0",
+    "react": "^17.0.2",
+    "react-dom": "^17.0.2"
   },
   "repository": {
     "type": "git",

integration-tests/gatsby-cli/package.json

@@ -4,10 +4,11 @@ import strip from "strip-ansi"
 import { createLogsMatcher } from "./matcher"
 
 const gatsbyBinLocation = join(
-  GLOBAL_GATSBY_CLI_LOCATION,
+  __dirname,
+  "..",
   `node_modules`,
-  `.bin`,
-  `gatsby`
+  `gatsby-cli`,
+  `cli.js`
 )
 
 // Use as `GatsbyCLI.cwd('execution-folder').invoke('new', 'foo')`
@@ -17,7 +18,7 @@ export const GatsbyCLI = {
       invoke(args) {
         const NODE_ENV = args[0] === `develop` ? `development` : `production`
         try {
-          const results = sync(gatsbyBinLocation, [].concat(args), {
+          const results = sync("node", [gatsbyBinLocation].concat(args), {
             cwd: join(__dirname, `../`, `./${relativeCwd}`),
             env: { NODE_ENV, CI: 1, GATSBY_LOGGER: `ink` },
           })
@@ -36,7 +37,7 @@ export const GatsbyCLI = {
 
       invokeAsync: (args, onExit) => {
         const NODE_ENV = args[0] === `develop` ? `development` : `production`
-        const res = execa(gatsbyBinLocation, [].concat(args), {
+        const res = execa("node", [gatsbyBinLocation].concat(args), {
           cwd: join(__dirname, `../`, `./${relativeCwd}`),
           env: { NODE_ENV, CI: 1, GATSBY_LOGGER: `ink` },
         })