Skip to content

Commit 96e39c5

Browse files
mydeamydea
authored
test: Run E2E tests in isolated tmp directory (#16783)
This PR updates our E2E test runner to run the apps from an isolated tmp directory, instead of running them inside the monorepo. The reason to do this is that running them inside the monorepo leads to slightly different behavior, as dependencies can be looked up from parent node_modules folders. This leads to behavior that differs from actual standalone apps. Now, the whole app is moved into a folder in the system tmp directory. The package.json is adjusted to make it work there (e.g. rewriting volta `extends` file paths etc), then normally run from there. Some things had to be changed/fixed to make tests work here properly: * Ensure all dependencies are actually defined. E.g. we sometimes used `@sentry/core` in tests but did not have it as dependency. * Ensure every test app has a volta config to ensure consistent versions. * Update wrangler in cloudflare apps as v3 had some issues * align playwright version used to ensure browsers are always installed * removed some unnecessary usage of `@sentry/core` in tests * nuxt & solidstart tests do not need to copy iitm around anymore, this just works now. I also got to remove almost all the overrides of the nft package etc.
1 parent 8f2f6e7 commit 96e39c5

File tree

122 files changed

+791
-663
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

122 files changed

+791
-663
lines changed

‎.github/workflows/build.yml‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -920,19 +920,23 @@ jobs:
920920
env:
921921
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
922922

923+
- name: Copy to temp
924+
run: yarn ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp }}/test-application
925+
working-directory: dev-packages/e2e-tests
926+
923927
- name: Build E2E app
924-
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
928+
working-directory: ${{ runner.temp }}/test-application
925929
timeout-minutes: 7
926930
run: pnpm ${{ matrix.build-command || 'test:build' }}
927931

928932
- name: Install Playwright
929933
uses: ./.github/actions/install-playwright
930934
with:
931935
browsers: chromium
932-
cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
936+
cwd: ${{ runner.temp }}/test-application
933937

934938
- name: Run E2E test
935-
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
939+
working-directory: ${{ runner.temp }}/test-application
936940
timeout-minutes: 10
937941
run: pnpm test:assert
938942

@@ -941,7 +945,7 @@ jobs:
941945
if: failure()
942946
with:
943947
name: playwright-traces-job_e2e_playwright_tests-${{ matrix.test-application}}
944-
path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application}}/test-results
948+
path: ${{ runner.temp }}/test-application/test-results
945949
overwrite: true
946950
retention-days: 7
947951

@@ -955,7 +959,7 @@ jobs:
955959
if: always()
956960
with:
957961
name: E2E Test Dump (${{ matrix.label || matrix.test-application }})
958-
path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/event-dumps
962+
path: ${{ runner.temp }}/test-application/event-dumps
959963
overwrite: true
960964
retention-days: 7
961965
if-no-files-found: ignore
@@ -1037,19 +1041,23 @@ jobs:
10371041
env:
10381042
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
10391043

1044+
- name: Copy to temp
1045+
run: yarn ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp }}/test-application
1046+
working-directory: dev-packages/e2e-tests
1047+
10401048
- name: Build E2E app
1041-
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
1049+
working-directory: ${{ runner.temp }}/test-application
10421050
timeout-minutes: 7
10431051
run: pnpm ${{ matrix.build-command || 'test:build' }}
10441052

10451053
- name: Install Playwright
10461054
uses: ./.github/actions/install-playwright
10471055
with:
10481056
browsers: chromium
1049-
cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
1057+
cwd: ${{ runner.temp }}/test-application
10501058

10511059
- name: Run E2E test
1052-
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
1060+
working-directory: ${{ runner.temp }}/test-application
10531061
timeout-minutes: 10
10541062
run: pnpm ${{ matrix.assert-command || 'test:assert' }}
10551063

@@ -1063,20 +1071,19 @@ jobs:
10631071
if: always()
10641072
with:
10651073
name: E2E Test Dump (${{ matrix.label || matrix.test-application }})
1066-
path: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/event-dumps
1074+
path: ${{ runner.temp }}/test-application/event-dumps
10671075
overwrite: true
10681076
retention-days: 7
10691077
if-no-files-found: ignore
10701078

10711079
- name: Deploy Astro to Cloudflare
1072-
uses: cloudflare/pages-action@v1
1080+
uses: cloudflare/wrangler-action@v3
10731081
if: matrix.test-application == 'cloudflare-astro'
10741082
with:
10751083
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
10761084
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
1077-
projectName: ${{ secrets.CLOUDFLARE_PROJECT_NAME }}
1078-
directory: dist
1079-
workingDirectory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
1085+
command: pages deploy dist --project-name=${{ secrets.CLOUDFLARE_PROJECT_NAME }}
1086+
workingDirectory: ${{ runner.temp }}/test-application
10801087

10811088
job_required_jobs_passed:
10821089
name: All required jobs passed or were skipped

‎.github/workflows/canary.yml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,19 +153,23 @@ jobs:
153153
env:
154154
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
155155

156+
- name: Copy to temp
157+
run: yarn ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp }}/test-application
158+
working-directory: dev-packages/e2e-tests
159+
156160
- name: Build E2E app
157-
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
161+
working-directory: ${{ runner.temp }}/test-application
158162
timeout-minutes: 7
159163
run: yarn ${{ matrix.build-command }}
160164

161165
- name: Install Playwright
162166
uses: ./.github/actions/install-playwright
163167
with:
164168
browsers: chromium
165-
cwd: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
169+
cwd: ${{ runner.temp }}/test-application
166170

167171
- name: Run E2E test
168-
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
172+
working-directory: ${{ runner.temp }}/test-application
169173
timeout-minutes: 15
170174
run: yarn test:assert
171175

‎dev-packages/e2e-tests/ciCopyToTemp.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/* eslint-disable no-console */
2+
3+
import { copyToTemp } from './lib/copyToTemp';
4+
5+
async function run(): Promise<void> {
6+
const originalPath = process.argv[2];
7+
const tmpDirPath = process.argv[3];
8+
9+
if (!originalPath || !tmpDirPath) {
10+
throw new Error('Original path and tmp dir path are required');
11+
}
12+
13+
console.log(`Copying ${originalPath} to ${tmpDirPath}...`);
14+
15+
await copyToTemp(originalPath, tmpDirPath);
16+
}
17+
18+
// eslint-disable-next-line @typescript-eslint/no-floating-promises
19+
run();

‎dev-packages/e2e-tests/lib/copyToTemp.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/* eslint-disable no-console */
2+
import { readFileSync, writeFileSync } from 'fs';
3+
import { cp } from 'fs/promises';
4+
import { join } from 'path';
5+
6+
export async function copyToTemp(originalPath: string, tmpDirPath: string): Promise<void> {
7+
// copy files to tmp dir
8+
await cp(originalPath, tmpDirPath, { recursive: true });
9+
10+
fixPackageJson(tmpDirPath);
11+
}
12+
13+
function fixPackageJson(cwd: string): void {
14+
const packageJsonPath = join(cwd, 'package.json');
15+
const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')) as {
16+
dependencies?: Record<string, string>;
17+
devDependencies?: Record<string, string>;
18+
volta?: Record<string, unknown>;
19+
};
20+
21+
// 1. Fix file dependencies
22+
if (packageJson.dependencies) {
23+
fixFileLinkDependencies(packageJson.dependencies);
24+
}
25+
if (packageJson.devDependencies) {
26+
fixFileLinkDependencies(packageJson.devDependencies);
27+
}
28+
29+
// 2. Fix volta extends
30+
if (!packageJson.volta) {
31+
throw new Error('No volta config found, please provide one!');
32+
}
33+
34+
if (typeof packageJson.volta.extends === 'string') {
35+
const extendsPath = packageJson.volta.extends;
36+
// We add a virtual dir to ensure that the relative depth is consistent
37+
// dirPath is relative to ./../test-applications/xxx
38+
const newPath = join(__dirname, 'virtual-dir/', extendsPath);
39+
packageJson.volta.extends = newPath;
40+
console.log(`Fixed volta.extends to ${newPath}`);
41+
} else {
42+
console.log('No volta.extends found');
43+
}
44+
45+
writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
46+
}
47+
48+
function fixFileLinkDependencies(dependencyObj: Record<string, string>): void {
49+
for (const [key, value] of Object.entries(dependencyObj)) {
50+
if (value.startsWith('link:')) {
51+
const dirPath = value.replace('link:', '');
52+
53+
// We add a virtual dir to ensure that the relative depth is consistent
54+
// dirPath is relative to ./../test-applications/xxx
55+
const newPath = join(__dirname, 'virtual-dir/', dirPath);
56+
57+
dependencyObj[key] = `link:${newPath}`;
58+
console.log(`Fixed ${key} dependency to ${newPath}`);
59+
}
60+
}
61+
}

‎dev-packages/e2e-tests/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
"clean": "rimraf tmp node_modules && yarn clean:test-applications && yarn clean:pnpm",
1717
"ci:build-matrix": "ts-node ./lib/getTestMatrix.ts",
1818
"ci:build-matrix-optional": "ts-node ./lib/getTestMatrix.ts --optional=true",
19-
"clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.nuxt,.sveltekit,.react-router,.astro,.output,pnpm-lock.yaml,.last-run.json,test-results}",
19+
"ci:copy-to-temp": "ts-node ./ciCopyToTemp.ts",
20+
"clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.nuxt,.sveltekit,.react-router,.astro,.output,pnpm-lock.yaml,.last-run.json,test-results,.angular,event-dumps}",
2021
"clean:pnpm": "pnpm store prune"
2122
},
2223
"devDependencies": {

‎dev-packages/e2e-tests/run.ts‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
/* eslint-disable no-console */
22
import { spawn } from 'child_process';
33
import * as dotenv from 'dotenv';
4+
import { mkdtemp, rm } from 'fs/promises';
45
import { sync as globSync } from 'glob';
5-
import { resolve } from 'path';
6+
import { tmpdir } from 'os';
7+
import { join, resolve } from 'path';
8+
import { copyToTemp } from './lib/copyToTemp';
69
import { registrySetup } from './registrySetup';
710

811
const DEFAULT_DSN = 'https://username@domain/123';
@@ -39,7 +42,7 @@ async function run(): Promise<void> {
3942
dotenv.config();
4043

4144
// Allow to run a single app only via `yarn test:run <app-name>`
42-
const appName = process.argv[2];
45+
const appName = process.argv[2] || '';
4346

4447
const dsn = process.env.E2E_TEST_DSN || DEFAULT_DSN;
4548

@@ -74,13 +77,20 @@ async function run(): Promise<void> {
7477
console.log('');
7578

7679
for (const testAppPath of testAppPaths) {
77-
const cwd = resolve('test-applications', testAppPath);
80+
const originalPath = resolve('test-applications', testAppPath);
81+
const tmpDirPath = await mkdtemp(join(tmpdir(), `sentry-e2e-tests-${appName}-`));
7882

79-
console.log(`Building ${testAppPath}...`);
83+
await copyToTemp(originalPath, tmpDirPath);
84+
const cwd = tmpDirPath;
85+
86+
console.log(`Building ${testAppPath} in ${tmpDirPath}...`);
8087
await asyncExec('pnpm test:build', { env, cwd });
8188

8289
console.log(`Testing ${testAppPath}...`);
8390
await asyncExec('pnpm test:assert', { env, cwd });
91+
92+
// clean up (although this is tmp, still nice to do)
93+
await rm(tmpDirPath, { recursive: true });
8494
}
8595
} catch (error) {
8696
console.error(error);

‎dev-packages/e2e-tests/test-applications/angular-17/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"ng": "ng",
66
"dev": "ng serve",
77
"proxy": "node start-event-proxy.mjs",
8-
"preview": "http-server dist/angular-17/browser --port 8080",
8+
"preview": "http-server dist/angular-17/browser --port 8080 --silent",
99
"build": "ng build",
1010
"watch": "ng build --watch --configuration development",
1111
"test": "playwright test",
@@ -31,6 +31,7 @@
3131
"devDependencies": {
3232
"@playwright/test": "~1.50.0",
3333
"@sentry-internal/test-utils": "link:../../../test-utils",
34+
"@sentry/core": "latest || *",
3435
"@angular-devkit/build-angular": "^17.1.1",
3536
"@angular/cli": "^17.1.1",
3637
"@angular/compiler-cli": "^17.1.0",

‎dev-packages/e2e-tests/test-applications/angular-17/tests/performance.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { expect, test } from '@playwright/test';
22
import { waitForTransaction } from '@sentry-internal/test-utils';
3+
// Cannot use @sentry/angular here due to build stuff
34
import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core';
45

56
test('sends a pageload transaction with a parameterized URL', async ({ page }) => {

‎dev-packages/e2e-tests/test-applications/angular-18/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"ng": "ng",
66
"dev": "ng serve",
77
"proxy": "node start-event-proxy.mjs",
8-
"preview": "http-server dist/angular-18/browser --port 8080",
8+
"preview": "http-server dist/angular-18/browser --port 8080 --silent",
99
"build": "ng build",
1010
"watch": "ng build --watch --configuration development",
1111
"test": "playwright test",
@@ -31,6 +31,7 @@
3131
"devDependencies": {
3232
"@playwright/test": "~1.50.0",
3333
"@sentry-internal/test-utils": "link:../../../test-utils",
34+
"@sentry/core": "latest || *",
3435
"@angular-devkit/build-angular": "^18.0.0",
3536
"@angular/cli": "^18.0.0",
3637
"@angular/compiler-cli": "^18.0.0",

‎dev-packages/e2e-tests/test-applications/angular-18/tests/performance.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { expect, test } from '@playwright/test';
22
import { waitForTransaction } from '@sentry-internal/test-utils';
3+
// Cannot use @sentry/angular here due to build stuff
34
import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core';
45

56
test('sends a pageload transaction with a parameterized URL', async ({ page }) => {

0 commit comments

Comments
 (0)