Skip to content

Commit e92b039

Browse files
cnotvcnotv
authored
E2E job parallelization by user role and functionality (#9887)
* Create matrix for E2E job * Create utility for mapping skipped tests for given dirs * Allow to filter any dir for given env vars Add Rancher setup phase in test CI Restore skip setup variable for other checks * Cypress config cleanup * Avoid use of tags for setup test Add setup tag * Add specs dir to the matrix Correct matrix to use each role Add tag prefix on command * Correct coverage path * Simplify cypress utility * Implement only env var filtering logic Update env var filtering with a menageable definition Update CI to filter paths by new logic Reorganize tests file tree Add tags for each section Combine roles and features specs Update new specs to the tag format Grouping tags Fix issues with specs and admin spec/username for login parallelization Fix issue pre-pending @ character Remove coupling Remove Docker stop Enable all the tests in groups Allow E2E matrix to fail with strategy flag fail-fast * Add role for setup in combination to features * Change tag AND to OR (space instead plus) * Exclude non tagged tests instead of skip * Correct standard user name * Use object for matrix definition; Restore usernames for tests; Remove hackish tag for role * Pair matrix role tag to multiple matrix features * Disable failing tests * Move diagnostic from extension to generic * Remove build dashboard flag for setup * Correct screenshot naming to match coverage file naming * Move Cypress utils out of shell * Remove View API check for fleet * Add documentation * Update skip logic and documentation * Update env template
1 parent c9b2f79 commit e92b039

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+302
-153
lines changed

‎.github/workflows/test.yaml

+29-16
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,22 @@ env:
3030
jobs:
3131
e2e-test:
3232
if: "!contains( github.event.pull_request.labels.*.name, 'ci/skip-e2e')"
33+
strategy:
34+
fail-fast: false
35+
matrix:
36+
role: [
37+
{ username: 'admin', tag: '@adminUser' },
38+
{ username: 'standard_user', tag: '@standardUser' }
39+
]
40+
features: [
41+
['@navigation', '@extensions'],
42+
['@charts'],
43+
['@explorer'],
44+
['@fleet'],
45+
['@generic', '@globalSettings'],
46+
['@manager'],
47+
['@userMenu', '@usersAndAuths']
48+
]
3349
runs-on: ubuntu-latest
3450
steps:
3551
- uses: actions/checkout@v3
@@ -39,6 +55,7 @@ jobs:
3955
with:
4056
node-version: '14.x'
4157

58+
# Installing fixed version of Chrome since latest version does not work
4259
- name: Install Chrome 116
4360
run: |
4461
sudo apt-get install -y wget
@@ -53,42 +70,38 @@ jobs:
5370
- name: Prepare build
5471
run: yarn e2e:pre-prod
5572

56-
- name: Run admin user tests
73+
- name: Setup Rancher and user
5774
run: |
5875
yarn e2e:prod
59-
[ "$BUILD_DASHBOARD" != "false" ] || exit 0
60-
mkdir -p coverage-artifacts/coverage
61-
cp coverage/e2e/coverage-final.json coverage-artifacts/coverage/coverage-e2e-admin.json
62-
cp -r coverage/e2e/ coverage-artifacts/coverage/e2e-admin/
6376
env:
64-
GREP_TAGS: '@adminUser'
77+
GREP_TAGS: ${{ matrix.role.tag }}Setup+${{ matrix.features[0] }} ${{ matrix.role.tag }}Setup+${{ matrix.features[1] || matrix.features[0] }}
6578
TEST_USERNAME: admin
66-
67-
- name: Run standard user tests
68-
if: ${{ success() || failure() }}
79+
TEST_ONLY: setup
80+
81+
- name: Run user tests
6982
run: |
7083
yarn e2e:prod
71-
yarn docker:local:stop
7284
[ "$BUILD_DASHBOARD" != "false" ] || exit 0
7385
mkdir -p coverage-artifacts/coverage
74-
cp coverage/e2e/coverage-final.json coverage-artifacts/coverage/coverage-e2e-user.json
75-
cp -r coverage/e2e/ coverage-artifacts/coverage/e2e-user/
86+
cp coverage/e2e/coverage-final.json coverage-artifacts/coverage/coverage-e2e.json
87+
cp -r coverage/e2e/ coverage-artifacts/coverage/e2e/
7688
env:
77-
GREP_TAGS: '@standardUser'
78-
TEST_USERNAME: standard_user
89+
TEST_SKIP: setup
90+
GREP_TAGS: ${{ matrix.role.tag }}+${{ matrix.features[0] }} ${{ matrix.role.tag }}+${{ matrix.features[1] || matrix.features[0] }}
91+
TEST_USERNAME: ${{ matrix.role.username }}
7992

8093
- name: Upload coverage
8194
uses: actions/upload-artifact@v3
8295
if: env.BUILD_DASHBOARD != 'false'
8396
with:
84-
name: ${{github.run_number}}-${{github.run_attempt}}-coverage
97+
name: ${{github.run_number}}-${{github.run_attempt}}-coverage-${{ matrix.role.tag }}+${{ matrix.features[0] }}
8598
path: coverage-artifacts/**/*
8699

87100
- name: Upload screenshots
88101
uses: actions/upload-artifact@v3
89102
if: ${{ failure() }}
90103
with:
91-
name: ${{github.run_number}}-${{github.run_attempt}}-screenshots
104+
name: ${{github.run_number}}-${{github.run_attempt}}-screenshots-${{ matrix.role.tag }}+${{ matrix.features[0] }}
92105
path: cypress/screenshots
93106

94107
unit-test:

‎cypress.config.ts

+33-56
Original file line numberDiff line numberDiff line change
@@ -1,74 +1,50 @@
1+
/* eslint-disable no-console */
12
import { defineConfig } from 'cypress';
23
import { removeDirectory } from 'cypress-delete-downloads-folder';
4+
import { getSpecPattern } from '@/scripts/cypress';
35
// Required for env vars to be available in cypress
46
require('dotenv').config();
57

6-
const skipSetup = process.env.TEST_SKIP_SETUP === 'true';
7-
const hasCoverage = (process.env.TEST_INSTRUMENT === 'true') || false; // Add coverage if instrumented
8-
98
/**
10-
* Filter test spec paths based on env var configuration
11-
* @returns
9+
* VARIABLES
1210
*/
13-
const getSpecPattern = (): string[] => {
14-
const optionalPaths = [
15-
{
16-
path: 'cypress/e2e/tests/setup/**/*.spec.ts',
17-
active: !skipSetup
18-
}
19-
];
20-
const activePaths = optionalPaths.filter(({ active }) => Boolean(active)).map(({ path }) => path);
21-
22-
// List the test directories to be included
23-
const testDirs = ['pages', 'navigation', 'global-ui'].map((dir) => `cypress/e2e/tests/${ dir }/**/*.spec.ts`);
24-
25-
return [
26-
...activePaths,
27-
...testDirs
28-
];
29-
};
11+
const hasCoverage = (process.env.TEST_INSTRUMENT === 'true') || false; // Add coverage if instrumented
12+
const testDirs = ['setup', 'pages', 'navigation', 'global-ui'];
13+
const skipSetup = process.env.TEST_SKIP?.includes('setup');
3014
const baseUrl = (process.env.TEST_BASE_URL || 'https://localhost:8005').replace(/\/$/, '');
31-
32-
// Default user name, if TEST_USERNAME is not provided
3315
const DEFAULT_USERNAME = 'admin';
16+
const username = process.env.TEST_USERNAME || DEFAULT_USERNAME;
17+
const apiUrl = process.env.API || (baseUrl.endsWith('/dashboard') ? baseUrl.split('/').slice(0, -1).join('/') : baseUrl);
3418

35-
// Log summary of the environment variables that we have detected (or are going ot use) - we won't show any passwords
36-
console.log('E2E Test Configuration'); // eslint-disable-line no-console
37-
console.log(''); // eslint-disable-line no-console
19+
/**
20+
* LOGS:
21+
* Summary of the environment variables that we have detected (or are going ot use)
22+
* We won't show any passwords
23+
*/
24+
console.log('E2E Test Configuration');
25+
console.log('');
26+
console.log(` Username: ${ username }`);
3827

39-
if (process.env.TEST_USERNAME) {
40-
console.log(` Username: ${ process.env.TEST_USERNAME }`); // eslint-disable-line no-console
41-
} else {
42-
console.log(` Username: ${ DEFAULT_USERNAME } (TEST_USERNAME not set, using default)`); // eslint-disable-line no-console
28+
if (!process.env.CATTLE_BOOTSTRAP_PASSWORD && !process.env.TEST_PASSWORD) {
29+
console.log(' ❌ You must provide either CATTLE_BOOTSTRAP_PASSWORD or TEST_PASSWORD');
4330
}
44-
4531
if (process.env.CATTLE_BOOTSTRAP_PASSWORD && process.env.TEST_PASSWORD) {
46-
console.log(' ❌ You should not set both CATTLE_BOOTSTRAP_PASSWORD and TEST_PASSWORD - CATTLE_BOOTSTRAP_PASSWORD will be used'); // eslint-disable-line no-console
32+
console.log(' ❗ If both CATTLE_BOOTSTRAP_PASSWORD and TEST_PASSWORD are provided, the first will be used');
4733
}
48-
4934
if (!skipSetup && !process.env.CATTLE_BOOTSTRAP_PASSWORD) {
50-
console.log(' ❌ You must provide CATTLE_BOOTSTRAP_PASSWORD when running setup tests'); // eslint-disable-line no-console
51-
}
52-
53-
if (!process.env.CATTLE_BOOTSTRAP_PASSWORD && !process.env.TEST_PASSWORD) {
54-
console.log(' ❌ You must provide one of CATTLE_BOOTSTRAP_PASSWORD or TEST_PASSWORD'); // eslint-disable-line no-console
35+
console.log(' ❌ You must provide CATTLE_BOOTSTRAP_PASSWORD when running setup tests');
5536
}
56-
5737
if (skipSetup && !process.env.TEST_PASSWORD) {
58-
console.log(' ❌ You should provide TEST_PASSWORD when running the tests without the setup tests'); // eslint-disable-line no-console
38+
console.log(' ❌ You must provide TEST_PASSWORD when running the tests without the setup tests');
5939
}
6040

61-
if (skipSetup) {
62-
console.log(` Setup tests will NOT be run`); // eslint-disable-line no-console
63-
} else {
64-
console.log(` Setup tests will be run`); // eslint-disable-line no-console
65-
}
41+
console.log(` Setup tests will ${ skipSetup ? 'NOT' : '' } be run`);
42+
console.log(` Dashboard URL: ${ baseUrl }`);
43+
console.log(` Rancher API URL: ${ apiUrl }`);
6644

67-
console.log(` Dashboard URL: ${ baseUrl }`); // eslint-disable-line no-console
68-
69-
const apiUrl = process.env.API || (baseUrl.endsWith('/dashboard') ? baseUrl.split('/').slice(0, -1).join('/') : baseUrl);
70-
71-
console.log(` Rancher API URL: ${ apiUrl }`); // eslint-disable-line no-console
45+
/**
46+
* CONFIGURATION
47+
*/
7248
export default defineConfig({
7349
projectId: process.env.TEST_PROJECT_ID,
7450
defaultCommandTimeout: process.env.TEST_TIMEOUT ? +process.env.TEST_TIMEOUT : 60000,
@@ -79,10 +55,11 @@ export default defineConfig({
7955
openMode: 0
8056
},
8157
env: {
82-
grepFilterSpecs: true,
58+
grepFilterSpecs: true,
59+
grepOmitFiltered: true,
8360
baseUrl,
84-
coverage: hasCoverage,
85-
codeCoverage: {
61+
coverage: hasCoverage,
62+
codeCoverage: {
8663
exclude: [
8764
'cypress/**/*.*',
8865
'**/__tests__/**/*.*',
@@ -100,7 +77,7 @@ export default defineConfig({
10077
]
10178
},
10279
api: apiUrl,
103-
username: process.env.TEST_USERNAME || DEFAULT_USERNAME,
80+
username,
10481
password: process.env.CATTLE_BOOTSTRAP_PASSWORD || process.env.TEST_PASSWORD,
10582
bootstrapPassword: process.env.CATTLE_BOOTSTRAP_PASSWORD,
10683
grepTags: process.env.GREP_TAGS
@@ -117,7 +94,7 @@ export default defineConfig({
11794
return config;
11895
},
11996
experimentalSessionAndOrigin: true,
120-
specPattern: getSpecPattern(),
97+
specPattern: getSpecPattern(testDirs, process.env),
12198
baseUrl
12299
},
123100
videoCompression: 15,

‎cypress/e2e/tests/pages/data/agent-configuration-rke2-data.ts ‎cypress/e2e/blueprints/agent-configuration/agent-configuration-rke2-data.ts

File renamed without changes.

‎cypress/e2e/tests/pages/data/agent-configuration-rke2-payload.ts ‎cypress/e2e/blueprints/agent-configuration/agent-configuration-rke2-payload.ts

File renamed without changes.

‎cypress/e2e/tests/navigation/page-actions.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import PageActions from '@/cypress/e2e/po/side-bars/page-actions.po';
22
import HomePagePo from '@/cypress/e2e/po/pages/home.po';
33

4-
describe('Page Actions', { tags: ['@adminUser', '@standardUser'] }, () => {
4+
describe('Page Actions', { tags: ['@navigation', '@adminUser', '@standardUser'] }, () => {
55
beforeEach(() => {
66
cy.login();
77

‎cypress/e2e/tests/navigation/side-nav/main-side-menu.spec.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,19 @@ describe('Side Menu: main', () => {
88
HomePagePo.goTo();
99
BurgerMenuPo.toggle();
1010
});
11-
it('Opens and closes on menu icon click', { tags: ['@adminUser', '@standardUser'] }, () => {
11+
it('Opens and closes on menu icon click', { tags: ['@navigation', '@adminUser', '@standardUser'] }, () => {
1212
BurgerMenuPo.checkOpen();
1313
BurgerMenuPo.toggle();
1414
BurgerMenuPo.checkClosed();
1515
});
1616

17-
it('Can display list of available clusters', { tags: '@adminUser' }, () => {
17+
it('Can display list of available clusters', { tags: ['@navigation', '@adminUser'] }, () => {
1818
const burgerMenuPo = new BurgerMenuPo();
1919

2020
burgerMenuPo.clusters().should('exist');
2121
});
2222

23-
it('Pinned and unpinned cluster', { tags: ['@adminUser', '@standardUser'] }, () => {
23+
it('Pinned and unpinned cluster', { tags: ['@navigation', '@adminUser', '@standardUser'] }, () => {
2424
const burgerMenuPo = new BurgerMenuPo();
2525

2626
burgerMenuPo.pinFirstCluster();
@@ -29,13 +29,13 @@ describe('Side Menu: main', () => {
2929
burgerMenuPo.clusterPinnedList().should('not.exist');
3030
});
3131

32-
it('Can display at least one menu category label', { tags: ['@adminUser', '@standardUser'] }, () => {
32+
it('Can display at least one menu category label', { tags: ['@navigation', '@adminUser', '@standardUser'] }, () => {
3333
const burgerMenuPo = new BurgerMenuPo();
3434

3535
burgerMenuPo.categories().should('have.length', 1);
3636
});
3737

38-
it('Should show tooltip on mouse-hover when the menu is collapsed', { tags: ['@adminUser', '@standardUser'] }, () => {
38+
it('Should show tooltip on mouse-hover when the menu is collapsed', { tags: ['@navigation', '@adminUser', '@standardUser'] }, () => {
3939
const burgerMenuPo = new BurgerMenuPo();
4040

4141
burgerMenuPo.clusters().first().trigger('mouseover');
@@ -45,7 +45,7 @@ describe('Side Menu: main', () => {
4545
});
4646

4747
// TODO: #5966: Verify cause of race condition issue making navigation link not trigger
48-
it.skip('Contains valid links', { tags: ['@adminUser', '@standardUser'] }, () => {
48+
it.skip('Contains valid links', { tags: ['@navigation', '@adminUser', '@standardUser'] }, () => {
4949
const burgerMenuPo = new BurgerMenuPo();
5050
// Navigate through all the links
5151

‎cypress/e2e/tests/navigation/side-nav/product-side-nav.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import BurgerMenuPo from '@/cypress/e2e/po/side-bars/burger-side-menu.po';
33
import ProductNavPo from '@/cypress/e2e/po/side-bars/product-side-nav.po';
44

55
Cypress.config();
6-
describe('Side navigation: Cluster ', { tags: '@adminUser' }, () => {
6+
describe('Side navigation: Cluster ', { tags: ['@navigation', '@adminUser'] }, () => {
77
beforeEach(() => {
88
cy.login();
99

‎cypress/e2e/tests/pages/charts/istio.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { ChartsPage } from '@/cypress/e2e/po/pages/charts.po';
22
import ClusterDashboardPagePo from '@/cypress/e2e/po/pages/explorer/cluster-dashboard.po';
33
import ProductNavPo from '@/cypress/e2e/po/side-bars/product-side-nav.po';
44

5-
describe('Charts', { tags: '@adminUser' }, () => {
5+
describe('Charts', { tags: ['@charts', '@adminUser'] }, () => {
66
const clusterName = 'local';
77
const chartsPageUrl = '/c/local/apps/charts/chart?repo-type=cluster&repo=rancher-charts';
88

‎cypress/e2e/tests/pages/charts/opa-gatekeeper.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/* eslint-disable cypress/no-unnecessary-waiting */
22
import { ChartsPage } from '@/cypress/e2e/po/pages/charts.po';
33

4-
describe('Charts', { tags: '@adminUser' }, () => {
4+
describe('Charts', { tags: ['@charts', '@adminUser'] }, () => {
55
beforeEach(() => {
66
cy.login();
77
});

‎cypress/e2e/tests/pages/charts/prometheus.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import CheckboxPo from '@/cypress/e2e/po/components/checkbox-input.po';
77

88
import { prometheusSpec } from '@/cypress/e2e/blueprints/charts/prometheus-chart';
99

10-
describe('Charts', { tags: '@adminUser' }, () => {
10+
describe('Charts', { tags: ['@charts', '@adminUser'] }, () => {
1111
const chartsPageUrl = '/c/local/apps/charts/chart?repo-type=cluster&repo=rancher-charts';
1212

1313
describe('Monitoring', () => {

‎cypress/e2e/tests/pages/explorer/api/api-services.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { APIServicesPagePo } from '@/cypress/e2e/po/pages/explorer/api-services.po';
22

3-
describe('Cluster Explorer', { tags: ['@adminUser'] }, () => {
3+
describe('Cluster Explorer', { tags: ['@explorer', '@adminUser'] }, () => {
44
beforeEach(() => {
55
cy.login();
66
});

‎cypress/e2e/tests/pages/explorer/cluster-project-members.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ const runPrefix = `e2e-test-${ runTimestamp }`;
77
const username = `${ runPrefix }-cluster-proj-member`;
88
const standardPassword = 'standard-password';
99

10-
describe('Cluster Project and Members', { tags: ['@adminUser'] }, () => {
10+
describe('Cluster Project and Members', { tags: ['@explorer', '@adminUser'] }, () => {
1111
it('Members added to both Cluster Membership should not show "Loading..." next to their names', () => {
1212
const usersAdmin = new UsersPo('_');
1313
const userCreate = usersAdmin.createEdit();

‎cypress/e2e/tests/pages/project-namespace.spec.ts ‎cypress/e2e/tests/pages/explorer/project-namespace.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import projectsNamespacesePo from '@/cypress/e2e/po/pages/explorer/projects-namespaces.po';
22

3-
describe('Projects/Namespaces', { tags: '@adminUser' }, () => {
3+
describe('Projects/Namespaces', { tags: ['@explorer', '@adminUser'] }, () => {
44
const projectsNamespacesesPage = new projectsNamespacesePo('local');
55

66
beforeEach(() => {

‎cypress/e2e/tests/pages/explorer/workloads/deployments.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ describe('Cluster Explorer', () => {
66
cy.login();
77
});
88

9-
describe('Workloads', { tags: ['@standardUser', '@adminUser', '@flaky'] }, () => {
9+
describe('Workloads', { tags: ['@explorer', '@standardUser', '@adminUser', '@flaky'] }, () => {
1010
let deploymentsListPage: WorkloadsDeploymentsListPagePo;
1111
let deploymentsCreatePage: WorkloadsDeploymentsCreatePagePo;
1212

‎cypress/e2e/tests/pages/explorer/workloads/jobs.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { WorkloadsJobsListPagePo, WorkLoadsJobDetailsPagePo } from '@/cypress/e2e/po/pages/explorer/workloads-jobs.po';
22

3-
describe('Cluster Explorer', { tags: '@adminUser' }, () => {
3+
describe('Cluster Explorer', { tags: ['@explorer', '@adminUser'] }, () => {
44
beforeEach(() => {
55
cy.login();
66
});

‎cypress/e2e/tests/pages/explorer/workloads/pods.spec.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ describe('Cluster Explorer', () => {
1111
describe('Pods', () => {
1212
const workloadsPodPage = new WorkloadsPodsListPagePo('local');
1313

14-
describe('Should open a terminal', { tags: ['@adminUser', '@standardUser'] }, () => {
14+
describe('Should open a terminal', { tags: ['@explorer', '@adminUser'] }, () => {
1515
beforeEach(() => {
1616
workloadsPodPage.goTo();
1717
});
@@ -23,7 +23,7 @@ describe('Cluster Explorer', () => {
2323
});
2424
});
2525

26-
describe('When cloning a pod', { tags: ['@adminUser'] }, () => {
26+
describe('When cloning a pod', { tags: ['@explorer', '@adminUser'] }, () => {
2727
const { name: origPodName, namespace } = createPodBlueprint.metadata;
2828
const { name: clonePodName } = clonePodBlueprint.metadata;
2929

‎cypress/e2e/tests/pages/explorer/workloads/workloads.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import WorkloadPagePo from '@/cypress/e2e/po/pages/explorer/workloads.po';
55

66
const podName = 'some-pod-name';
77

8-
describe('Workloads', { tags: '@adminUser' }, () => {
8+
describe('Workloads', { tags: ['@explorer', '@adminUser'] }, () => {
99
beforeEach(() => {
1010
cy.login();
1111
});

‎cypress/e2e/tests/pages/extensions.spec.ts ‎cypress/e2e/tests/pages/extensions/extensions.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ const EXTENSION_NAME = 'clock';
66
const UI_PLUGINS_PARTNERS_REPO_URL = 'https://github.com/rancher/partner-extensions';
77
const UI_PLUGINS_PARTNERS_REPO_NAME = 'partner-extensions';
88

9-
describe('Extensions page', { tags: '@adminUser' }, () => {
9+
describe('Extensions page', { tags: ['@extensions', '@adminUser'] }, () => {
1010
before(() => {
1111
cy.login();
1212

0 commit comments

Comments
 (0)