PoC performance tests in CI (#2474)
* PoC performance tests in CI * Add a naive measurement * Change in spec descriptions * More minimal reporter and 2 tests: 1 failing, 1 succeeding * Fix circleci config * Fix circleci part 2 * Make performance tests a workflow * add npm install in performance tests * change executor * Experimenting: Install crossenv * Experimenting * More experimenting * Revert some changes * Run update_image and cache node modules * Sandbox e2e tests run * fix code residual from rebase * Add GH action for ci perf * Use ubuntu instead of macos * Use windows instead of ubuntu * Fix env set command * Use post-test script to check performance * Use macos-latest * Throw error when tests fail * Use measudePerformance function instead * Use measudePerformance function instead * Try to run e2e on github actions * Try headless script * Install linux packages * Add shell * Use if: runner.. * ADd sudo * Add more linux packages * Go back to macos * Try recursive find for main window * Remove performance tests from circleCi * Increase retries * Fix * Revert "Fix" This reverts commit c19fc56600ada4eee3e26bf3988b27d2ce2616d1. * Revert "Increase retries" This reverts commit 564ac95c34def892595c17b98eec245cacb0d4a4. * Revert "Remove performance tests from circleCi" This reverts commit c49c73b6f811df12631036bc33dae30da2271173. * Revert "Try recursive find for main window" This reverts commit 19d9367f9317f6e9d7ef9aa6a5d69eccdff2df67. * Revert "Go back to macos" This reverts commit 72b796ad983e89940f84d53039affd867d0e7f34. * Revert "Add more linux packages" This reverts commit 4bd510f8ea5356c13fbc4722be874aeea32de43e. * Revert "ADd sudo" This reverts commit acb95e02609e7cc02b12863910d29c03694936c1. * Revert "Use if: runner.." This reverts commit 69beed6fe24e452b1d7f0b0e4578bbfa9ca36d2f. * Revert "Add shell" This reverts commit e1743664a8b01797da4112314af0b88318ad2fd7. * Revert "Install linux packages" This reverts commit 25195dd11c827dff0fd8cf4fbdf2dc709377d525. * Revert "Try headless script" This reverts commit 385be5609d5ee45363d26828f6a14c6be0ef661a. * Revert "Try to run e2e on github actions" This reverts commit be8f682fd8a82b265403471b5106fd5107f9fecb. * Revert "Use measudePerformance function instead" This reverts commit bfd46768ae845e2839e9f124735922da6301fa4e. * Revert "Use measudePerformance function instead" This reverts commit ab4bc22b8689ff35260b4ab55724a77dc5993426. * Update workflow command * Remove circleci config for perf tests * Run on ubuntu * Add packages for linux * Fix order of commands * Move env var at the beginning * Upload test results to s3 * Update name * Update name * Add trigger for flag * Add step to upload artifact to github workflow artifacts * Update yml * Add step to comment the results in the PR * Fix env bug * move constant to workflow env * Wrap workflow env with single quotes * Add start comment and remove label steps * Update body of comment * Remove unused file * Move PR comment function into file * Fix import * Remove spaces from line start * Remove label even if workflow failed * Add logging * Fix markdown table data * Remove trailing semi string --------- Co-authored-by: Tasos Boulis <tboulis@hotmail.com> Co-authored-by: Mattermost Build <build@mattermost.com>
This commit is contained in:
parent
bbb29a0d1b
commit
2bb622c314
|
@ -51,7 +51,7 @@ commands:
|
|||
type: string
|
||||
default: "https://mattermost.com/wp-content/uploads/2022/02/icon.png"
|
||||
steps:
|
||||
- run:
|
||||
- run:
|
||||
command: |
|
||||
export VERSION=$(jq -r .version package.json)
|
||||
echo '{}' | jq "{
|
||||
|
@ -108,7 +108,7 @@ commands:
|
|||
type: string
|
||||
default: "./linux/"
|
||||
steps:
|
||||
- run:
|
||||
- run:
|
||||
name: npn run
|
||||
command: npm run package:<< parameters.os >>
|
||||
no_output_timeout: 60m
|
||||
|
@ -123,7 +123,7 @@ commands:
|
|||
description: "Running automated tests"
|
||||
steps:
|
||||
- run: npm run check-types
|
||||
- run:
|
||||
- run:
|
||||
name: i18n check
|
||||
command: |
|
||||
cp i18n/en.json /tmp/en.json
|
||||
|
@ -139,7 +139,7 @@ commands:
|
|||
description: "Running automated tests"
|
||||
steps:
|
||||
- run: npm run check-types
|
||||
- run:
|
||||
- run:
|
||||
name: i18n check
|
||||
command: |
|
||||
cp .\i18n\en.json $env:TEMP\en.json
|
||||
|
@ -459,7 +459,7 @@ jobs:
|
|||
grep "win" ./templist.txt | awk -F/ '{print "- ["$NF"]("$0")"}' >> ./build/linklist.txt
|
||||
- persist_to_workspace:
|
||||
root: ./build
|
||||
paths:
|
||||
paths:
|
||||
- ./linklist.txt
|
||||
- ./artifactlist.txt
|
||||
|
||||
|
@ -477,7 +477,7 @@ jobs:
|
|||
command: |
|
||||
echo "Links for $(date +"%b-%d-%Y")" >> ./links/linklist.txt
|
||||
cat ./build/linklist.txt >> ./links/linklist.txt
|
||||
- run:
|
||||
- run:
|
||||
command: |
|
||||
linklist=$(<./links/linklist.txt);
|
||||
echo '{}' | jq "{
|
||||
|
@ -500,7 +500,7 @@ jobs:
|
|||
command: |
|
||||
if [ -z `git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null` ]; then
|
||||
circleci-agent step halt
|
||||
fi
|
||||
fi
|
||||
- run:
|
||||
name: "Setup files for aws-s3"
|
||||
command: |
|
||||
|
@ -510,7 +510,7 @@ jobs:
|
|||
from: ./aws-s3-dist/
|
||||
to: s3://releases.mattermost.com/desktop/
|
||||
arguments: --acl public-read --cache-control "no-cache" --recursive
|
||||
|
||||
|
||||
upload_to_s3_daily:
|
||||
executor: aws
|
||||
steps:
|
||||
|
@ -547,7 +547,7 @@ jobs:
|
|||
command: |
|
||||
if [ -z `git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null` ]; then
|
||||
circleci-agent step halt
|
||||
fi
|
||||
fi
|
||||
- run:
|
||||
name: "Setup files for ghr"
|
||||
command: |
|
||||
|
@ -578,7 +578,7 @@ jobs:
|
|||
apt_opts: "--no-install-recommends jq"
|
||||
- notify:
|
||||
message: "Release process for the desktop app has started, it should take about 30 minutes to complete."
|
||||
|
||||
|
||||
end-notification:
|
||||
executor: wine-chrome
|
||||
steps:
|
||||
|
@ -608,7 +608,6 @@ workflows:
|
|||
ignore:
|
||||
- /^build-pr-.*/
|
||||
- nightly
|
||||
|
||||
- build-win-no-installer:
|
||||
filters:
|
||||
branches:
|
||||
|
@ -742,11 +741,11 @@ workflows:
|
|||
when: << pipeline.parameters.run_nightly >>
|
||||
jobs:
|
||||
- msi_installer:
|
||||
context:
|
||||
context:
|
||||
- windows-codesign
|
||||
- desktop-rainforest-build
|
||||
- mac_installer:
|
||||
context:
|
||||
context:
|
||||
- codesign-certificates
|
||||
- desktop-rainforest-build
|
||||
- upload_to_s3_daily:
|
||||
|
|
93
.github/workflows/e2e-performance.yml
vendored
Normal file
93
.github/workflows/e2e-performance.yml
vendored
Normal file
|
@ -0,0 +1,93 @@
|
|||
name: E2E Performance Tests (Desktop)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
types:
|
||||
- labeled
|
||||
|
||||
env:
|
||||
RESULTS_PATH: e2e/performance/perf-test-report.json
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: ${{ github.event.label.name == 'Run E2E Performance Tests' }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
node-version: [16]
|
||||
|
||||
steps:
|
||||
|
||||
- name: Add start comment
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: `E2E Performance Tests started 🏎️`,
|
||||
});
|
||||
|
||||
- name: Set env variable for timestamp
|
||||
run: echo "NOW=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install packages
|
||||
run: sudo apt-get install libxtst-dev libpng++-dev
|
||||
|
||||
- name: Install dependencies 👨🏻💻
|
||||
run: npm ci
|
||||
|
||||
- name: E2E Performance Tests for Electron 🧪
|
||||
run: ELECTRON_DISABLE_SANDBOX=1 xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test:e2e:performance
|
||||
|
||||
- name: Upload artifact to Github
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: perf-test-report.json
|
||||
path: ${{ env.RESULTS_PATH }}
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERFORMANCE_TESTS_PUT_BUCKET }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERFORMANCE_TESTS_PUT_BUCKET }}
|
||||
aws-region: ${{ secrets.AWS_REGION_PERFORMANCE_TESTS_PUT_BUCKET }}
|
||||
mask-aws-account-id: true
|
||||
|
||||
- name: Upload report to S3
|
||||
run: aws s3 cp ${{ env.RESULTS_PATH }} s3://${{ secrets.AWS_BUCKET_PERFORMANCE_TESTS }}/${{ github.head_ref }}-${{ github.sha }}-${{ env.NOW }}.json
|
||||
|
||||
- name: Add results in PR comment
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const {generateCommentBody} = require('./e2e/utils/pr-e2e-durations-report.js');
|
||||
const fileContents = fs.readFileSync('${{ env.RESULTS_PATH }}');
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: generateCommentBody(fileContents),
|
||||
});
|
||||
|
||||
- name: Remove "Run E2E Performance Tests" label
|
||||
if: always()
|
||||
uses: actions-ecosystem/action-remove-labels@v1
|
||||
with:
|
||||
labels: |
|
||||
Run E2E Performance Tests
|
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -13,6 +13,7 @@ dist/
|
|||
mochawesome-report/
|
||||
|
||||
test-results.xml
|
||||
/e2e/performance/perf-test-report.json
|
||||
test_config.json
|
||||
.idea
|
||||
testUserData
|
||||
|
|
17
.vscode/launch.json
vendored
17
.vscode/launch.json
vendored
|
@ -49,6 +49,23 @@
|
|||
"internalConsoleOptions": "openOnSessionStart",
|
||||
"preLaunchTask": "prepare-e2e"
|
||||
},
|
||||
{
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
"name": "E2E Performance Tests",
|
||||
"program": "${workspaceRoot}/node_modules/electron-mocha/bin/electron-mocha",
|
||||
"args": [
|
||||
"-r",
|
||||
"@babel/register",
|
||||
"--recursive",
|
||||
"--timeout",
|
||||
"999999",
|
||||
"--colors",
|
||||
"${workspaceRoot}/dist/tests/e2e_bundle.js"
|
||||
],
|
||||
"internalConsoleOptions": "openOnSessionStart",
|
||||
"preLaunchTask": "prepare-e2e-performance",
|
||||
},
|
||||
{
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
|
|
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
|
@ -24,6 +24,7 @@
|
|||
"gsettings",
|
||||
"ICONNAME",
|
||||
"inputflash",
|
||||
"libxtst",
|
||||
"loadscreen",
|
||||
"mailhost",
|
||||
"mailserver",
|
||||
|
@ -37,6 +38,7 @@
|
|||
"officedocument",
|
||||
"openxmlformats",
|
||||
"presentationml",
|
||||
"servernum",
|
||||
"showunreadbadge",
|
||||
"spreadsheetml",
|
||||
"textbox",
|
||||
|
@ -45,6 +47,7 @@
|
|||
"Unreads",
|
||||
"webcontents",
|
||||
"wordprocessingml",
|
||||
"xvfb",
|
||||
"Yuya"
|
||||
],
|
||||
"i18n-ally.keystyle": "nested"
|
||||
|
|
6
.vscode/tasks.json
vendored
6
.vscode/tasks.json
vendored
|
@ -14,6 +14,12 @@
|
|||
"type": "shell",
|
||||
"command": "npm run build; npm run build-robotjs; npm run test:e2e:build",
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "prepare-e2e-performance",
|
||||
"type": "shell",
|
||||
"command": "cross-env NODE_ENV=test PERFORMANCE=true npm run build; npm run build-robotjs; npm run test:e2e:build-performance",
|
||||
"problemMatcher": []
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
29
e2e/performance/app.test.js
Normal file
29
e2e/performance/app.test.js
Normal file
|
@ -0,0 +1,29 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
'use strict';
|
||||
|
||||
const env = require('../modules/environment');
|
||||
|
||||
describe('startup/app', function desc() {
|
||||
this.timeout(30000);
|
||||
|
||||
beforeEach(async () => {
|
||||
env.createTestUserDataDir();
|
||||
env.cleanTestConfig();
|
||||
this.app = await env.getApp();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (this.app) {
|
||||
await this.app.close();
|
||||
}
|
||||
await env.clearElectronInstances();
|
||||
});
|
||||
|
||||
it('should show the welcome screen modal when no servers exist', async () => {
|
||||
const welcomeScreenModal = this.app.windows().find((window) => window.url().includes('welcomeScreen'));
|
||||
const modalButton = await welcomeScreenModal.innerText('.WelcomeScreen .WelcomeScreen__button');
|
||||
modalButton.should.equal('Get Started');
|
||||
});
|
||||
});
|
|
@ -2,7 +2,9 @@
|
|||
// See LICENSE.txt for license information.
|
||||
|
||||
const MOCHAWESOME_REPORT_DIR = './mochawesome-report';
|
||||
const PERFORMANCE_REPORT_DIR = './e2e/performance';
|
||||
|
||||
module.exports = {
|
||||
MOCHAWESOME_REPORT_DIR,
|
||||
PERFORMANCE_REPORT_DIR,
|
||||
};
|
||||
|
|
36
e2e/utils/pr-e2e-durations-report.js
Normal file
36
e2e/utils/pr-e2e-durations-report.js
Normal file
|
@ -0,0 +1,36 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
function generateCommentBody(fileContents) {
|
||||
const data = JSON.parse(fileContents);
|
||||
|
||||
return `
|
||||
E2E Performance Test results:
|
||||
|
||||
| Test | Duration |
|
||||
| --- | --- |
|
||||
${data?.passes?.reduce((acc, pass) => {
|
||||
return `${acc}| ${pass.fullTitle || 'title'} | ${pass.duration}ms |\n`;
|
||||
}, '')
|
||||
}
|
||||
|
||||
${data?.failures?.length > 0 ? `
|
||||
Some tests failed:
|
||||
| Test | Duration |
|
||||
| --- | --- |
|
||||
${data.failures.forEach((failure) => `| ${failure.fullTitle} | ${failure.duration}`)}
|
||||
` : ''}
|
||||
|
||||
<details>
|
||||
<summary>Raw results</summary>
|
||||
|
||||
\`\`\`js
|
||||
${fileContents}
|
||||
\`\`\`
|
||||
</details>
|
||||
`;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
generateCommentBody,
|
||||
};
|
|
@ -42,8 +42,11 @@
|
|||
"test": "npm-run-all lint:js test:unit test:e2e",
|
||||
"test:docker": "node scripts/setup_e2e_docker.js",
|
||||
"test:e2e": "cross-env NODE_ENV=test npm-run-all build build-robotjs test:e2e:build test:e2e:run",
|
||||
"test:e2e:performance": "cross-env NODE_ENV=test PERFORMANCE=true npm-run-all build build-robotjs test:e2e:build-performance test:e2e:run-performance",
|
||||
"test:e2e:nobuild": "cross-env NODE_ENV=test npm-run-all test:e2e:build test:e2e:run",
|
||||
"test:e2e:build": "webpack-cli --config webpack.config.test.js",
|
||||
"test:e2e:build-performance": "webpack-cli --config webpack.config.performance.test.js",
|
||||
"test:e2e:run-performance": "electron-mocha -r @babel/register --reporter json --reporter-option output=./e2e/performance/perf-test-report.json dist/tests/e2e_bundle.js",
|
||||
"test:e2e:run": "electron-mocha -r @babel/register --reporter mochawesome dist/tests/e2e_bundle.js",
|
||||
"test:e2e:send-report": "node ./e2e/save_report.js",
|
||||
"test:unit": "cross-env NODE_ENV=jest jest",
|
||||
|
|
53
webpack.config.performance.test.js
Normal file
53
webpack.config.performance.test.js
Normal file
|
@ -0,0 +1,53 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
// This file uses CommonJS.
|
||||
/* eslint-disable import/no-commonjs */
|
||||
'use strict';
|
||||
|
||||
const path = require('path');
|
||||
|
||||
const glob = require('glob');
|
||||
const {merge} = require('webpack-merge');
|
||||
|
||||
const base = require('./webpack.config.base');
|
||||
|
||||
const WEBSERVER_PORT = process.env.WEBSERVER_PORT ?? 9001;
|
||||
|
||||
module.exports = merge(base, {
|
||||
entry: {
|
||||
e2e: glob.sync('./e2e/performance/**/*.test.js'),
|
||||
},
|
||||
output: {
|
||||
path: path.resolve(__dirname, 'dist/tests'),
|
||||
filename: '[name]_bundle.js',
|
||||
},
|
||||
module: {
|
||||
rules: [{
|
||||
test: /\.(js|jsx|ts|tsx)?$/,
|
||||
use: ['babel-loader', 'shebang-loader'],
|
||||
}],
|
||||
},
|
||||
externals: {
|
||||
fs: 'require("fs")',
|
||||
ws: 'require("ws")',
|
||||
child_process: 'require("child_process")',
|
||||
dns: 'require("dns")',
|
||||
http2: 'require("http2")',
|
||||
net: 'require("net")',
|
||||
repl: 'require("repl")',
|
||||
tls: 'require("tls")',
|
||||
playwright: 'require("playwright")',
|
||||
robotjs: 'require("robotjs")',
|
||||
},
|
||||
node: {
|
||||
__filename: false,
|
||||
__dirname: false,
|
||||
},
|
||||
devServer: {
|
||||
port: WEBSERVER_PORT,
|
||||
},
|
||||
target: 'electron-main',
|
||||
});
|
||||
|
||||
/* eslint-enable import/no-commonjs */
|
Loading…
Reference in a new issue