diff --git a/.circleci/config.yml b/.circleci/config.yml index 8bc1c0d8..3748e257 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,7 +51,7 @@ commands: type: string default: "https://mattermost.com/wp-content/uploads/2022/02/icon.png" steps: - - run: + - run: command: | export VERSION=$(jq -r .version package.json) echo '{}' | jq "{ @@ -108,7 +108,7 @@ commands: type: string default: "./linux/" steps: - - run: + - run: name: npn run command: npm run package:<< parameters.os >> no_output_timeout: 60m @@ -123,7 +123,7 @@ commands: description: "Running automated tests" steps: - run: npm run check-types - - run: + - run: name: i18n check command: | cp i18n/en.json /tmp/en.json @@ -139,7 +139,7 @@ commands: description: "Running automated tests" steps: - run: npm run check-types - - run: + - run: name: i18n check command: | cp .\i18n\en.json $env:TEMP\en.json @@ -459,7 +459,7 @@ jobs: grep "win" ./templist.txt | awk -F/ '{print "- ["$NF"]("$0")"}' >> ./build/linklist.txt - persist_to_workspace: root: ./build - paths: + paths: - ./linklist.txt - ./artifactlist.txt @@ -477,7 +477,7 @@ jobs: command: | echo "Links for $(date +"%b-%d-%Y")" >> ./links/linklist.txt cat ./build/linklist.txt >> ./links/linklist.txt - - run: + - run: command: | linklist=$(<./links/linklist.txt); echo '{}' | jq "{ @@ -500,7 +500,7 @@ jobs: command: | if [ -z `git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null` ]; then circleci-agent step halt - fi + fi - run: name: "Setup files for aws-s3" command: | @@ -510,7 +510,7 @@ jobs: from: ./aws-s3-dist/ to: s3://releases.mattermost.com/desktop/ arguments: --acl public-read --cache-control "no-cache" --recursive - + upload_to_s3_daily: executor: aws steps: @@ -547,7 +547,7 @@ jobs: command: | if [ -z `git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null` ]; then circleci-agent step halt - fi + fi - run: name: "Setup files for ghr" command: | @@ -578,7 +578,7 @@ jobs: apt_opts: "--no-install-recommends jq" - notify: message: "Release process for the desktop app has started, it should take about 30 minutes to complete." - + end-notification: executor: wine-chrome steps: @@ -608,7 +608,6 @@ workflows: ignore: - /^build-pr-.*/ - nightly - - build-win-no-installer: filters: branches: @@ -742,11 +741,11 @@ workflows: when: << pipeline.parameters.run_nightly >> jobs: - msi_installer: - context: + context: - windows-codesign - desktop-rainforest-build - mac_installer: - context: + context: - codesign-certificates - desktop-rainforest-build - upload_to_s3_daily: diff --git a/.github/workflows/e2e-performance.yml b/.github/workflows/e2e-performance.yml new file mode 100644 index 00000000..8ec92c29 --- /dev/null +++ b/.github/workflows/e2e-performance.yml @@ -0,0 +1,93 @@ +name: E2E Performance Tests (Desktop) + +on: + pull_request: + branches: [ master ] + types: + - labeled + +env: + RESULTS_PATH: e2e/performance/perf-test-report.json + +jobs: + build: + if: ${{ github.event.label.name == 'Run E2E Performance Tests' }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest] + node-version: [16] + + steps: + + - name: Add start comment + uses: actions/github-script@v6 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `E2E Performance Tests started ๐ŸŽ๏ธ`, + }); + + - name: Set env variable for timestamp + run: echo "NOW=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV + + - uses: actions/checkout@v2 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install packages + run: sudo apt-get install libxtst-dev libpng++-dev + + - name: Install dependencies ๐Ÿ‘จ๐Ÿปโ€๐Ÿ’ป + run: npm ci + + - name: E2E Performance Tests for Electron ๐Ÿงช + run: ELECTRON_DISABLE_SANDBOX=1 xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test:e2e:performance + + - name: Upload artifact to Github + uses: actions/upload-artifact@v3 + with: + name: perf-test-report.json + path: ${{ env.RESULTS_PATH }} + if-no-files-found: error + retention-days: 14 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERFORMANCE_TESTS_PUT_BUCKET }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERFORMANCE_TESTS_PUT_BUCKET }} + aws-region: ${{ secrets.AWS_REGION_PERFORMANCE_TESTS_PUT_BUCKET }} + mask-aws-account-id: true + + - name: Upload report to S3 + run: aws s3 cp ${{ env.RESULTS_PATH }} s3://${{ secrets.AWS_BUCKET_PERFORMANCE_TESTS }}/${{ github.head_ref }}-${{ github.sha }}-${{ env.NOW }}.json + + - name: Add results in PR comment + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const {generateCommentBody} = require('./e2e/utils/pr-e2e-durations-report.js'); + const fileContents = fs.readFileSync('${{ env.RESULTS_PATH }}'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: generateCommentBody(fileContents), + }); + + - name: Remove "Run E2E Performance Tests" label + if: always() + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: | + Run E2E Performance Tests diff --git a/.gitignore b/.gitignore index b10e1cff..fc93d6fe 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ dist/ mochawesome-report/ test-results.xml +/e2e/performance/perf-test-report.json test_config.json .idea testUserData diff --git a/.vscode/launch.json b/.vscode/launch.json index 92feb1e9..a2fbae16 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -49,6 +49,23 @@ "internalConsoleOptions": "openOnSessionStart", "preLaunchTask": "prepare-e2e" }, + { + "type": "node", + "request": "launch", + "name": "E2E Performance Tests", + "program": "${workspaceRoot}/node_modules/electron-mocha/bin/electron-mocha", + "args": [ + "-r", + "@babel/register", + "--recursive", + "--timeout", + "999999", + "--colors", + "${workspaceRoot}/dist/tests/e2e_bundle.js" + ], + "internalConsoleOptions": "openOnSessionStart", + "preLaunchTask": "prepare-e2e-performance", + }, { "type": "node", "request": "launch", diff --git a/.vscode/settings.json b/.vscode/settings.json index adee569c..d932621c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,6 +24,7 @@ "gsettings", "ICONNAME", "inputflash", + "libxtst", "loadscreen", "mailhost", "mailserver", @@ -37,6 +38,7 @@ "officedocument", "openxmlformats", "presentationml", + "servernum", "showunreadbadge", "spreadsheetml", "textbox", @@ -45,6 +47,7 @@ "Unreads", "webcontents", "wordprocessingml", + "xvfb", "Yuya" ], "i18n-ally.keystyle": "nested" diff --git a/.vscode/tasks.json b/.vscode/tasks.json index e15b0793..17d2d6e0 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -14,6 +14,12 @@ "type": "shell", "command": "npm run build; npm run build-robotjs; npm run test:e2e:build", "problemMatcher": [] + }, + { + "label": "prepare-e2e-performance", + "type": "shell", + "command": "cross-env NODE_ENV=test PERFORMANCE=true npm run build; npm run build-robotjs; npm run test:e2e:build-performance", + "problemMatcher": [] } ] } diff --git a/e2e/performance/app.test.js b/e2e/performance/app.test.js new file mode 100644 index 00000000..bd3742dd --- /dev/null +++ b/e2e/performance/app.test.js @@ -0,0 +1,29 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +'use strict'; + +const env = require('../modules/environment'); + +describe('startup/app', function desc() { + this.timeout(30000); + + beforeEach(async () => { + env.createTestUserDataDir(); + env.cleanTestConfig(); + this.app = await env.getApp(); + }); + + afterEach(async () => { + if (this.app) { + await this.app.close(); + } + await env.clearElectronInstances(); + }); + + it('should show the welcome screen modal when no servers exist', async () => { + const welcomeScreenModal = this.app.windows().find((window) => window.url().includes('welcomeScreen')); + const modalButton = await welcomeScreenModal.innerText('.WelcomeScreen .WelcomeScreen__button'); + modalButton.should.equal('Get Started'); + }); +}); diff --git a/e2e/utils/constants.js b/e2e/utils/constants.js index 89ada713..44dcd69c 100644 --- a/e2e/utils/constants.js +++ b/e2e/utils/constants.js @@ -2,7 +2,9 @@ // See LICENSE.txt for license information. const MOCHAWESOME_REPORT_DIR = './mochawesome-report'; +const PERFORMANCE_REPORT_DIR = './e2e/performance'; module.exports = { MOCHAWESOME_REPORT_DIR, + PERFORMANCE_REPORT_DIR, }; diff --git a/e2e/utils/pr-e2e-durations-report.js b/e2e/utils/pr-e2e-durations-report.js new file mode 100644 index 00000000..55f8af55 --- /dev/null +++ b/e2e/utils/pr-e2e-durations-report.js @@ -0,0 +1,36 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +function generateCommentBody(fileContents) { + const data = JSON.parse(fileContents); + + return ` +E2E Performance Test results: + +| Test | Duration | +| --- | --- | +${data?.passes?.reduce((acc, pass) => { + return `${acc}| ${pass.fullTitle || 'title'} | ${pass.duration}ms |\n`; + }, '') +} + +${data?.failures?.length > 0 ? ` + Some tests failed: + | Test | Duration | + | --- | --- | + ${data.failures.forEach((failure) => `| ${failure.fullTitle} | ${failure.duration}`)} +` : ''} + +
+Raw results + +\`\`\`js +${fileContents} +\`\`\` +
+`; +} + +module.exports = { + generateCommentBody, +}; diff --git a/package.json b/package.json index 817ff03e..7736c13c 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,11 @@ "test": "npm-run-all lint:js test:unit test:e2e", "test:docker": "node scripts/setup_e2e_docker.js", "test:e2e": "cross-env NODE_ENV=test npm-run-all build build-robotjs test:e2e:build test:e2e:run", + "test:e2e:performance": "cross-env NODE_ENV=test PERFORMANCE=true npm-run-all build build-robotjs test:e2e:build-performance test:e2e:run-performance", "test:e2e:nobuild": "cross-env NODE_ENV=test npm-run-all test:e2e:build test:e2e:run", "test:e2e:build": "webpack-cli --config webpack.config.test.js", + "test:e2e:build-performance": "webpack-cli --config webpack.config.performance.test.js", + "test:e2e:run-performance": "electron-mocha -r @babel/register --reporter json --reporter-option output=./e2e/performance/perf-test-report.json dist/tests/e2e_bundle.js", "test:e2e:run": "electron-mocha -r @babel/register --reporter mochawesome dist/tests/e2e_bundle.js", "test:e2e:send-report": "node ./e2e/save_report.js", "test:unit": "cross-env NODE_ENV=jest jest", diff --git a/webpack.config.performance.test.js b/webpack.config.performance.test.js new file mode 100644 index 00000000..ee5a18bc --- /dev/null +++ b/webpack.config.performance.test.js @@ -0,0 +1,53 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +// This file uses CommonJS. +/* eslint-disable import/no-commonjs */ +'use strict'; + +const path = require('path'); + +const glob = require('glob'); +const {merge} = require('webpack-merge'); + +const base = require('./webpack.config.base'); + +const WEBSERVER_PORT = process.env.WEBSERVER_PORT ?? 9001; + +module.exports = merge(base, { + entry: { + e2e: glob.sync('./e2e/performance/**/*.test.js'), + }, + output: { + path: path.resolve(__dirname, 'dist/tests'), + filename: '[name]_bundle.js', + }, + module: { + rules: [{ + test: /\.(js|jsx|ts|tsx)?$/, + use: ['babel-loader', 'shebang-loader'], + }], + }, + externals: { + fs: 'require("fs")', + ws: 'require("ws")', + child_process: 'require("child_process")', + dns: 'require("dns")', + http2: 'require("http2")', + net: 'require("net")', + repl: 'require("repl")', + tls: 'require("tls")', + playwright: 'require("playwright")', + robotjs: 'require("robotjs")', + }, + node: { + __filename: false, + __dirname: false, + }, + devServer: { + port: WEBSERVER_PORT, + }, + target: 'electron-main', +}); + +/* eslint-enable import/no-commonjs */