From 7c2b4ec5020c76fdef122ff1af88661ef5191800 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Tue, 19 Mar 2024 22:18:29 +0530 Subject: [PATCH] Modify run conditions for nightly e2e job to create test cycle and post in channel. (#2971) --- .github/PULL_REQUEST_TEMPLATE.md | 1 + .github/workflows/e2e-functional.yml | 240 ++++++++++++------ .github/workflows/e2e-performance.yml | 4 +- e2e/modules/environment.js | 17 ++ e2e/save_report.js | 3 +- e2e/specs/focus.test.js | 31 ++- e2e/specs/menu_bar/edit_menu.test.js | 46 +--- e2e/specs/menu_bar/view_menu.test.js | 50 +--- e2e/specs/menu_bar/window_menu.test.js | 1 - e2e/specs/popup.test.js | 121 ++++----- .../long_server_name.test.js | 13 +- e2e/specs/settings/keyboard_shortcuts.test.js | 76 +++--- e2e/utils/analyze-flaky-test.js | 81 ++++++ e2e/utils/known_flaky_tests.json | 14 + e2e/utils/pr-e2e-durations-report.js | 4 +- e2e/utils/report.js | 1 + e2e/utils/test_cases.js | 43 +++- webpack.config.test.js | 2 +- 18 files changed, 480 insertions(+), 268 deletions(-) create mode 100644 e2e/utils/analyze-flaky-test.js create mode 100644 e2e/utils/known_flaky_tests.json diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e3e3dce9..a7c65983 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -28,6 +28,7 @@ Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fie - [ ] read and understood our [Contributing Guidelines](https://github.com/mattermost/desktop/blob/master/CONTRIBUTING.md) - [ ] completed [Mattermost Contributor Agreement](https://mattermost.com/contribute/) - [ ] executed `npm run lint:js` for proper code formatting +- [ ] Run E2E tests by adding label `Run Desktop E2E Tests` #### Device Information This PR was tested on: diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index d31f41c2..43fb8c02 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -32,28 +32,28 @@ on: env: AWS_S3_BUCKET: "mattermost-cypress-report" - BRANCH: ${{ github.ref }} - BUILD_SUFFIX: 'desktop-pr' + BRANCH: ${{ github.head_ref || github.ref_name }} + BUILD_TAG: ${{ github.sha }} JIRA_PROJECT_KEY: 'MM' MM_TEST_SERVER_URL: ${{ secrets.MM_DESKTOP_E2E_SERVER_URL }} MM_TEST_USER_NAME: ${{ secrets.MM_DESKTOP_E2E_USER_NAME }} MM_TEST_PASSWORD: ${{ secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }} - PULL_REQUEST_BASE_URL: "https://github.com/mattermost/desktop/pull/" + PULL_REQUEST: "https://github.com/mattermost/desktop/pull/${{ github.event.number }}" ZEPHYR_ENVIRONMENT_NAME: 'Desktop app' - ZEPHYR_FOLDER_ID: "3256491" + ZEPHYR_FOLDER_ID: "12413253" TEST_CYCLE_LINK_PREFIX: ${{ secrets.MM_DESKTOP_E2E_TEST_CYCLE_LINK_PREFIX }} AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }} ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }} + REPORT_LINK: "none" jobs: e2e-linux: if: ${{ ( - (inputs.job_name == 'e2e-linux' || - inputs.job_name == 'All') + (inputs.job_name == 'e2e-linux' || inputs.job_name == 'All') && github.event_name == 'workflow_dispatch' ) || @@ -65,17 +65,16 @@ jobs: github.event.pull_request.labels && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') ) || - ( - github.event_name == 'schedule' || github.event_name == 'workflow_call' - ) + ( + (inputs.tag != '') + ) }} - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - - name: ci/checkout-repo uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: ci/setup-node - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 + uses: actions/setup-node@v4 with: node-version-file: "package.json" cache: "npm" @@ -87,52 +86,86 @@ jobs: run: | wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add - wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq - sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata + sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel npm ci + npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs - name: Set Environment Variables run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-pr" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-pr-linux" >> $GITHUB_ENV echo "TYPE=PR" >> $GITHUB_ENV elif [ "${{ github.event_name }}" == "release" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-release" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-release-linux" >> $GITHUB_ENV echo "TYPE=RELEASE" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV - elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-manual-trigger" >> $GITHUB_ENV + elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then + echo "BUILD_SUFFIX=desktop-manual-trigger-linux" >> $GITHUB_ENV echo "TYPE=MANUAL" >> $GITHUB_ENV elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-master-push-linux" >> $GITHUB_ENV echo "TYPE=MASTER" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-master-push" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - elif [ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_call" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-nightly" >> $GITHUB_ENV + elif [ "${{ inputs.tag }}" != "" ]; then + echo "BUILD_SUFFIX=desktop-nightly-linux" >> $GITHUB_ENV echo "TYPE=NIGHTLY" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV fi + - name: Set Build ID + run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV + - name: Run Playwright tests (Ubuntu OS) run: | export DISPLAY=:99 - Xvfb $DISPLAY -screen 0 1024x768x24 > /dev/null 2>&1 & + Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 & npm run test:e2e || true # making job pass even if the tests fail due to flakyness npm run test:e2e:send-report + - name: Analyze flaky tests + id: analyze-flaky-tests + uses: actions/github-script@v7 + with: + script: | + const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); + const { newFailedTests } = analyzeFlakyTests(); + if (newFailedTests.length === 0) { + process.env.NEW_FAILED_TESTS = 'true'; + } + env: + NEW_FAILED_TESTS: false + + - name: Add results in PR comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); + const { commentBody } = analyzeFlakyTests(); + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody, + }); + + - name: Mark job as failed if there are new failed tests + if: env.NEW_FAILED_TESTS == 'true' + run: exit 1 + shell: bash + e2e-macos: if: ${{ ( - (inputs.job_name == 'e2e-macos' || - inputs.job_name == 'All') + (inputs.job_name == 'e2e-macos' || inputs.job_name == 'All') && github.event_name == 'workflow_dispatch' ) || @@ -144,16 +177,16 @@ jobs: github.event.pull_request.labels && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') ) || - ( - github.event_name == 'schedule' || github.event_name == 'workflow_call' - ) + ( + (inputs.tag != '') + ) }} runs-on: macos-13 steps: - name: ci/checkout-repo uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: ci/setup-node - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 + uses: actions/setup-node@v4 with: node-version-file: "package.json" cache: "npm" @@ -170,59 +203,88 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.10' - + - name: ci/install-dependencies env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 run: | + npm cache clean --force jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json . npm ci - npm install -g node-gyp - npm i robotjs - npx electron-rebuild -f -t prod,optional,dev -w robotjs + npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs - name: Set Environment Variables run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-pr" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-pr-macos" >> $GITHUB_ENV echo "TYPE=PR" >> $GITHUB_ENV elif [ "${{ github.event_name }}" == "release" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-release" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-release-macos" >> $GITHUB_ENV echo "TYPE=RELEASE" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV - elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-manual-trigger" >> $GITHUB_ENV + elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then + echo "BUILD_SUFFIX=desktop-manual-trigger-macos" >> $GITHUB_ENV echo "TYPE=MANUAL" >> $GITHUB_ENV elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-master-push-macos" >> $GITHUB_ENV echo "TYPE=MASTER" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-master-push" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - elif [ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_call" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-nightly" >> $GITHUB_ENV + elif [ "${{ inputs.tag }}" != "" ]; then + echo "BUILD_SUFFIX=desktop-nightly-macos" >> $GITHUB_ENV echo "TYPE=NIGHTLY" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV fi + - name: Set Build ID + run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV + - name: Run Playwright tests (macOS) run: | - cat $GITHUB_ENV npm run test:e2e || true # making job pass even if the tests fail due to flakyness npm run test:e2e:send-report + - name: Analyze flaky tests + id: analyze-flaky-tests + uses: actions/github-script@v7 + with: + script: | + const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); + const { newFailedTests } = analyzeFlakyTests(); + if (newFailedTests.length === 0) { + process.env.NEW_FAILED_TESTS = 'true'; + } + + - name: Add results in PR comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); + const { commentBody } = analyzeFlakyTests(); + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody, + }); + + - name: Mark job as failed if there are new failed tests + if: env.NEW_FAILED_TESTS == 'true' + run: exit 1 + shell: bash + e2e-windows: if: ${{ ( - (inputs.job_name == 'e2e-windows' || - inputs.job_name == 'All') + (inputs.job_name == 'e2e-windows' || inputs.job_name == 'All') && github.event_name == 'workflow_dispatch' ) || @@ -234,16 +296,16 @@ jobs: github.event.pull_request.labels && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') ) || - ( - github.event_name == 'schedule' || github.event_name == 'workflow_call' - ) + ( + (inputs.tag != '') + ) }} runs-on: windows-2022 steps: - name: ci/checkout-repo uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: ci/setup-node - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 + uses: actions/setup-node@v4 with: node-version-file: "package.json" cache: "npm" @@ -266,59 +328,93 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 run: | choco install yq --version 4.15.1 -y + npm install -g windows-build-tools npm i -g node-gyp node-gyp install node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64 node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch ia32 npm ci --openssl_fips='' + npm i robotjs + npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs - name: Set Environment Variables shell: bash run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "BRANCH=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-pr" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-pr-windows" >> $GITHUB_ENV echo "TYPE=PR" >> $GITHUB_ENV elif [ "${{ github.event_name }}" == "release" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-release" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-release-windows" >> $GITHUB_ENV echo "TYPE=RELEASE" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV - elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-manual-trigger" >> $GITHUB_ENV + elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then + echo "BUILD_SUFFIX=desktop-manual-trigger-windows" >> $GITHUB_ENV echo "TYPE=MANUAL" >> $GITHUB_ENV elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV echo "TYPE=MASTER" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-master-push" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-master-push-windows" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - elif [ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_call" ]; then - echo "BRANCH=${{ github.ref }}" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-nightly" >> $GITHUB_ENV + elif [ "${{ inputs.tag }}" != "" ]; then echo "TYPE=NIGHTLY" >> $GITHUB_ENV + echo "BUILD_SUFFIX=desktop-nightly-windows" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV fi - + + - name: Set Build ID + run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV + shell: bash + - name: Run Playwright tests (Windows OS) run: | - cat $GITHUB_ENV npm run test:e2e || true npm run test:e2e:send-report shell: bash + - name: Analyze flaky tests + id: analyze-flaky-tests + uses: actions/github-script@v7 + with: + script: | + const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); + const { newFailedTests } = analyzeFlakyTests(); + if (newFailedTests.length === 0) { + process.env.NEW_FAILED_TESTS = 'true'; + } + + - name: Add results in PR comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); + const { commentBody } = analyzeFlakyTests(); + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody, + }); + + - name: Mark job as failed if there are new failed tests + if: env.NEW_FAILED_TESTS == 'true' + run: exit 1 + shell: bash + e2e-remove-label: - if: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }} + if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }} needs: [e2e-linux, e2e-macos, e2e-windows] runs-on: ubuntu-latest steps: - name: Remove "Run Desktop E2E Tests" label uses: actions-ecosystem/action-remove-labels@v1 with: - labels: | - Run Desktop E2E Tests + labels: Run Desktop E2E Tests diff --git a/.github/workflows/e2e-performance.yml b/.github/workflows/e2e-performance.yml index 8ec92c29..0b24792d 100644 --- a/.github/workflows/e2e-performance.yml +++ b/.github/workflows/e2e-performance.yml @@ -76,13 +76,13 @@ jobs: with: script: | const fs = require('fs'); - const {generateCommentBody} = require('./e2e/utils/pr-e2e-durations-report.js'); + const {generateCommentBodyPerformanceTest} = require('./e2e/utils/pr-e2e-durations-report.js'); const fileContents = fs.readFileSync('${{ env.RESULTS_PATH }}'); github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: generateCommentBody(fileContents), + body: generateCommentBodyPerformanceTest(fileContents), }); - name: Remove "Run E2E Performance Tests" label diff --git a/e2e/modules/environment.js b/e2e/modules/environment.js index 5e9165b7..2bc8c238 100644 --- a/e2e/modules/environment.js +++ b/e2e/modules/environment.js @@ -3,6 +3,7 @@ // See LICENSE.txt for license information. 'use strict'; +const {execSync} = require('child_process'); const fs = require('fs'); const path = require('path'); @@ -180,6 +181,7 @@ module.exports = { } } }, + cleanDataDirAsync() { return rmDirAsync(userDataDir); }, @@ -189,6 +191,21 @@ module.exports = { fs.mkdirSync(userDataDir); } }, + + clipboard(textToCopy) { + switch (process.platform) { + case 'linux': + execSync(`echo "${textToCopy}" | xsel --clipboard`); + break; + case 'win32': + execSync(`echo ${textToCopy} | clip`); + break; + case 'darwin': + execSync(`pbcopy <<< ${textToCopy}`); + break; + } + }, + async createTestUserDataDirAsync() { await mkDirAsync(userDataDir); }, diff --git a/e2e/save_report.js b/e2e/save_report.js index ea9b2cfd..35478a1b 100644 --- a/e2e/save_report.js +++ b/e2e/save_report.js @@ -71,7 +71,8 @@ const saveReport = async () => { ); // Generate short summary, write to file and then send report via webhook - const summary = generateShortSummary(jsonReport); + const {stats, statsFieldValue} = generateShortSummary(jsonReport); + const summary = {stats, statsFieldValue}; console.log(summary); writeJsonToFile(summary, 'summary.json', MOCHAWESOME_REPORT_DIR); diff --git a/e2e/specs/focus.test.js b/e2e/specs/focus.test.js index a6446b33..82be3459 100644 --- a/e2e/specs/focus.test.js +++ b/e2e/specs/focus.test.js @@ -43,7 +43,10 @@ describe('focus', function desc() { ], }; - beforeEach(async () => { + let firstServer; + let loadingScreen; + + before(async () => { env.cleanDataDir(); env.createTestUserDataDir(); env.cleanTestConfig(); @@ -51,9 +54,16 @@ describe('focus', function desc() { await asyncSleep(1000); this.app = await env.getApp(); this.serverMap = await env.getServerMap(this.app); + + loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); + await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); + firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; + await env.loginToMattermost(firstServer); + const textbox = await firstServer.waitForSelector('#post_textbox'); + textbox.focus(); }); - afterEach(async () => { + after(async () => { if (this.app) { await this.app.close(); } @@ -61,17 +71,6 @@ describe('focus', function desc() { }); describe('Focus textbox tests', () => { - let firstServer; - - beforeEach(async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - const textbox = await firstServer.waitForSelector('#post_textbox'); - textbox.focus(); - }); - it('MM-T1315 should return focus to the message box when closing the settings window', async () => { this.app.evaluate(({ipcMain}, showWindow) => { ipcMain.emit(showWindow); @@ -85,6 +84,8 @@ describe('focus', function desc() { const isTextboxFocused = await firstServer.$eval('#post_textbox', (el) => el === document.activeElement); isTextboxFocused.should.be.true; + await firstServer.fill('#post_textbox', ''); + // Make sure you can just start typing and it'll go in the post textbox await asyncSleep(500); robot.typeString('Mattermost'); @@ -108,6 +109,8 @@ describe('focus', function desc() { const isTextboxFocused = await firstServer.$eval('#post_textbox', (el) => el === document.activeElement); isTextboxFocused.should.be.true; + await firstServer.fill('#post_textbox', ''); + // Make sure you can just start typing and it'll go in the post textbox await asyncSleep(500); robot.typeString('Mattermost'); @@ -132,6 +135,8 @@ describe('focus', function desc() { const isTextboxFocused = await firstServer.$eval('#post_textbox', (el) => el === document.activeElement); isTextboxFocused.should.be.true; + await firstServer.fill('#post_textbox', ''); + // Make sure you can just start typing and it'll go in the post textbox await asyncSleep(500); robot.typeString('Mattermost'); diff --git a/e2e/specs/menu_bar/edit_menu.test.js b/e2e/specs/menu_bar/edit_menu.test.js index fd8df6f1..39b6c6a6 100644 --- a/e2e/specs/menu_bar/edit_menu.test.js +++ b/e2e/specs/menu_bar/edit_menu.test.js @@ -13,8 +13,9 @@ describe('edit_menu', function desc() { this.timeout(40000); const config = env.demoMattermostConfig; + let firstServer; - beforeEach(async () => { + before(async () => { env.cleanDataDir(); env.createTestUserDataDir(); env.cleanTestConfig(); @@ -22,9 +23,14 @@ describe('edit_menu', function desc() { await asyncSleep(1000); this.app = await env.getApp(); this.serverMap = await env.getServerMap(this.app); + + const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); + await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); + firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; + await env.loginToMattermost(firstServer); }); - afterEach(async () => { + after(async () => { if (this.app) { await this.app.close(); } @@ -32,13 +38,9 @@ describe('edit_menu', function desc() { }); it('MM-T807 Undo in the Menu Bar', async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - // click on sint channel await firstServer.click('#post_textbox'); + await firstServer.fill('#post_textbox', ''); await firstServer.type('#post_textbox', 'Mattermost'); await firstServer.click('#post_textbox'); robot.keyTap('z', [env.cmdOrCtrl]); @@ -48,13 +50,9 @@ describe('edit_menu', function desc() { }); it('MM-T808 Redo in the Menu Bar', async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - // click on sint channel await firstServer.click('#post_textbox'); + await firstServer.fill('#post_textbox', ''); await firstServer.type('#post_textbox', 'Mattermost'); await firstServer.click('#post_textbox'); robot.keyTap('z', [env.cmdOrCtrl]); @@ -69,13 +67,9 @@ describe('edit_menu', function desc() { }); it('MM-T809 Cut in the Menu Bar', async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - // click on sint channel await firstServer.click('#post_textbox'); + await firstServer.fill('#post_textbox', ''); await firstServer.type('#post_textbox', 'Mattermost'); robot.keyTap('a', [env.cmdOrCtrl]); await asyncSleep(500); @@ -86,13 +80,9 @@ describe('edit_menu', function desc() { }); it('MM-T810 Copy in the Menu Bar', async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - // click on sint channel await firstServer.click('#post_textbox'); + await firstServer.fill('#post_textbox', ''); await firstServer.type('#post_textbox', 'Mattermost'); robot.keyTap('a', [env.cmdOrCtrl]); await asyncSleep(500); @@ -106,13 +96,9 @@ describe('edit_menu', function desc() { }); it('MM-T811 Paste in the Menu Bar', async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - // click on sint channel await firstServer.click('#post_textbox'); + await firstServer.fill('#post_textbox', ''); await firstServer.type('#post_textbox', 'Mattermost'); robot.keyTap('a', [env.cmdOrCtrl]); await asyncSleep(500); @@ -127,13 +113,9 @@ describe('edit_menu', function desc() { }); it('MM-T812 Select All in the Menu Bar', async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - // click on sint channel await firstServer.click('#post_textbox'); + await firstServer.fill('#post_textbox', ''); await firstServer.fill('#post_textbox', 'Mattermost'); robot.keyTap('a', [env.cmdOrCtrl]); await asyncSleep(500); diff --git a/e2e/specs/menu_bar/view_menu.test.js b/e2e/specs/menu_bar/view_menu.test.js index 2498b8ce..ecb60498 100644 --- a/e2e/specs/menu_bar/view_menu.test.js +++ b/e2e/specs/menu_bar/view_menu.test.js @@ -201,7 +201,7 @@ describe('menu/view', function desc() { }); }); - it('MM-T820 should open Developer Tools For Application Wrapper for main window', async () => { + it('MM-T820 should open Developer Tools For Application Wrapper for main window', async () => { const mainWindow = this.app.windows().find((window) => window.url().includes('index.html')); const browserWindow = await this.app.browserWindow(mainWindow); const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); @@ -212,46 +212,22 @@ describe('menu/view', function desc() { }); isDevToolsOpen.should.be.false; - robot.keyTap('alt'); - robot.keyTap('enter'); - robot.keyTap('v'); - robot.keyTap('d'); - robot.keyTap('enter'); - await asyncSleep(1000); + if (process.platform === 'darwin') { + // Press Command + Option + I + robot.keyTap('i', ['command', 'alt']); + await asyncSleep(3000); + } + if (process.platform === 'win32') { + robot.keyToggle('shift', 'down'); + robot.keyToggle('control', 'down'); + robot.keyTap('i'); + } + + await asyncSleep(1000); isDevToolsOpen = await browserWindow.evaluate((window) => { return window.webContents.isDevToolsOpened(); }); isDevToolsOpen.should.be.true; }); - - // TODO: Missing shortcut for macOS - if (process.platform !== 'darwin') { - it('MM-T821 should open Developer Tools For Current Server for the active tab', async () => { - const mainWindow = this.app.windows().find((window) => window.url().includes('index')); - const browserWindow = await this.app.browserWindow(mainWindow); - const webContentsId = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].webContentsId; - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - - let isDevToolsOpen = await browserWindow.evaluate((window, id) => { - return window.getBrowserViews().find((view) => view.webContents.id === id).webContents.isDevToolsOpened(); - }, webContentsId); - isDevToolsOpen.should.be.false; - - // Open Developer Tools for Current Server - robot.keyTap('alt'); - robot.keyTap('enter'); - robot.keyTap('v'); - robot.keyTap('d'); - robot.keyTap('d'); - robot.keyTap('enter'); - await asyncSleep(1000); - - isDevToolsOpen = await browserWindow.evaluate((window, id) => { - return window.getBrowserViews().find((view) => view.webContents.id === id).webContents.isDevToolsOpened(); - }, webContentsId); - isDevToolsOpen.should.be.true; - }); - } }); diff --git a/e2e/specs/menu_bar/window_menu.test.js b/e2e/specs/menu_bar/window_menu.test.js index b61ff439..6b02dd8e 100644 --- a/e2e/specs/menu_bar/window_menu.test.js +++ b/e2e/specs/menu_bar/window_menu.test.js @@ -44,7 +44,6 @@ describe('Menu/window_menu', function desc() { }; const beforeFunc = async () => { - env.cleanDataDir(); env.createTestUserDataDir(); env.cleanTestConfig(); fs.writeFileSync(env.configFilePath, JSON.stringify(config)); diff --git a/e2e/specs/popup.test.js b/e2e/specs/popup.test.js index d5fdf832..76ac63de 100644 --- a/e2e/specs/popup.test.js +++ b/e2e/specs/popup.test.js @@ -13,8 +13,10 @@ describe('popup', function desc() { this.timeout(40000); const config = env.demoMattermostConfig; + let popupWindow; + let firstServer; - beforeEach(async () => { + before(async () => { env.cleanDataDir(); env.createTestUserDataDir(); env.cleanTestConfig(); @@ -22,9 +24,28 @@ describe('popup', function desc() { await asyncSleep(1000); this.app = await env.getApp(); this.serverMap = await env.getServerMap(this.app); + + const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); + await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); + firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; + await env.loginToMattermost(firstServer); + + await firstServer.click('#post_textbox'); + await firstServer.fill('#post_textbox', ''); + await firstServer.type('#post_textbox', '/github connect '); + await firstServer.click('button[data-testid="SendMessageButton"]'); + + const githubLink = await firstServer.waitForSelector('a.theme.markdown__link:has-text("GitHub account")'); + githubLink.click(); + popupWindow = await this.app.waitForEvent('window'); + + const loginField = await popupWindow.waitForSelector('#login_field'); + await loginField.focus(); + robot.typeString('Mattermost'); + await asyncSleep(3000); }); - afterEach(async () => { + after(async () => { if (this.app) { await this.app.close(); } @@ -32,80 +53,48 @@ describe('popup', function desc() { }); // NOTE: These tests requires that the test server have the GitHub plugin configured - describe('MM-T2827 Keyboard shortcuts in popup windows', () => { - let popupWindow; + it('MM-T2827_1 should be able to select all in popup windows', async () => { + robot.keyTap('a', env.cmdOrCtrl); + await asyncSleep(1000); - beforeEach(async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - await firstServer.waitForSelector('#sidebarItem_suscipit-4'); - await firstServer.click('#sidebarItem_suscipit-4'); - await firstServer.click('#post_textbox'); - await firstServer.type('#post_textbox', '/github connect '); - await firstServer.click('button[data-testid="SendMessageButton"]'); - - const githubLink = await firstServer.waitForSelector('a.theme.markdown__link:has-text("GitHub account")'); - githubLink.click(); - popupWindow = await this.app.waitForEvent('window'); - const loginField = await popupWindow.waitForSelector('#login_field'); - await loginField.focus(); - await loginField.type('mattermost'); + const selectedText = await popupWindow.evaluate(() => { + const box = document.querySelectorAll('#login_field')[0]; + return box.value.substring(box.selectionStart, + box.selectionEnd); }); + await asyncSleep(3000); + selectedText.should.equal('Mattermost'); + }); - it('MM-T2827_1 should be able to select all in popup windows', async () => { - robot.keyTap('a', [process.platform === 'darwin' ? 'command' : 'control']); - const selectedText = await popupWindow.evaluate(() => { - const box = document.querySelectorAll('#login_field')[0]; - return box.value.substring(box.selectionStart, - box.selectionEnd); - }); - selectedText.should.equal('mattermost'); - }); + it('MM-T2827_2 should be able to cut and paste in popup windows', async () => { + await asyncSleep(1000); + const textbox = await popupWindow.waitForSelector('#login_field'); - it('MM-T2827_2 should be able to cut and paste in popup windows', async () => { - const textbox = await popupWindow.waitForSelector('#login_field'); + await textbox.selectText({force: true}); + robot.keyTap('x', env.cmdOrCtrl); + let textValue = await textbox.inputValue(); + textValue.should.equal(''); - await textbox.selectText({force: true}); - robot.keyTap('x', [process.platform === 'darwin' ? 'command' : 'control']); - let textValue = await textbox.inputValue(); - textValue.should.equal(''); + await textbox.focus(); + robot.keyTap('v', env.cmdOrCtrl); + textValue = await textbox.inputValue(); + textValue.should.equal('Mattermost'); + }); - await textbox.focus(); - robot.keyTap('v', [process.platform === 'darwin' ? 'command' : 'control']); - textValue = await textbox.inputValue(); - textValue.should.equal('mattermost'); - }); + it('MM-T2827_3 should be able to copy and paste in popup windows', async () => { + await asyncSleep(1000); + const textbox = await popupWindow.waitForSelector('#login_field'); - it('MM-T2827_3 should be able to copy and paste in popup windows', async () => { - const textbox = await popupWindow.waitForSelector('#login_field'); - - await textbox.selectText({force: true}); - robot.keyTap('c', [process.platform === 'darwin' ? 'command' : 'control']); - await textbox.focus(); - await textbox.type('other-text'); - robot.keyTap('v', [process.platform === 'darwin' ? 'command' : 'control']); - const textValue = await textbox.inputValue(); - textValue.should.equal('other-textmattermost'); - }); + await textbox.selectText({force: true}); + robot.keyTap('c', env.cmdOrCtrl); + await textbox.focus(); + await textbox.type('other-text'); + robot.keyTap('v', env.cmdOrCtrl); + const textValue = await textbox.inputValue(); + textValue.should.equal('other-textMattermost'); }); it('MM-T1659 should not be able to go Back or Forward in the popup window', async () => { - const loadingScreen = this.app.windows().find((window) => window.url().includes('loadingScreen')); - await loadingScreen.waitForSelector('.LoadingScreen', {state: 'hidden'}); - const firstServer = this.serverMap[`${config.teams[0].name}___TAB_MESSAGING`].win; - await env.loginToMattermost(firstServer); - await firstServer.waitForSelector('#sidebarItem_suscipit-4'); - await firstServer.click('#sidebarItem_suscipit-4'); - await firstServer.click('#post_textbox'); - await firstServer.type('#post_textbox', '/github connect '); - await firstServer.click('button[data-testid="SendMessageButton"]'); - - const githubLink = await firstServer.waitForSelector('a.theme.markdown__link:has-text("GitHub account")'); - githubLink.click(); - const popupWindow = await this.app.waitForEvent('window'); - await popupWindow.bringToFront(); const currentURL = popupWindow.url(); // Try and go back diff --git a/e2e/specs/server_management/long_server_name.test.js b/e2e/specs/server_management/long_server_name.test.js index 61417174..dc0a0e0f 100644 --- a/e2e/specs/server_management/long_server_name.test.js +++ b/e2e/specs/server_management/long_server_name.test.js @@ -13,6 +13,7 @@ describe('LongServerName', function desc() { const config = env.demoConfig; const longServerName = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus quis malesuada dolor, vel scelerisque sem'; const longServerUrl = 'https://example.org'; + let newServerView; beforeEach(async () => { env.createTestUserDataDir(); @@ -31,7 +32,7 @@ describe('LongServerName', function desc() { }); // wait for autofocus to finish - await asyncSleep(500); + await asyncSleep(1000); }); afterEach(async () => { @@ -41,26 +42,24 @@ describe('LongServerName', function desc() { await env.clearElectronInstances(); }); - let newServerView; - it('MM-T4050 Long server name', async () => { await newServerView.type('#serverNameInput', longServerName); await newServerView.type('#serverUrlInput', longServerUrl); await newServerView.click('#saveNewServerModal'); await asyncSleep(1000); - const existing = Boolean(await this.app.windows().find((window) => window.url().includes('newServer'))); + const existing = Boolean(this.app.windows().find((window) => window.url().includes('newServer'))); existing.should.be.false; const mainView = this.app.windows().find((window) => window.url().includes('index')); const dropdownView = this.app.windows().find((window) => window.url().includes('dropdown')); - const isServerTabExists = Boolean(await mainView.locator(`text=${longServerName}`)); - const isServerAddedDropdown = Boolean(await dropdownView.locator(`text=${longServerName}`)); + const isServerTabExists = Boolean(mainView.locator(`text=${longServerName}`)); + const isServerAddedDropdown = Boolean(dropdownView.locator(`text=${longServerName}`)); isServerTabExists.should.be.true; isServerAddedDropdown.should.be.true; - const serverNameLocator = await mainView.locator(`text=${longServerName}`); + const serverNameLocator = mainView.locator(`text=${longServerName}`); const isTruncated = await serverNameLocator.evaluate((element) => { return element.offsetWidth < element.scrollWidth; diff --git a/e2e/specs/settings/keyboard_shortcuts.test.js b/e2e/specs/settings/keyboard_shortcuts.test.js index d4edc926..ce4854e6 100644 --- a/e2e/specs/settings/keyboard_shortcuts.test.js +++ b/e2e/specs/settings/keyboard_shortcuts.test.js @@ -14,16 +14,28 @@ const {asyncSleep} = require('../../modules/utils'); describe('settings/keyboard_shortcuts', function desc() { this.timeout(30000); const config = env.demoConfig; + let settingsWindow; - beforeEach(async () => { + before(async () => { env.createTestUserDataDir(); env.cleanTestConfig(); fs.writeFileSync(env.configFilePath, JSON.stringify(config)); await asyncSleep(1000); this.app = await env.getApp(); + + this.app.evaluate(({ipcMain}, showWindow) => { + ipcMain.emit(showWindow); + }, SHOW_SETTINGS_WINDOW); + settingsWindow = await this.app.waitForEvent('window', { + predicate: (window) => window.url().includes('settings'), + }); + await settingsWindow.waitForSelector('.settingsPage.container'); + + const textbox = await settingsWindow.waitForSelector('#inputSpellCheckerLocalesDropdown'); + await textbox.scrollIntoViewIfNeeded(); }); - afterEach(async () => { + after(async () => { if (this.app) { await this.app.close(); } @@ -31,57 +43,57 @@ describe('settings/keyboard_shortcuts', function desc() { }); describe('MM-T1288 Manipulating Text', () => { - let settingsWindow; - - beforeEach(async () => { - this.app.evaluate(({ipcMain}, showWindow) => { - ipcMain.emit(showWindow); - }, SHOW_SETTINGS_WINDOW); - settingsWindow = await this.app.waitForEvent('window', { - predicate: (window) => window.url().includes('settings'), - }); - await settingsWindow.waitForSelector('.settingsPage.container'); - - const textbox = await settingsWindow.waitForSelector('#inputSpellCheckerLocalesDropdown'); - await textbox.scrollIntoViewIfNeeded(); - await textbox.type('mattermost'); - }); - - it('MM-T1288_1 should be able to select all in the settings window', async () => { + it('MM-T1288_1 should be able to select and deselect language in the settings window', async () => { + let textboxString; await settingsWindow.click('#inputSpellCheckerLocalesDropdown'); - robot.keyTap('a', [process.platform === 'darwin' ? 'command' : 'control']); - const selectedText = await settingsWindow.evaluate(() => { - const box = document.querySelectorAll('#inputSpellCheckerLocalesDropdown')[0]; - return box.value.substring(box.selectionStart, - box.selectionEnd); - }); - selectedText.should.equal('mattermost'); + await settingsWindow.type('#inputSpellCheckerLocalesDropdown', 'Afrikaans'); + robot.keyTap('tab'); + + await settingsWindow.isVisible('#appOptionsSaveIndicator'); + + textboxString = await settingsWindow.innerText('div.SettingsPage__spellCheckerLocalesDropdown__multi-value__label'); + textboxString.should.equal('Afrikaans'); + + await settingsWindow.isVisible('#appOptionsSaveIndicator'); + + await settingsWindow.click('[aria-label="Remove Afrikaans"]'); + + await settingsWindow.isVisible('#appOptionsSaveIndicator'); + + textboxString = await settingsWindow.inputValue('#inputSpellCheckerLocalesDropdown'); + textboxString.should.equal(''); }); it('MM-T1288_2 should be able to cut and paste in the settings window', async () => { + const textToCopy = 'Afrikaans'; + env.clipboard(textToCopy); + const textbox = await settingsWindow.waitForSelector('#inputSpellCheckerLocalesDropdown'); await textbox.selectText({force: true}); - robot.keyTap('x', [process.platform === 'darwin' ? 'command' : 'control']); + robot.keyTap('x', [env.cmdOrCtrl]); let textValue = await textbox.getAttribute('value'); textValue.should.equal(''); await textbox.focus(); - robot.keyTap('v', [process.platform === 'darwin' ? 'command' : 'control']); + robot.keyTap('v', [env.cmdOrCtrl]); textValue = await textbox.getAttribute('value'); - textValue.should.equal('mattermost'); + textValue.trim().should.equal('Afrikaans'); }); it('MM-T1288_3 should be able to copy and paste in the settings window', async () => { + const textToCopy = 'Afrikaans'; + env.clipboard(textToCopy); + const textbox = await settingsWindow.waitForSelector('#inputSpellCheckerLocalesDropdown'); await textbox.selectText({force: true}); - robot.keyTap('c', [process.platform === 'darwin' ? 'command' : 'control']); + robot.keyTap('c', [env.cmdOrCtrl]); await textbox.focus(); await textbox.type('other-text'); - robot.keyTap('v', [process.platform === 'darwin' ? 'command' : 'control']); + robot.keyTap('v', [env.cmdOrCtrl]); const textValue = await textbox.getAttribute('value'); - textValue.should.equal('other-textmattermost'); + textValue.trim().should.equal('other-textAfrikaans'); }); }); }); diff --git a/e2e/utils/analyze-flaky-test.js b/e2e/utils/analyze-flaky-test.js new file mode 100644 index 00000000..f06b2d97 --- /dev/null +++ b/e2e/utils/analyze-flaky-test.js @@ -0,0 +1,81 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +/* eslint-disable no-console */ +const path = require('path'); + +const {MOCHAWESOME_REPORT_DIR} = require('./constants'); +const knownFlakyTests = require('./known_flaky_tests.json'); +const { + generateShortSummary, + readJsonFromFile, +} = require('./report'); + +function analyzeFlakyTests() { + const os = process.platform; + try { + // Import + const jsonReport = readJsonFromFile(path.join(MOCHAWESOME_REPORT_DIR, 'mochawesome.json')); + + const {failedFullTitles} = generateShortSummary(jsonReport); + + // Get the list of known flaky tests for the provided operating system + const knownFlakyTestsForOS = new Set(knownFlakyTests[os] || []); + + // Filter out the known flaky tests from the failed test titles + const newFailedTests = failedFullTitles.filter((test) => !knownFlakyTestsForOS.has(test)); + + // Check if any known failed tests are fixed + const fixedTests = [...knownFlakyTestsForOS].filter((test) => !failedFullTitles.includes(test)); + + const commentBody = generateCommentBodyFunctionalTest(newFailedTests, fixedTests); + + // Print on CI + console.log(commentBody); + + return {commentBody, newFailedTests}; + } catch (error) { + console.error('Error analyzing failures:', error); + } +} + +function generateCommentBodyFunctionalTest(newFailedTests, fixedTests) { + const osName = process.env.RUNNER_OS; + const build = process.env.BUILD_TAG; + + let commentBody = ` + ## Test Summary for ${osName} on commit ${build} +`; + + if (newFailedTests.length === 0 && fixedTests.length === 0) { + commentBody += ` + All stable tests passed on ${osName}. + `; + return commentBody; + } + + if (newFailedTests.length > 0) { + const newTestFailure = `New failed tests found on ${osName}:\n\t${newFailedTests.map((test) => `- ${test}`).join('\n\t')}`; + commentBody += ` + ### New Failed Tests + + | Test | + | --- | + ${newTestFailure} + `; + } + + if (fixedTests.length > 0) { + const fixedTestMessage = `The following known failed tests have been fixed on ${osName}:\n\t${fixedTests.map((test) => `- ${test}`).join('\n\t')}`; + commentBody += ` + ### Fixed Tests + + ${fixedTestMessage} + `; + } + + return commentBody; +} + +module.exports = { + analyzeFlakyTests, +}; diff --git a/e2e/utils/known_flaky_tests.json b/e2e/utils/known_flaky_tests.json new file mode 100644 index 00000000..46317a6f --- /dev/null +++ b/e2e/utils/known_flaky_tests.json @@ -0,0 +1,14 @@ +{ + "darwin": [ + "popup MM-T2827_1 should be able to select all in popup windows" + ], + "linux": [ + "menu/view MM-T820 should open Developer Tools For Application Wrapper for main window", + "Menu/window_menu MM-T824 should be minimized when keyboard shortcuts are pressed", + "Menu/window_menu MM-T825 should be hidden when keyboard shortcuts are pressed", + "header MM-T2637 Double-Clicking on the header should minimize/maximize the app MM-T2637_1 should maximize on double-clicking the header" + ], + "win32": [ + "application MM-T1304/MM-T1306 should open the app on the requested deep link" + ] +} \ No newline at end of file diff --git a/e2e/utils/pr-e2e-durations-report.js b/e2e/utils/pr-e2e-durations-report.js index 55f8af55..75b78a09 100644 --- a/e2e/utils/pr-e2e-durations-report.js +++ b/e2e/utils/pr-e2e-durations-report.js @@ -1,7 +1,7 @@ // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -function generateCommentBody(fileContents) { +function generateCommentBodyPerformanceTest(fileContents) { const data = JSON.parse(fileContents); return ` @@ -32,5 +32,5 @@ ${fileContents} } module.exports = { - generateCommentBody, + generateCommentBodyPerformanceTest, }; diff --git a/e2e/utils/report.js b/e2e/utils/report.js index d1f0cc5c..4152f199 100644 --- a/e2e/utils/report.js +++ b/e2e/utils/report.js @@ -76,6 +76,7 @@ function generateShortSummary(report) { return { stats, statsFieldValue, + failedFullTitles, }; } diff --git a/e2e/utils/test_cases.js b/e2e/utils/test_cases.js index 561e5658..4fb7fef8 100644 --- a/e2e/utils/test_cases.js +++ b/e2e/utils/test_cases.js @@ -5,6 +5,8 @@ // See reference: https://support.smartbear.com/tm4j-cloud/api-docs/ +const os = require('os'); + const axios = require('axios'); const chalk = require('chalk'); @@ -83,13 +85,50 @@ function saveToEndpoint(url, data) { }); } +async function getZEPHYRFolderID() { + const { + TYPE, + ZEPHYR_FOLDER_ID, + ZEPHYR_FOLDER_LINUX_REPORT, + ZEPHYR_FOLDER_MACOS_REPORT, + ZEPHYR_FOLDER_WIN_REPORT, + } = process.env; + if (TYPE === 'MASTER') { + return ZEPHYR_FOLDER_ID; + } + const platform = os.platform(); + + // Define Zephyr folder IDs for different run types and platforms. + // For PR we dont generate reports. + // Post Merge to master branch, default folderID will be used. + const folderIDs = { + RELEASE: { + darwin: ZEPHYR_FOLDER_MACOS_REPORT, + win32: ZEPHYR_FOLDER_WIN_REPORT, + linux: ZEPHYR_FOLDER_LINUX_REPORT, + default: ZEPHYR_FOLDER_ID, + }, + NIGHTLY: { + darwin: ZEPHYR_FOLDER_MACOS_REPORT, + win32: ZEPHYR_FOLDER_WIN_REPORT, + linux: ZEPHYR_FOLDER_LINUX_REPORT, + default: ZEPHYR_FOLDER_ID, + }, + }; + + // Get the folder ID based on the type and platform + const typeFolderIDs = folderIDs[TYPE]; + const folderID = typeFolderIDs?.[platform] ?? typeFolderIDs?.default ?? ZEPHYR_FOLDER_ID; + + return folderID; +} + async function createTestCycle(startDate, endDate) { const { BRANCH, BUILD_ID, JIRA_PROJECT_KEY, ZEPHYR_CYCLE_NAME, - ZEPHYR_FOLDER_ID, } = process.env; const testCycle = { @@ -99,7 +138,7 @@ async function createTestCycle(startDate, endDate) { plannedStartDate: startDate, plannedEndDate: endDate, statusName: 'Done', - folderId: ZEPHYR_FOLDER_ID, + folderId: await getZEPHYRFolderID(), }; const response = await saveToEndpoint('https://api.zephyrscale.smartbear.com/v2/testcycles', testCycle); diff --git a/webpack.config.test.js b/webpack.config.test.js index 680d1a7a..6693783e 100644 --- a/webpack.config.test.js +++ b/webpack.config.test.js @@ -10,7 +10,7 @@ const base = require('./webpack.config.base'); module.exports = merge(base, { entry: { - e2e: glob.sync('./e2e/specs/**/*.test.js'), + e2e: glob.sync('./e2e/specs/**/*.js'), }, output: { path: path.resolve(__dirname, 'dist/tests'),