diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 26be36a0..5abb608f 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -96,6 +96,7 @@ jobs: cmt: true MM_TEST_SERVER_URL: ${{ matrix.server.url }} DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} + MM_SERVER_VERSION: ${{ matrix.server.version }} # We need to duplicate here in order to set the proper commit status # https://mattermost.atlassian.net/browse/CLD-5815 diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index afd39ac8..c1aa25b7 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -35,6 +35,10 @@ on: description: "True if this is Compatibility Matrix Testing" required: false default: false + MM_SERVER_VERSION: + type: string + required: false + default: "9.9.1" outputs: COMMENT_BODY: description: "The output to comment" @@ -77,9 +81,9 @@ jobs: echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT echo "TYPE=PR" >> $GITHUB_ENV - elif [ "${{ github.event_name }}" == "release" ]; then + elif [ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.cmt }}" ]; then echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT - echo "TYPE=RELEASE" >> $GITHUB_ENV + echo "TYPE=CMT" >> $GITHUB_ENV echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV @@ -148,8 +152,7 @@ jobs: npm ci cd e2e npm ci - npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs - cd ../ + npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs --module-dir ../ - name: e2e/install-dependencies-macos if: runner.os == 'macOS' @@ -160,8 +163,7 @@ jobs: npm ci cd e2e npm ci - npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs - cd ../ + npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs --module-dir ../ ## Windows Dependencies - name: e2e/install-dependencies-windows @@ -177,8 +179,7 @@ jobs: npm ci --openssl_fips='' cd e2e npm ci - npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs - cd ../ + npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs --module-dir ../ - name: e2e/run-playright-tests-${{ runner.os }} run: | @@ -186,9 +187,13 @@ jobs: export DISPLAY=:99 Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 & fi - npm run e2e || true # making job pass even if the tests fail due to flakyness + npm run build-test cd e2e + npm run run:e2e || true # making job pass even if the tests fail due to flakyness npm run send-report + env: + SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION}} + DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} - name: e2e/analyze-flaky-tests id: analyze-flaky-tests diff --git a/e2e/package.json b/e2e/package.json index 0111f81f..0b292ca6 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -5,7 +5,7 @@ "main": "dist/e2e_bundle.js", "scripts": { "clean": "rm -rf dist/ mochawesome-report/ node_modules/ testUserData/", - "run": "npm run build && npm run test", + "run:e2e": "npm run build && npm run test", "build": "webpack-cli --config webpack.config.js", "build:performance": "webpack-cli --config webpack.config.performance.js", "test": "electron-mocha --reporter mochawesome dist/e2e_bundle.js", diff --git a/e2e/specs/startup/welcome_screen_modal.test.js b/e2e/specs/startup/welcome_screen_modal.test.js index 297319d9..018d7c64 100644 --- a/e2e/specs/startup/welcome_screen_modal.test.js +++ b/e2e/specs/startup/welcome_screen_modal.test.js @@ -31,21 +31,27 @@ describe('Welcome Screen Modal', function desc() { it('MM-T4976 should show the slides in the expected order', async () => { const welcomeSlideClass = await welcomeScreenModal.getAttribute('#welcome', 'class'); welcomeSlideClass.should.contain('Carousel__slide-current'); - + const welcomeSlideTitle = await welcomeScreenModal.innerText('#welcome .WelcomeScreenSlide__title'); + welcomeSlideTitle.should.equal('Welcome'); await welcomeScreenModal.click('#nextCarouselButton'); - const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class'); + const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); channelSlideClass.should.contain('Carousel__slide-current'); - + const channelSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title'); + channelSlideTitle.should.equal('Collaborate in real time'); await welcomeScreenModal.click('#nextCarouselButton'); - const playbooksSlideClass = await welcomeScreenModal.getAttribute('#playbooks', 'class'); - playbooksSlideClass.should.contain('Carousel__slide-current'); - + const callsSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); + callsSlideClass.should.contain('Carousel__slide-current'); + const callsSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title'); + callsSlideTitle.should.equal('Start secure calls instantly'); await welcomeScreenModal.click('#nextCarouselButton'); - const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class'); - boardsSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); + integrationSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title'); + integrationSlideTitle.should.equal('Integrate with tools you love'); + await welcomeScreenModal.click('#nextCarouselButton'); }); it('MM-T4977 should be able to move through slides clicking the navigation buttons', async () => { @@ -54,7 +60,7 @@ describe('Welcome Screen Modal', function desc() { await welcomeScreenModal.click('#nextCarouselButton'); - const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class'); + const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); channelSlideClass.should.contain('Carousel__slide-current'); await welcomeScreenModal.click('#prevCarouselButton'); @@ -69,13 +75,13 @@ describe('Welcome Screen Modal', function desc() { await welcomeScreenModal.click('#PaginationIndicator3'); - const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class'); - boardsSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); + integrationSlideClass.should.contain('Carousel__slide-current'); await welcomeScreenModal.click('#PaginationIndicator2'); - const playbooksSlideClass = await welcomeScreenModal.getAttribute('#playbooks', 'class'); - playbooksSlideClass.should.contain('Carousel__slide-current'); + const callsSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromLeft', 'class'); + callsSlideClass.should.contain('Carousel__slide-current'); }); it('MM-T4979 should be able to move forward through slides automatically every 5 seconds', async () => { @@ -84,8 +90,11 @@ describe('Welcome Screen Modal', function desc() { await asyncSleep(5500); - const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class'); + const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); channelSlideClass.should.contain('Carousel__slide-current'); + const channelSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title'); + channelSlideTitle.should.equal('Collaborate in real time'); + await welcomeScreenModal.click('#nextCarouselButton'); }); it('MM-T4980 should show the slides in the expected order', async () => { @@ -94,25 +103,27 @@ describe('Welcome Screen Modal', function desc() { await welcomeScreenModal.click('#nextCarouselButton'); - const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class'); + const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); channelSlideClass.should.contain('Carousel__slide-current'); await welcomeScreenModal.click('#nextCarouselButton'); - const playbooksSlideClass = await welcomeScreenModal.getAttribute('#playbooks', 'class'); - playbooksSlideClass.should.contain('Carousel__slide-current'); + const callsSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); + callsSlideClass.should.contain('Carousel__slide-current'); await welcomeScreenModal.click('#nextCarouselButton'); - const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class'); - boardsSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); + integrationSlideClass.should.contain('Carousel__slide-current'); }); it('MM-T4981 should be able to move from last to first slide', async () => { await welcomeScreenModal.click('#PaginationIndicator3'); - const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class'); - boardsSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class'); + integrationSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title'); + integrationSlideTitle.should.equal('Integrate with tools you love'); await welcomeScreenModal.click('#nextCarouselButton'); @@ -126,8 +137,10 @@ describe('Welcome Screen Modal', function desc() { await welcomeScreenModal.click('#prevCarouselButton'); - const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class'); - boardsSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromLeft', 'class'); + integrationSlideClass.should.contain('Carousel__slide-current'); + const integrationSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromLeft .WelcomeScreenSlide__title'); + integrationSlideTitle.should.equal('Integrate with tools you love'); }); it('MM-T4983 should be able to click the get started button and be redirected to new server modal', async () => { diff --git a/e2e/utils/report.js b/e2e/utils/report.js index 4152f199..70d56ad3 100644 --- a/e2e/utils/report.js +++ b/e2e/utils/report.js @@ -11,6 +11,7 @@ const fse = require('fs-extra'); const {MOCHAWESOME_REPORT_DIR} = require('./constants'); const package = require('../../package.json'); +const e2ePackage = require('../package.json'); const MAX_FAILED_TITLES = 5; @@ -117,7 +118,7 @@ function getOS() { function getEnvironmentValues() { return { - playwrightVersion: package.devDependencies.playwright, + playwrightVersion: e2ePackage.dependencies.playwright, electronVersion: package.devDependencies.electron, osName: getOS(), osVersion: os.release(), @@ -230,8 +231,10 @@ function generateTestReport(summary, isUploadedToS3, reportLink, testCycleKey) { function generateTitle() { const { BRANCH, + DESKTOP_VERSION, PULL_REQUEST, RELEASE_VERSION, + SERVER_VERSION, TYPE, } = process.env; @@ -258,6 +261,9 @@ function generateTitle() { case 'MANUAL': title = `E2E for Manually triggered for ${BRANCH}`; break; + case 'CMT': + title = `Compatibility Matrix Testing Report for Server v${SERVER_VERSION} and Desktop version v${DESKTOP_VERSION}`; + break; default: title = 'E2E for Build$'; } diff --git a/package.json b/package.json index 9a1a6805..5237f5aa 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "test": "run-s lint:js test:unit", "test:unit": "jest", "test:unit-coverage": "jest --coverage", - "e2e": "npm run build-test && cd e2e && npm i && npm run run", + "e2e": "npm run build-test && cd e2e && npm i && npm run run:e2e", "build": "run-p build:*", "build:main": "webpack-cli --config webpack.config.main.js", "build:preload": "webpack-cli --config webpack.config.preload.js",