Fix(e2e): Fix playwright version and reporting for CMT (#3081)

.github/workflows/compatibility-matrix-testing.yml

@@ -96,6 +96,7 @@ jobs:
       cmt: true
       MM_TEST_SERVER_URL: ${{ matrix.server.url }}
       DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }}
+      MM_SERVER_VERSION: ${{ matrix.server.version }}
 
   # We need to duplicate here in order to set the proper commit status
   # https://mattermost.atlassian.net/browse/CLD-5815

.github/workflows/e2e-functional-template.yml

@@ -35,6 +35,10 @@ on:
         description: "True if this is Compatibility Matrix Testing"
         required: false
         default: false
+      MM_SERVER_VERSION:
+        type: string
+        required: false
+        default: "9.9.1"
     outputs:
       COMMENT_BODY:
         description: "The output to comment"
@@ -77,9 +81,9 @@ jobs:
             echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT
             echo "TYPE=PR" >> $GITHUB_ENV
 
-          elif [ "${{ github.event_name }}" == "release" ]; then
+          elif [ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.cmt }}" ]; then
             echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT
-            echo "TYPE=RELEASE" >> $GITHUB_ENV
+            echo "TYPE=CMT" >> $GITHUB_ENV
             echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
             echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV
             echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV
@@ -148,8 +152,7 @@ jobs:
           npm ci
           cd e2e
           npm ci
-          npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs
+          npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs --module-dir ../
-          cd ../
 
       - name: e2e/install-dependencies-macos
         if: runner.os == 'macOS'
@@ -160,8 +163,7 @@ jobs:
           npm ci
           cd e2e
           npm ci
-          npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs
+          npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs --module-dir ../
-          cd ../
 
       ## Windows Dependencies
       - name: e2e/install-dependencies-windows
@@ -177,8 +179,7 @@ jobs:
           npm ci --openssl_fips=''
           cd e2e
           npm ci
-          npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs
+          npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs --module-dir ../
-          cd ../
 
       - name: e2e/run-playright-tests-${{ runner.os }}
         run: |
@@ -186,9 +187,13 @@ jobs:
             export DISPLAY=:99
             Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 &
           fi
-          npm run e2e || true # making job pass even if the tests fail due to flakyness
+          npm run build-test
           cd e2e
+          npm run run:e2e || true # making job pass even if the tests fail due to flakyness
           npm run send-report
+        env:
+          SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION}}
+          DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }}
 
       - name: e2e/analyze-flaky-tests
         id: analyze-flaky-tests

e2e/package.json

@@ -5,7 +5,7 @@
   "main": "dist/e2e_bundle.js",
   "scripts": {
     "clean": "rm -rf dist/ mochawesome-report/ node_modules/ testUserData/",
-    "run": "npm run build && npm run test",
+    "run:e2e": "npm run build && npm run test",
     "build": "webpack-cli --config webpack.config.js",
     "build:performance": "webpack-cli --config webpack.config.performance.js",
     "test": "electron-mocha --reporter mochawesome dist/e2e_bundle.js",

e2e/specs/startup/welcome_screen_modal.test.js

@@ -31,21 +31,27 @@ describe('Welcome Screen Modal', function desc() {
     it('MM-T4976 should show the slides in the expected order', async () => {
         const welcomeSlideClass = await welcomeScreenModal.getAttribute('#welcome', 'class');
         welcomeSlideClass.should.contain('Carousel__slide-current');
-
+        const welcomeSlideTitle = await welcomeScreenModal.innerText('#welcome .WelcomeScreenSlide__title');
+        welcomeSlideTitle.should.equal('Welcome');
         await welcomeScreenModal.click('#nextCarouselButton');
 
-        const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class');
+        const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
         channelSlideClass.should.contain('Carousel__slide-current');
-
+        const channelSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title');
+        channelSlideTitle.should.equal('Collaborate in real time');
         await welcomeScreenModal.click('#nextCarouselButton');
 
-        const playbooksSlideClass = await welcomeScreenModal.getAttribute('#playbooks', 'class');
+        const callsSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
-        playbooksSlideClass.should.contain('Carousel__slide-current');
+        callsSlideClass.should.contain('Carousel__slide-current');
-
+        const callsSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title');
+        callsSlideTitle.should.equal('Start secure calls instantly');
         await welcomeScreenModal.click('#nextCarouselButton');
 
-        const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class');
+        const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
-        boardsSlideClass.should.contain('Carousel__slide-current');
+        integrationSlideClass.should.contain('Carousel__slide-current');
+        const integrationSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title');
+        integrationSlideTitle.should.equal('Integrate with tools you love');
+        await welcomeScreenModal.click('#nextCarouselButton');
     });
 
     it('MM-T4977 should be able to move through slides clicking the navigation buttons', async () => {
@@ -54,7 +60,7 @@ describe('Welcome Screen Modal', function desc() {
 
         await welcomeScreenModal.click('#nextCarouselButton');
 
-        const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class');
+        const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
         channelSlideClass.should.contain('Carousel__slide-current');
 
         await welcomeScreenModal.click('#prevCarouselButton');
@@ -69,13 +75,13 @@ describe('Welcome Screen Modal', function desc() {
 
         await welcomeScreenModal.click('#PaginationIndicator3');
 
-        const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class');
+        const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
-        boardsSlideClass.should.contain('Carousel__slide-current');
+        integrationSlideClass.should.contain('Carousel__slide-current');
 
         await welcomeScreenModal.click('#PaginationIndicator2');
 
-        const playbooksSlideClass = await welcomeScreenModal.getAttribute('#playbooks', 'class');
+        const callsSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromLeft', 'class');
-        playbooksSlideClass.should.contain('Carousel__slide-current');
+        callsSlideClass.should.contain('Carousel__slide-current');
     });
 
     it('MM-T4979 should be able to move forward through slides automatically every 5 seconds', async () => {
@@ -84,8 +90,11 @@ describe('Welcome Screen Modal', function desc() {
 
         await asyncSleep(5500);
 
-        const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class');
+        const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
         channelSlideClass.should.contain('Carousel__slide-current');
+        const channelSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title');
+        channelSlideTitle.should.equal('Collaborate in real time');
+        await welcomeScreenModal.click('#nextCarouselButton');
     });
 
     it('MM-T4980 should show the slides in the expected order', async () => {
@@ -94,25 +103,27 @@ describe('Welcome Screen Modal', function desc() {
 
         await welcomeScreenModal.click('#nextCarouselButton');
 
-        const channelSlideClass = await welcomeScreenModal.getAttribute('#channels', 'class');
+        const channelSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
         channelSlideClass.should.contain('Carousel__slide-current');
 
         await welcomeScreenModal.click('#nextCarouselButton');
 
-        const playbooksSlideClass = await welcomeScreenModal.getAttribute('#playbooks', 'class');
+        const callsSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
-        playbooksSlideClass.should.contain('Carousel__slide-current');
+        callsSlideClass.should.contain('Carousel__slide-current');
 
         await welcomeScreenModal.click('#nextCarouselButton');
 
-        const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class');
+        const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
-        boardsSlideClass.should.contain('Carousel__slide-current');
+        integrationSlideClass.should.contain('Carousel__slide-current');
     });
 
     it('MM-T4981 should be able to move from last to first slide', async () => {
         await welcomeScreenModal.click('#PaginationIndicator3');
 
-        const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class');
+        const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromRight', 'class');
-        boardsSlideClass.should.contain('Carousel__slide-current');
+        integrationSlideClass.should.contain('Carousel__slide-current');
+        const integrationSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromRight .WelcomeScreenSlide__title');
+        integrationSlideTitle.should.equal('Integrate with tools you love');
 
         await welcomeScreenModal.click('#nextCarouselButton');
 
@@ -126,8 +137,10 @@ describe('Welcome Screen Modal', function desc() {
 
         await welcomeScreenModal.click('#prevCarouselButton');
 
-        const boardsSlideClass = await welcomeScreenModal.getAttribute('#boards', 'class');
+        const integrationSlideClass = await welcomeScreenModal.getAttribute('div.Carousel__slide.inFromLeft', 'class');
-        boardsSlideClass.should.contain('Carousel__slide-current');
+        integrationSlideClass.should.contain('Carousel__slide-current');
+        const integrationSlideTitle = await welcomeScreenModal.innerText('div.Carousel__slide.inFromLeft .WelcomeScreenSlide__title');
+        integrationSlideTitle.should.equal('Integrate with tools you love');
     });
 
     it('MM-T4983 should be able to click the get started button and be redirected to new server modal', async () => {

e2e/utils/report.js

@@ -11,6 +11,7 @@ const fse = require('fs-extra');
 const {MOCHAWESOME_REPORT_DIR} = require('./constants');
 
 const package = require('../../package.json');
+const e2ePackage = require('../package.json');
 
 const MAX_FAILED_TITLES = 5;
 
@@ -117,7 +118,7 @@ function getOS() {
 
 function getEnvironmentValues() {
     return {
-        playwrightVersion: package.devDependencies.playwright,
+        playwrightVersion: e2ePackage.dependencies.playwright,
         electronVersion: package.devDependencies.electron,
         osName: getOS(),
         osVersion: os.release(),
@@ -230,8 +231,10 @@ function generateTestReport(summary, isUploadedToS3, reportLink, testCycleKey) {
 function generateTitle() {
     const {
         BRANCH,
+        DESKTOP_VERSION,
         PULL_REQUEST,
         RELEASE_VERSION,
+        SERVER_VERSION,
         TYPE,
     } = process.env;
 
@@ -258,6 +261,9 @@ function generateTitle() {
     case 'MANUAL':
         title = `E2E for Manually triggered for ${BRANCH}`;
         break;
+    case 'CMT':
+        title = `Compatibility Matrix Testing Report for Server v${SERVER_VERSION} and Desktop version v${DESKTOP_VERSION}`;
+        break;
     default:
         title = 'E2E for Build$';
     }

package.json

@@ -31,7 +31,7 @@
     "test": "run-s lint:js test:unit",
     "test:unit": "jest",
     "test:unit-coverage": "jest --coverage",
-    "e2e": "npm run build-test && cd e2e && npm i && npm run run",
+    "e2e": "npm run build-test && cd e2e && npm i && npm run run:e2e",
     "build": "run-p build:*",
     "build:main": "webpack-cli --config webpack.config.main.js",
     "build:preload": "webpack-cli --config webpack.config.preload.js",