feat: Refactor E2E functional for readability and reusability (#2988)

.github/workflows/e2e-functional-template.yml

@@ -0,0 +1,195 @@
+name: E2E Functional Tests Template
+
+on:
+  workflow_call:
+    inputs:
+      MM_TEST_SERVER_URL:
+        description: "The test server URL"
+        required: false
+        type: string
+      MM_TEST_USER_NAME:
+        description: "The admin username of the test instance"
+        required: false
+        type: string
+      MM_TEST_PASSWORD:
+        description: "The admin password of the test instance"
+        required: false
+        type: string
+      DESKTOP_VERSION:
+        description: "The desktop version to test"
+        required: false
+        default: ${{ github.ref }}
+        type: string
+      runs-on:
+        type: string
+        description: "The E2E tests underlying OS"
+        required: true
+        default: "ubuntu-22.04"
+      nightly:
+        type: boolean
+        description: "True if this is nigtly build"
+        required: false
+        default: false
+      cmt:
+        type: boolean
+        description: "True if this is Comatibility Matrix Testing"        
+        required: false
+        default: false
+    outputs:
+      COMMENT_BODY:
+        description: "The output to comment"
+        value: ${{ jobs.e2e.outputs.COMMENT_BODY }}
+
+env:
+  AWS_S3_BUCKET: "mattermost-cypress-report"
+  BRANCH: ${{ github.head_ref || github.ref_name }}
+  BUILD_TAG: ${{ github.event.pull_request.head.sha || github.sha }}
+  JIRA_PROJECT_KEY: "MM"
+  MM_TEST_SERVER_URL: ${{ inputs.MM_TEST_SERVER_URL || secrets.MM_DESKTOP_E2E_SERVER_URL }}
+  MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_DESKTOP_E2E_USER_NAME }}
+  MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }}
+  PULL_REQUEST: "https://github.com/mattermost/desktop/pull/${{ github.event.number }}"
+  ZEPHYR_ENVIRONMENT_NAME: "Desktop app"
+  ZEPHYR_FOLDER_ID: "12413253"
+  TEST_CYCLE_LINK_PREFIX: ${{ secrets.MM_DESKTOP_E2E_TEST_CYCLE_LINK_PREFIX }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: "us-east-1"
+  WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }}
+  ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }}
+  REPORT_LINK: "none"
+
+jobs:
+  e2e:
+    runs-on: ${{ inputs.runs-on }}
+    defaults:
+      run:
+        shell: bash
+    outputs:
+      COMMENT_BODY: ${{ steps.analyze-flaky-tests.outputs.COMMENT_BODY }}
+    steps:
+      - name: e2e/set-required-variables
+        id: variables
+        run: |
+          RUNNER_OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]')
+
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT
+            echo "TYPE=PR" >> $GITHUB_ENV
+
+          elif [ "${{ github.event_name }}" == "release" ]; then
+            echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT
+            echo "TYPE=RELEASE" >> $GITHUB_ENV
+            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
+            echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV
+            echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV
+            echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV
+
+          elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && ! ${{ inputs.nightly }}; then
+            echo "BUILD_SUFFIX=desktop-manual-trigger-${RUNNER_OS}" >> $GITHUB_OUTPUT
+            echo "TYPE=MANUAL" >> $GITHUB_ENV
+
+          elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            echo "BUILD_SUFFIX=desktop-master-push-${RUNNER_OS}" >> $GITHUB_OUTPUT
+            echo "TYPE=MASTER" >> $GITHUB_ENV
+            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
+
+          elif ${{ inputs.nightly }}; then
+            echo "BUILD_SUFFIX=desktop-nightly-${RUNNER_OS}" >> $GITHUB_OUTPUT
+            echo "TYPE=NIGHTLY" >> $GITHUB_ENV
+            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
+            echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV
+            echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV
+            echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV
+          fi
+
+      - name: e2e/set-build-id
+        run: echo "BUILD_ID=${{ github.run_id }}-${{ steps.variables.outputs.BUILD_SUFFIX }}" >> $GITHUB_ENV
+
+      - name: e2e/checkout-repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ inputs.DESKTOP_VERSION }}
+
+      - name: e2e/setup-node
+        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+        with:
+          node-version-file: "package.json"
+          cache: "npm"
+          cache-dependency-path: package-lock.json
+
+      - name: e2e/cache-node-modules
+        id: cache-node-modules
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            node_modules
+            C:\Users\runneradmin\.electron-gyp
+          key: ${{ runner.os }}-build-node-modules-${{ hashFiles('./package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-node-modules
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: e2e/setup-python
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        with:
+          python-version: "3.10"
+
+      ## Linux Depdendencies
+      - name: e2e/install-dependencies-linux
+        if: runner.os == 'Linux'
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0
+        run: |
+          wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add -
+          wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq
+          sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel
+          npm ci
+          npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs
+
+      - name: e2e/install-dependencies-macos
+        if: runner.os == 'macOS'
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        run: |
+          jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json .
+          npm ci
+          npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs
+
+      ## Windows Dependencies
+      - name: e2e/install-dependencies-windows
+        if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows'
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        run: |
+          choco install yq --version 4.15.1 -y
+          npm i -g node-gyp
+          node-gyp install
+          node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers"
+          node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64
+          node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch ia32
+          npm ci --openssl_fips=''
+          npm i robotjs
+          npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs
+
+      - name: e2e/run-playright-tests-${{ runner.os }}
+        run: |
+          if [ ${{ runner.os }} == 'Linux' ]; then
+            export DISPLAY=:99
+            Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 &
+          fi
+          npm run test:e2e || true # making job pass even if the tests fail due to flakyness
+          npm run test:e2e:send-report
+
+      - name: e2e/analyze-flaky-tests
+        id: analyze-flaky-tests
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
+            const { commentBody, newFailedTests } = analyzeFlakyTests();
+            core.setOutput('COMMENT_BODY', commentBody);
+            if (newFailedTests.length > 0) {
+              core.setFailed('E2E tests failed.');
+            }

.github/workflows/e2e-functional.yml

@@ -1,420 +1,129 @@
 name: Electron Playwright Tests
 
 on:
-  workflow_call:
-    inputs:
-      tag:
-        description: "Reference tag of the nightly build"
-        required: true
-        type: string
   push:
     branches:
       - master
   pull_request:
     types:
-        - labeled
+      - labeled
   workflow_dispatch:
     inputs:
       version_name:
         type: string
-        description: 'Desktop Version name eg: 5.6'
+        description: "Desktop Version name eg: 5.6"
         required: true
       job_name:
         type: choice
-        description: 'Job name'
+        description: "Job name"
         required: true
-        default: 'e2e-linux'
+        default: "All"
         options:
-          - 'e2e-linux'
-          - 'e2e-macos'
-          - 'e2e-windows'
-          - 'All'
-      
-env:
-  AWS_S3_BUCKET: "mattermost-cypress-report"
-  BRANCH: ${{ github.head_ref || github.ref_name }} 
-  BUILD_TAG: ${{ github.sha }}
-  JIRA_PROJECT_KEY: 'MM'
-  MM_TEST_SERVER_URL: ${{ secrets.MM_DESKTOP_E2E_SERVER_URL }}
-  MM_TEST_USER_NAME: ${{ secrets.MM_DESKTOP_E2E_USER_NAME }}
-  MM_TEST_PASSWORD: ${{ secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }}
-  PULL_REQUEST: "https://github.com/mattermost/desktop/pull/${{ github.event.number }}"
-  ZEPHYR_ENVIRONMENT_NAME: 'Desktop app'
-  ZEPHYR_FOLDER_ID: "12413253"
-  TEST_CYCLE_LINK_PREFIX: ${{ secrets.MM_DESKTOP_E2E_TEST_CYCLE_LINK_PREFIX }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }}
-  AWS_REGION: "us-east-1"
-  WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }}
-  ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }}
-  REPORT_LINK: "none"
+          - "e2e-linux"
+          - "e2e-macos"
+          - "e2e-windows"
+          - "All"
 
 jobs:
   e2e-linux:
     if: ${{
-        (
-          (inputs.job_name == 'e2e-linux' || inputs.job_name == 'All')
-          &&
-            github.event_name == 'workflow_dispatch'
-        ) || 
-        (
-            github.event_name == 'push' && github.ref == 'refs/heads/master'
-        ) || 
-        (
-            github.event_name == 'pull_request' && 
-            github.event.pull_request.labels && 
-            contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
-        ) ||
-        (
-          (inputs.tag != '')
-        )
+      (
+      (inputs.job_name == 'e2e-linux' || inputs.job_name == 'All')
+      &&
+      github.event_name == 'workflow_dispatch'
+      ) ||
+      (
+      github.event_name == 'push'
+      ) ||
+      (
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.labels &&
+      contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
+      )
       }}
-    runs-on: ubuntu-22.04
-    steps:
-      - name: ci/checkout-repo
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: ci/setup-node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: "package.json"
-          cache: "npm"
-          cache-dependency-path: package-lock.json
-
-      - name: Install dependencies
-        env:
-          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0
-        run: |
-          wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add -
-          wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq
-          sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel
-          npm ci
-          npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs
-
-      - name: Set Environment Variables
-        run: |
-
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "BUILD_SUFFIX=desktop-pr-linux" >> $GITHUB_ENV
-            echo "TYPE=PR" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "release" ]; then
-            echo "BUILD_SUFFIX=desktop-release-linux" >> $GITHUB_ENV
-            echo "TYPE=RELEASE" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-            echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then
-            echo "BUILD_SUFFIX=desktop-manual-trigger-linux" >> $GITHUB_ENV
-            echo "TYPE=MANUAL" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then
-            echo "BUILD_SUFFIX=desktop-master-push-linux" >> $GITHUB_ENV
-            echo "TYPE=MASTER" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-
-          elif [ "${{ inputs.tag }}" != "" ]; then
-            echo "BUILD_SUFFIX=desktop-nightly-linux" >> $GITHUB_ENV
-            echo "TYPE=NIGHTLY" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-            echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV
-          fi
-
-      - name: Set Build ID
-        run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV
-
-      - name: Run Playwright tests (Ubuntu OS)
-        run: |
-          export DISPLAY=:99
-          Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 &
-          npm run test:e2e || true # making job pass even if the tests fail due to flakyness
-          npm run test:e2e:send-report
-
-      - name: Analyze flaky tests
-        id: analyze-flaky-tests
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
-            const { newFailedTests } = analyzeFlakyTests();
-            if (newFailedTests.length === 0) {
-              process.env.NEW_FAILED_TESTS = 'true';
-            }
-        env:
-          NEW_FAILED_TESTS: false
-    
-      - name: Add results in PR comment
-        if: github.event_name == 'pull_request'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
-            const { commentBody } = analyzeFlakyTests();
-
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: commentBody,
-            });
-
-      - name: Mark job as failed if there are new failed tests
-        if: env.NEW_FAILED_TESTS == 'true'
-        run: exit 1
-        shell: bash
+    uses: ./.github/workflows/e2e-functional-template.yml
+    secrets: inherit
+    with:
+      runs-on: ubuntu-22.04
+      DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }}
 
   e2e-macos:
     if: ${{
-        (
-          (inputs.job_name == 'e2e-macos' || inputs.job_name == 'All')
-          &&
-            github.event_name == 'workflow_dispatch'
-        ) || 
-        (
-            github.event_name == 'push' && github.ref == 'refs/heads/master'
-        ) || 
-        (
-            github.event_name == 'pull_request' && 
-            github.event.pull_request.labels && 
-            contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
-        ) ||
-        (
-          (inputs.tag != '')
-        )
+      (
+      (inputs.job_name == 'e2e-macos' || inputs.job_name == 'All')
+      &&
+      github.event_name == 'workflow_dispatch'
+      ) ||
+      (
+      github.event_name == 'push'
+      ) ||
+      (
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.labels &&
+      contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
+      )
       }}
-    runs-on: macos-13
-    steps:
-      - name: ci/checkout-repo
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: ci/setup-node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: "package.json"
-          cache: "npm"
-          cache-dependency-path: package-lock.json
-      - name: Cache Electron Headers
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.electron
-            ~/.cache/electron
-          key: electron-headers-${{ runner.os }}-${{ hashFiles('**/*.gyp') }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-  
-      - name: ci/install-dependencies
-        env:
-          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-        run: |
-          npm cache clean --force
-          jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json .
-          npm ci
-          npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs
-
-      - name: Set Environment Variables
-        run: |
-
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "BUILD_SUFFIX=desktop-pr-macos" >> $GITHUB_ENV
-            echo "TYPE=PR" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "release" ]; then
-            echo "BUILD_SUFFIX=desktop-release-macos" >> $GITHUB_ENV
-            echo "TYPE=RELEASE" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-            echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then
-            echo "BUILD_SUFFIX=desktop-manual-trigger-macos" >> $GITHUB_ENV
-            echo "TYPE=MANUAL" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then
-            echo "BUILD_SUFFIX=desktop-master-push-macos" >> $GITHUB_ENV
-            echo "TYPE=MASTER" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-
-          elif [ "${{ inputs.tag }}" != "" ]; then
-            echo "BUILD_SUFFIX=desktop-nightly-macos" >> $GITHUB_ENV
-            echo "TYPE=NIGHTLY" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-            echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV
-          fi
-
-      - name: Set Build ID
-        run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV
-  
-      - name: Run Playwright tests (macOS)
-        run: |
-          npm run test:e2e || true # making job pass even if the tests fail due to flakyness
-          npm run test:e2e:send-report
-
-      - name: Analyze flaky tests
-        id: analyze-flaky-tests
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
-            const { newFailedTests } = analyzeFlakyTests();
-            if (newFailedTests.length === 0) {
-              process.env.NEW_FAILED_TESTS = 'true';
-            }
-
-      - name: Add results in PR comment
-        if: github.event_name == 'pull_request'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
-            const { commentBody } = analyzeFlakyTests();
-
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: commentBody,
-            });
-
-      - name: Mark job as failed if there are new failed tests
-        if: env.NEW_FAILED_TESTS == 'true'
-        run: exit 1
-        shell: bash
+    uses: ./.github/workflows/e2e-functional-template.yml
+    secrets: inherit
+    with:
+      runs-on: macos-13
+      DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }}
 
   e2e-windows:
     if: ${{
-        (
-          (inputs.job_name == 'e2e-windows' || inputs.job_name == 'All')
-          &&
-            github.event_name == 'workflow_dispatch'
-        ) || 
-        (
-            github.event_name == 'push' && github.ref == 'refs/heads/master'
-        ) || 
-        (
-            github.event_name == 'pull_request' && 
-            github.event.pull_request.labels && 
-            contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
-        ) ||
-        (
-          (inputs.tag != '')
-        )
+      (
+      (inputs.job_name == 'e2e-windows' || inputs.job_name == 'All')
+      &&
+      github.event_name == 'workflow_dispatch'
+      ) ||
+      (
+      github.event_name == 'push'
+      ) ||
+      (
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.labels &&
+      contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests')
+      )
       }}
-    runs-on: windows-2022
-    steps:
-      - name: ci/checkout-repo
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: ci/setup-node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: "package.json"
-          cache: "npm"
-          cache-dependency-path: package-lock.json
-
-      - name: ci/cache-node-modules
-        id: cache-node-modules
-        uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
-        with:
-          path: node_modules
-          key: ${{ runner.os }}-build-node-modules-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-build-node-modules
-            ${{ runner.os }}-build-
-            ${{ runner.os }}-
-
-      - name: ci/install-node-gyp
-        if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        env:
-          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-        run: |
-          choco install yq --version 4.15.1 -y
-          npm install -g windows-build-tools
-          npm i -g node-gyp
-          node-gyp install
-          node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers"
-          node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64
-          node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch ia32
-          npm ci --openssl_fips=''
-          npm i robotjs
-          npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs
-
-      - name: Set Environment Variables
-        shell: bash
-        run: |
-
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "BUILD_SUFFIX=desktop-pr-windows" >> $GITHUB_ENV
-            echo "TYPE=PR" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "release" ]; then
-            echo "BUILD_SUFFIX=desktop-release-windows" >> $GITHUB_ENV
-            echo "TYPE=RELEASE" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-            echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV
-
-          elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then
-            echo "BUILD_SUFFIX=desktop-manual-trigger-windows" >> $GITHUB_ENV
-            echo "TYPE=MANUAL" >> $GITHUB_ENV
-  
-          elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then
-            echo "TYPE=MASTER" >> $GITHUB_ENV
-            echo "BUILD_SUFFIX=desktop-master-push-windows" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-
-          elif [ "${{ inputs.tag }}" != "" ]; then
-            echo "TYPE=NIGHTLY" >> $GITHUB_ENV
-            echo "BUILD_SUFFIX=desktop-nightly-windows" >> $GITHUB_ENV
-            echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV
-            echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV
-          fi
-
-      - name: Set Build ID
-        run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV
-        shell: bash
-
-      - name: Run Playwright tests (Windows OS)
-        run: |
-          npm run test:e2e || true
-          npm run test:e2e:send-report
-        shell: bash
-
-      - name: Analyze flaky tests
-        id: analyze-flaky-tests
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
-            const { newFailedTests } = analyzeFlakyTests();
-            if (newFailedTests.length === 0) {
-              process.env.NEW_FAILED_TESTS = 'true';
-            }
-
-      - name: Add results in PR comment
-        if: github.event_name == 'pull_request'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js');
-            const { commentBody } = analyzeFlakyTests();
-
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: commentBody,
-            });
-
-      - name: Mark job as failed if there are new failed tests
-        if: env.NEW_FAILED_TESTS == 'true'
-        run: exit 1
-        shell: bash
+    uses: ./.github/workflows/e2e-functional-template.yml
+    secrets: inherit
+    with:
+      runs-on: windows-2022
+      DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }}
 
   e2e-remove-label:
     if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }}
     needs: [e2e-linux, e2e-macos, e2e-windows]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - name: Remove "Run Desktop E2E Tests" label
-        uses: actions-ecosystem/action-remove-labels@v1
+      - name: e2e/unify-comments-in-single-comment
+        run: |
+          echo "PR_COMMENT<<EOF" >> "${GITHUB_ENV}"
+          echo "Here are the test results below:" >> "${GITHUB_ENV}"
+          echo "${{ needs.e2e-linux.outputs.COMMENT_BODY }}" >> "${GITHUB_ENV}"
+          echo "${{ needs.e2e-macos.outputs.COMMENT_BODY }}" >> "${GITHUB_ENV}"
+          echo "${{ needs.e2e-windows.outputs.COMMENT_BODY }}" >> "${GITHUB_ENV}"
+          echo "EOF" >> "${GITHUB_ENV}"
+
+      - name: e2e/send-comment-results-in-pr
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
-          labels: Run Desktop E2E Tests
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: process.env.PR_COMMENT,
+            });
+
+      - name: e2e/remove-label-from-pr
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        continue-on-error: true # Label might have been removed manually
+        with:
+          script: |
+            github.rest.issues.removeLabel({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'Run Desktop E2E Tests',
+            });

.github/workflows/nightly-builds.yaml

@@ -46,8 +46,15 @@ jobs:
   nightly-e2e:
     needs:
       - tag-nightly-build
-    uses: ./.github/workflows/e2e-functional.yml
+    strategy:
+      matrix:
+        runs-on:
+          - macos-13
+          - ubuntu-22.04
+          - windows-2022
+    uses: ./.github/workflows/e2e-functional-template.yml
     secrets: inherit
     with:
-      tag: ${{ needs.tag-nightly-build.outputs.tag }}
-    
+      runs-on: ${{ matrix.runs-on }}
+      nightly: true
+      DESKTOP_VERSION: ${{ needs.tag-nightly-build.outputs.tag }}

e2e/utils/analyze-flaky-test.js

@@ -43,33 +43,27 @@ function generateCommentBodyFunctionalTest(newFailedTests, fixedTests) {
     const build = process.env.BUILD_TAG;
 
     let commentBody = `
-    ## Test Summary for ${osName} on commit ${build}
-`;
+## Test Summary for ${osName} on commit ${build}
+    `;
 
     if (newFailedTests.length === 0 && fixedTests.length === 0) {
         commentBody += `
-        All stable tests passed on ${osName}.
+All stable tests passed on ${osName}.
     `;
         return commentBody;
     }
 
     if (newFailedTests.length > 0) {
-        const newTestFailure = `New failed tests found on ${osName}:\n\t${newFailedTests.map((test) => `- ${test}`).join('\n\t')}`;
+        const newTestFailure = `New failed tests found on ${osName}:\n${newFailedTests.map((test) => `- ${test}`).join('\n')}`;
         commentBody += `
-    ### New Failed Tests
-    
-    | Test |
-    | --- |
-       ${newTestFailure}
+${newTestFailure}
     `;
     }
 
     if (fixedTests.length > 0) {
         const fixedTestMessage = `The following known failed tests have been fixed on ${osName}:\n\t${fixedTests.map((test) => `- ${test}`).join('\n\t')}`;
         commentBody += `
-    ### Fixed Tests
-    
-    ${fixedTestMessage}
+${fixedTestMessage}
     `;
     }