diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml new file mode 100644 index 00000000..b5d0d08e --- /dev/null +++ b/.github/workflows/e2e-functional-template.yml @@ -0,0 +1,195 @@ +name: E2E Functional Tests Template + +on: + workflow_call: + inputs: + MM_TEST_SERVER_URL: + description: "The test server URL" + required: false + type: string + MM_TEST_USER_NAME: + description: "The admin username of the test instance" + required: false + type: string + MM_TEST_PASSWORD: + description: "The admin password of the test instance" + required: false + type: string + DESKTOP_VERSION: + description: "The desktop version to test" + required: false + default: ${{ github.ref }} + type: string + runs-on: + type: string + description: "The E2E tests underlying OS" + required: true + default: "ubuntu-22.04" + nightly: + type: boolean + description: "True if this is nigtly build" + required: false + default: false + cmt: + type: boolean + description: "True if this is Comatibility Matrix Testing" + required: false + default: false + outputs: + COMMENT_BODY: + description: "The output to comment" + value: ${{ jobs.e2e.outputs.COMMENT_BODY }} + +env: + AWS_S3_BUCKET: "mattermost-cypress-report" + BRANCH: ${{ github.head_ref || github.ref_name }} + BUILD_TAG: ${{ github.event.pull_request.head.sha || github.sha }} + JIRA_PROJECT_KEY: "MM" + MM_TEST_SERVER_URL: ${{ inputs.MM_TEST_SERVER_URL || secrets.MM_DESKTOP_E2E_SERVER_URL }} + MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_DESKTOP_E2E_USER_NAME }} + MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }} + PULL_REQUEST: "https://github.com/mattermost/desktop/pull/${{ github.event.number }}" + ZEPHYR_ENVIRONMENT_NAME: "Desktop app" + ZEPHYR_FOLDER_ID: "12413253" + TEST_CYCLE_LINK_PREFIX: ${{ secrets.MM_DESKTOP_E2E_TEST_CYCLE_LINK_PREFIX }} + AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }} + ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }} + REPORT_LINK: "none" + +jobs: + e2e: + runs-on: ${{ inputs.runs-on }} + defaults: + run: + shell: bash + outputs: + COMMENT_BODY: ${{ steps.analyze-flaky-tests.outputs.COMMENT_BODY }} + steps: + - name: e2e/set-required-variables + id: variables + run: | + RUNNER_OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]') + + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "BUILD_SUFFIX=desktop-pr-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=PR" >> $GITHUB_ENV + + elif [ "${{ github.event_name }}" == "release" ]; then + echo "BUILD_SUFFIX=desktop-release-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=RELEASE" >> $GITHUB_ENV + echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV + + elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && ! ${{ inputs.nightly }}; then + echo "BUILD_SUFFIX=desktop-manual-trigger-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=MANUAL" >> $GITHUB_ENV + + elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then + echo "BUILD_SUFFIX=desktop-master-push-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=MASTER" >> $GITHUB_ENV + echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + + elif ${{ inputs.nightly }}; then + echo "BUILD_SUFFIX=desktop-nightly-${RUNNER_OS}" >> $GITHUB_OUTPUT + echo "TYPE=NIGHTLY" >> $GITHUB_ENV + echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV + echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV + fi + + - name: e2e/set-build-id + run: echo "BUILD_ID=${{ github.run_id }}-${{ steps.variables.outputs.BUILD_SUFFIX }}" >> $GITHUB_ENV + + - name: e2e/checkout-repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.DESKTOP_VERSION }} + + - name: e2e/setup-node + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + with: + node-version-file: "package.json" + cache: "npm" + cache-dependency-path: package-lock.json + + - name: e2e/cache-node-modules + id: cache-node-modules + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: | + node_modules + C:\Users\runneradmin\.electron-gyp + key: ${{ runner.os }}-build-node-modules-${{ hashFiles('./package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-node-modules + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: e2e/setup-python + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + with: + python-version: "3.10" + + ## Linux Depdendencies + - name: e2e/install-dependencies-linux + if: runner.os == 'Linux' + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0 + run: | + wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add - + wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq + sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel + npm ci + npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs + + - name: e2e/install-dependencies-macos + if: runner.os == 'macOS' + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + run: | + jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json . + npm ci + npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs + + ## Windows Dependencies + - name: e2e/install-dependencies-windows + if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows' + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + run: | + choco install yq --version 4.15.1 -y + npm i -g node-gyp + node-gyp install + node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" + node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64 + node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch ia32 + npm ci --openssl_fips='' + npm i robotjs + npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs + + - name: e2e/run-playright-tests-${{ runner.os }} + run: | + if [ ${{ runner.os }} == 'Linux' ]; then + export DISPLAY=:99 + Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 & + fi + npm run test:e2e || true # making job pass even if the tests fail due to flakyness + npm run test:e2e:send-report + + - name: e2e/analyze-flaky-tests + id: analyze-flaky-tests + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); + const { commentBody, newFailedTests } = analyzeFlakyTests(); + core.setOutput('COMMENT_BODY', commentBody); + if (newFailedTests.length > 0) { + core.setFailed('E2E tests failed.'); + } diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 43fb8c02..07c74dcc 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -1,420 +1,129 @@ name: Electron Playwright Tests on: - workflow_call: - inputs: - tag: - description: "Reference tag of the nightly build" - required: true - type: string push: branches: - master pull_request: types: - - labeled + - labeled workflow_dispatch: inputs: version_name: type: string - description: 'Desktop Version name eg: 5.6' + description: "Desktop Version name eg: 5.6" required: true job_name: type: choice - description: 'Job name' + description: "Job name" required: true - default: 'e2e-linux' + default: "All" options: - - 'e2e-linux' - - 'e2e-macos' - - 'e2e-windows' - - 'All' - -env: - AWS_S3_BUCKET: "mattermost-cypress-report" - BRANCH: ${{ github.head_ref || github.ref_name }} - BUILD_TAG: ${{ github.sha }} - JIRA_PROJECT_KEY: 'MM' - MM_TEST_SERVER_URL: ${{ secrets.MM_DESKTOP_E2E_SERVER_URL }} - MM_TEST_USER_NAME: ${{ secrets.MM_DESKTOP_E2E_USER_NAME }} - MM_TEST_PASSWORD: ${{ secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }} - PULL_REQUEST: "https://github.com/mattermost/desktop/pull/${{ github.event.number }}" - ZEPHYR_ENVIRONMENT_NAME: 'Desktop app' - ZEPHYR_FOLDER_ID: "12413253" - TEST_CYCLE_LINK_PREFIX: ${{ secrets.MM_DESKTOP_E2E_TEST_CYCLE_LINK_PREFIX }} - AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }} - ZEPHYR_API_KEY: ${{ secrets.MM_DESKTOP_E2E_ZEPHYR_API_KEY }} - REPORT_LINK: "none" + - "e2e-linux" + - "e2e-macos" + - "e2e-windows" + - "All" jobs: e2e-linux: - if: ${{ - ( - (inputs.job_name == 'e2e-linux' || inputs.job_name == 'All') - && - github.event_name == 'workflow_dispatch' - ) || - ( - github.event_name == 'push' && github.ref == 'refs/heads/master' - ) || - ( - github.event_name == 'pull_request' && - github.event.pull_request.labels && - contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') - ) || - ( - (inputs.tag != '') - ) + if: ${{ + ( + (inputs.job_name == 'e2e-linux' || inputs.job_name == 'All') + && + github.event_name == 'workflow_dispatch' + ) || + ( + github.event_name == 'push' + ) || + ( + github.event_name == 'pull_request' && + github.event.pull_request.labels && + contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') + ) }} - runs-on: ubuntu-22.04 - steps: - - name: ci/checkout-repo - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - - name: ci/setup-node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - cache: "npm" - cache-dependency-path: package-lock.json - - - name: Install dependencies - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0 - run: | - wget -qO - https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_22.04/Release.key | sudo apt-key add - - wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.20.1/yq_linux_amd64 && chmod a+x /usr/local/bin/yq - sudo apt-get update || true && sudo apt-get install -y ca-certificates libxtst-dev libpng++-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu jq icnsutils graphicsmagick tzdata xsel - npm ci - npx electron-rebuild --platform=linux -f -t prod,optional,dev -w robotjs - - - name: Set Environment Variables - run: | - - if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "BUILD_SUFFIX=desktop-pr-linux" >> $GITHUB_ENV - echo "TYPE=PR" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "release" ]; then - echo "BUILD_SUFFIX=desktop-release-linux" >> $GITHUB_ENV - echo "TYPE=RELEASE" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_LINUX_REPORT=12358649" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then - echo "BUILD_SUFFIX=desktop-manual-trigger-linux" >> $GITHUB_ENV - echo "TYPE=MANUAL" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then - echo "BUILD_SUFFIX=desktop-master-push-linux" >> $GITHUB_ENV - echo "TYPE=MASTER" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - - elif [ "${{ inputs.tag }}" != "" ]; then - echo "BUILD_SUFFIX=desktop-nightly-linux" >> $GITHUB_ENV - echo "TYPE=NIGHTLY" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_LINUX_REPORT=12363689" >> $GITHUB_ENV - fi - - - name: Set Build ID - run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV - - - name: Run Playwright tests (Ubuntu OS) - run: | - export DISPLAY=:99 - Xvfb $DISPLAY -screen 0 1280x960x24 > /dev/null 2>&1 & - npm run test:e2e || true # making job pass even if the tests fail due to flakyness - npm run test:e2e:send-report - - - name: Analyze flaky tests - id: analyze-flaky-tests - uses: actions/github-script@v7 - with: - script: | - const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); - const { newFailedTests } = analyzeFlakyTests(); - if (newFailedTests.length === 0) { - process.env.NEW_FAILED_TESTS = 'true'; - } - env: - NEW_FAILED_TESTS: false - - - name: Add results in PR comment - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); - const { commentBody } = analyzeFlakyTests(); - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody, - }); - - - name: Mark job as failed if there are new failed tests - if: env.NEW_FAILED_TESTS == 'true' - run: exit 1 - shell: bash + uses: ./.github/workflows/e2e-functional-template.yml + secrets: inherit + with: + runs-on: ubuntu-22.04 + DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} e2e-macos: - if: ${{ - ( - (inputs.job_name == 'e2e-macos' || inputs.job_name == 'All') - && - github.event_name == 'workflow_dispatch' - ) || - ( - github.event_name == 'push' && github.ref == 'refs/heads/master' - ) || - ( - github.event_name == 'pull_request' && - github.event.pull_request.labels && - contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') - ) || - ( - (inputs.tag != '') - ) + if: ${{ + ( + (inputs.job_name == 'e2e-macos' || inputs.job_name == 'All') + && + github.event_name == 'workflow_dispatch' + ) || + ( + github.event_name == 'push' + ) || + ( + github.event_name == 'pull_request' && + github.event.pull_request.labels && + contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') + ) }} - runs-on: macos-13 - steps: - - name: ci/checkout-repo - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - - name: ci/setup-node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - cache: "npm" - cache-dependency-path: package-lock.json - - name: Cache Electron Headers - uses: actions/cache@v3 - with: - path: | - ~/.electron - ~/.cache/electron - key: electron-headers-${{ runner.os }}-${{ hashFiles('**/*.gyp') }} + uses: ./.github/workflows/e2e-functional-template.yml + secrets: inherit + with: + runs-on: macos-13 + DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: ci/install-dependencies - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: | - npm cache clean --force - jq '.mac.target=["zip"]' electron-builder.json | jq '.mac.gatekeeperAssess=false' > /tmp/electron-builder.json && cp /tmp/electron-builder.json . - npm ci - npx electron-rebuild --platform=darwin -f -t prod,optional,dev -w robotjs - - - name: Set Environment Variables - run: | - - if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "BUILD_SUFFIX=desktop-pr-macos" >> $GITHUB_ENV - echo "TYPE=PR" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "release" ]; then - echo "BUILD_SUFFIX=desktop-release-macos" >> $GITHUB_ENV - echo "TYPE=RELEASE" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_MACOS_REPORT=12358650" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then - echo "BUILD_SUFFIX=desktop-manual-trigger-macos" >> $GITHUB_ENV - echo "TYPE=MANUAL" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then - echo "BUILD_SUFFIX=desktop-master-push-macos" >> $GITHUB_ENV - echo "TYPE=MASTER" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - - elif [ "${{ inputs.tag }}" != "" ]; then - echo "BUILD_SUFFIX=desktop-nightly-macos" >> $GITHUB_ENV - echo "TYPE=NIGHTLY" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_MACOS_REPORT=12363687" >> $GITHUB_ENV - fi - - - name: Set Build ID - run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV - - - name: Run Playwright tests (macOS) - run: | - npm run test:e2e || true # making job pass even if the tests fail due to flakyness - npm run test:e2e:send-report - - - name: Analyze flaky tests - id: analyze-flaky-tests - uses: actions/github-script@v7 - with: - script: | - const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); - const { newFailedTests } = analyzeFlakyTests(); - if (newFailedTests.length === 0) { - process.env.NEW_FAILED_TESTS = 'true'; - } - - - name: Add results in PR comment - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); - const { commentBody } = analyzeFlakyTests(); - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody, - }); - - - name: Mark job as failed if there are new failed tests - if: env.NEW_FAILED_TESTS == 'true' - run: exit 1 - shell: bash - e2e-windows: - if: ${{ - ( - (inputs.job_name == 'e2e-windows' || inputs.job_name == 'All') - && - github.event_name == 'workflow_dispatch' - ) || - ( - github.event_name == 'push' && github.ref == 'refs/heads/master' - ) || - ( - github.event_name == 'pull_request' && - github.event.pull_request.labels && - contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') - ) || - ( - (inputs.tag != '') - ) + if: ${{ + ( + (inputs.job_name == 'e2e-windows' || inputs.job_name == 'All') + && + github.event_name == 'workflow_dispatch' + ) || + ( + github.event_name == 'push' + ) || + ( + github.event_name == 'pull_request' && + github.event.pull_request.labels && + contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') + ) }} - runs-on: windows-2022 - steps: - - name: ci/checkout-repo - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - - name: ci/setup-node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - cache: "npm" - cache-dependency-path: package-lock.json - - - name: ci/cache-node-modules - id: cache-node-modules - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: node_modules - key: ${{ runner.os }}-build-node-modules-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-node-modules - ${{ runner.os }}-build- - ${{ runner.os }}- - - - name: ci/install-node-gyp - if: steps.cache-node-modules.outputs.cache-hit != 'true' - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: | - choco install yq --version 4.15.1 -y - npm install -g windows-build-tools - npm i -g node-gyp - node-gyp install - node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" - node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch arm64 - node-gyp install --devdir="C:\Users\runneradmin\.electron-gyp" --target=$(jq -r .devDependencies.electron package.json) --dist-url="https://electronjs.org/headers" --arch ia32 - npm ci --openssl_fips='' - npm i robotjs - npx electron-rebuild --platform=win32 -f -t prod,optional,dev -w robotjs - - - name: Set Environment Variables - shell: bash - run: | - - if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "BUILD_SUFFIX=desktop-pr-windows" >> $GITHUB_ENV - echo "TYPE=PR" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "release" ]; then - echo "BUILD_SUFFIX=desktop-release-windows" >> $GITHUB_ENV - echo "TYPE=RELEASE" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_WIN_REPORT=12358651" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ inputs.tag }}" == "" ]; then - echo "BUILD_SUFFIX=desktop-manual-trigger-windows" >> $GITHUB_ENV - echo "TYPE=MANUAL" >> $GITHUB_ENV - - elif [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/master" ]; then - echo "TYPE=MASTER" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-master-push-windows" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - - elif [ "${{ inputs.tag }}" != "" ]; then - echo "TYPE=NIGHTLY" >> $GITHUB_ENV - echo "BUILD_SUFFIX=desktop-nightly-windows" >> $GITHUB_ENV - echo "ZEPHYR_ENABLE=true" >> $GITHUB_ENV - echo "ZEPHYR_FOLDER_WIN_REPORT=12363690" >> $GITHUB_ENV - fi - - - name: Set Build ID - run: echo "BUILD_ID=${{ github.run_id }}-${{ env.BUILD_SUFFIX }}" >> $GITHUB_ENV - shell: bash - - - name: Run Playwright tests (Windows OS) - run: | - npm run test:e2e || true - npm run test:e2e:send-report - shell: bash - - - name: Analyze flaky tests - id: analyze-flaky-tests - uses: actions/github-script@v7 - with: - script: | - const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); - const { newFailedTests } = analyzeFlakyTests(); - if (newFailedTests.length === 0) { - process.env.NEW_FAILED_TESTS = 'true'; - } - - - name: Add results in PR comment - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const { analyzeFlakyTests } = require('./e2e/utils/analyze-flaky-test.js'); - const { commentBody } = analyzeFlakyTests(); - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody, - }); - - - name: Mark job as failed if there are new failed tests - if: env.NEW_FAILED_TESTS == 'true' - run: exit 1 - shell: bash + uses: ./.github/workflows/e2e-functional-template.yml + secrets: inherit + with: + runs-on: windows-2022 + DESKTOP_VERSION: ${{ inputs.version_name || github.head_ref || github.ref }} e2e-remove-label: if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'Run Desktop E2E Tests') }} needs: [e2e-linux, e2e-macos, e2e-windows] - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - - name: Remove "Run Desktop E2E Tests" label - uses: actions-ecosystem/action-remove-labels@v1 + - name: e2e/unify-comments-in-single-comment + run: | + echo "PR_COMMENT<> "${GITHUB_ENV}" + echo "Here are the test results below:" >> "${GITHUB_ENV}" + echo "${{ needs.e2e-linux.outputs.COMMENT_BODY }}" >> "${GITHUB_ENV}" + echo "${{ needs.e2e-macos.outputs.COMMENT_BODY }}" >> "${GITHUB_ENV}" + echo "${{ needs.e2e-windows.outputs.COMMENT_BODY }}" >> "${GITHUB_ENV}" + echo "EOF" >> "${GITHUB_ENV}" + + - name: e2e/send-comment-results-in-pr + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: - labels: Run Desktop E2E Tests + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: process.env.PR_COMMENT, + }); + + - name: e2e/remove-label-from-pr + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + continue-on-error: true # Label might have been removed manually + with: + script: | + github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: 'Run Desktop E2E Tests', + }); diff --git a/.github/workflows/nightly-builds.yaml b/.github/workflows/nightly-builds.yaml index 4960621f..c1aa51e0 100644 --- a/.github/workflows/nightly-builds.yaml +++ b/.github/workflows/nightly-builds.yaml @@ -46,8 +46,15 @@ jobs: nightly-e2e: needs: - tag-nightly-build - uses: ./.github/workflows/e2e-functional.yml + strategy: + matrix: + runs-on: + - macos-13 + - ubuntu-22.04 + - windows-2022 + uses: ./.github/workflows/e2e-functional-template.yml secrets: inherit with: - tag: ${{ needs.tag-nightly-build.outputs.tag }} - \ No newline at end of file + runs-on: ${{ matrix.runs-on }} + nightly: true + DESKTOP_VERSION: ${{ needs.tag-nightly-build.outputs.tag }} diff --git a/e2e/utils/analyze-flaky-test.js b/e2e/utils/analyze-flaky-test.js index f06b2d97..ff6ed24a 100644 --- a/e2e/utils/analyze-flaky-test.js +++ b/e2e/utils/analyze-flaky-test.js @@ -43,33 +43,27 @@ function generateCommentBodyFunctionalTest(newFailedTests, fixedTests) { const build = process.env.BUILD_TAG; let commentBody = ` - ## Test Summary for ${osName} on commit ${build} -`; +## Test Summary for ${osName} on commit ${build} + `; if (newFailedTests.length === 0 && fixedTests.length === 0) { commentBody += ` - All stable tests passed on ${osName}. +All stable tests passed on ${osName}. `; return commentBody; } if (newFailedTests.length > 0) { - const newTestFailure = `New failed tests found on ${osName}:\n\t${newFailedTests.map((test) => `- ${test}`).join('\n\t')}`; + const newTestFailure = `New failed tests found on ${osName}:\n${newFailedTests.map((test) => `- ${test}`).join('\n')}`; commentBody += ` - ### New Failed Tests - - | Test | - | --- | - ${newTestFailure} +${newTestFailure} `; } if (fixedTests.length > 0) { const fixedTestMessage = `The following known failed tests have been fixed on ${osName}:\n\t${fixedTests.map((test) => `- ${test}`).join('\n\t')}`; commentBody += ` - ### Fixed Tests - - ${fixedTestMessage} +${fixedTestMessage} `; }