[MM-46305] Add reporting code for E2E tests (#2222)
* E2E reporting * Test report * Fix artifacts.js * Revert "Test report" This reverts commit f4d44b881a19c0e9d63066807f5cb6b9fe9017ee. * PR feedback
This commit is contained in:
parent
1270859d39
commit
9faaa61e48
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -10,6 +10,7 @@ npm-debug.log*
|
|||
build/
|
||||
coverage/
|
||||
dist/
|
||||
mochawesome-report/
|
||||
|
||||
test-results.xml
|
||||
test_config.json
|
||||
|
|
106
e2e/save_report.js
Normal file
106
e2e/save_report.js
Normal file
|
@ -0,0 +1,106 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
/*
|
||||
* This is used for saving artifacts to AWS S3, sending data to automation dashboard and
|
||||
* publishing quick summary to community channels.
|
||||
*
|
||||
* Usage: [ENV] node save_report.js
|
||||
*
|
||||
* Environment variables:
|
||||
* BRANCH=[branch] : Branch identifier from CI
|
||||
* BUILD_ID=[build_id] : Build identifier from CI
|
||||
* BUILD_TAG=[build_tag] : Docker image used to run the test
|
||||
*
|
||||
* For saving artifacts to AWS S3
|
||||
* - AWS_S3_BUCKET, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
|
||||
* For saving test cases to Test Management
|
||||
* - ZEPHYR_ENABLE=true|false
|
||||
* - ZEPHYR_API_KEY=[api_key]
|
||||
* - JIRA_PROJECT_KEY=[project_key], e.g. "MM",
|
||||
* - ZEPHYR_FOLDER_ID=[folder_id], e.g. 847997
|
||||
* For sending hooks to Mattermost channels
|
||||
* - FULL_REPORT, WEBHOOK_URL and DIAGNOSTIC_WEBHOOK_URL
|
||||
* Test type
|
||||
* - TYPE=[type], e.g. "MASTER", "PR", "RELEASE", "CLOUD"
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
|
||||
const chai = require('chai');
|
||||
const generator = require('mochawesome-report-generator');
|
||||
|
||||
const {
|
||||
generateShortSummary,
|
||||
generateTestReport,
|
||||
removeOldGeneratedReports,
|
||||
sendReport,
|
||||
readJsonFromFile,
|
||||
writeJsonToFile,
|
||||
} = require('./utils/report');
|
||||
const {saveArtifacts} = require('./utils/artifacts');
|
||||
const {MOCHAWESOME_REPORT_DIR} = require('./utils/constants');
|
||||
const {createTestCycle, createTestExecutions} = require('./utils/test_cases');
|
||||
|
||||
require('dotenv').config();
|
||||
|
||||
const saveReport = async () => {
|
||||
const {
|
||||
BRANCH,
|
||||
BUILD_ID,
|
||||
BUILD_TAG,
|
||||
FAILURE_MESSAGE,
|
||||
ZEPHYR_ENABLE,
|
||||
ZEPHYR_CYCLE_KEY,
|
||||
TYPE,
|
||||
WEBHOOK_URL,
|
||||
} = process.env;
|
||||
|
||||
removeOldGeneratedReports();
|
||||
|
||||
// Import
|
||||
const jsonReport = readJsonFromFile(path.join(MOCHAWESOME_REPORT_DIR, 'mochawesome.json'));
|
||||
|
||||
// Generate the html report file
|
||||
await generator.create(
|
||||
jsonReport,
|
||||
{
|
||||
reportDir: MOCHAWESOME_REPORT_DIR,
|
||||
reportTitle: `Desktop E2E - Build: ${BUILD_ID} Branch: ${BRANCH} Tag: ${BUILD_TAG}`,
|
||||
},
|
||||
);
|
||||
|
||||
// Generate short summary, write to file and then send report via webhook
|
||||
const summary = generateShortSummary(jsonReport);
|
||||
console.log(summary);
|
||||
writeJsonToFile(summary, 'summary.json', MOCHAWESOME_REPORT_DIR);
|
||||
|
||||
const result = await saveArtifacts();
|
||||
if (result && result.success) {
|
||||
console.log('Successfully uploaded artifacts to S3:', result.reportLink);
|
||||
}
|
||||
|
||||
// Create or use an existing test cycle
|
||||
let testCycle = {};
|
||||
if (ZEPHYR_ENABLE === 'true') {
|
||||
const {start, end} = jsonReport.stats;
|
||||
testCycle = ZEPHYR_CYCLE_KEY ? {key: ZEPHYR_CYCLE_KEY} : await createTestCycle(start, end);
|
||||
}
|
||||
|
||||
// Send test report to "QA: UI Test Automation" channel via webhook
|
||||
if (TYPE && TYPE !== 'NONE' && WEBHOOK_URL) {
|
||||
const data = generateTestReport(summary, result && result.success, result && result.reportLink, testCycle.key);
|
||||
await sendReport('summary report to Community channel', WEBHOOK_URL, data);
|
||||
}
|
||||
|
||||
// Save test cases to Test Management
|
||||
if (ZEPHYR_ENABLE === 'true') {
|
||||
await createTestExecutions(jsonReport, testCycle);
|
||||
}
|
||||
|
||||
chai.expect(Boolean(jsonReport.stats.failures), FAILURE_MESSAGE).to.be.false;
|
||||
};
|
||||
|
||||
saveReport();
|
90
e2e/utils/artifacts.js
Normal file
90
e2e/utils/artifacts.js
Normal file
|
@ -0,0 +1,90 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
|
||||
/* eslint-disable no-console,consistent-return */
|
||||
|
||||
const fs = require('fs');
|
||||
|
||||
const path = require('path');
|
||||
|
||||
const async = require('async');
|
||||
const AWS = require('aws-sdk');
|
||||
const mime = require('mime-types');
|
||||
const readdir = require('recursive-readdir');
|
||||
|
||||
const {MOCHAWESOME_REPORT_DIR} = require('./constants');
|
||||
|
||||
require('dotenv').config();
|
||||
|
||||
const {
|
||||
AWS_S3_BUCKET,
|
||||
AWS_ACCESS_KEY_ID,
|
||||
AWS_SECRET_ACCESS_KEY,
|
||||
BUILD_ID,
|
||||
BRANCH,
|
||||
BUILD_TAG,
|
||||
} = process.env;
|
||||
|
||||
const s3 = new AWS.S3({
|
||||
signatureVersion: 'v4',
|
||||
accessKeyId: AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: AWS_SECRET_ACCESS_KEY,
|
||||
});
|
||||
|
||||
function getFiles(dirPath) {
|
||||
return fs.existsSync(dirPath) ? readdir(dirPath) : [];
|
||||
}
|
||||
|
||||
async function saveArtifacts() {
|
||||
if (!AWS_S3_BUCKET || !AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY) {
|
||||
console.log('No AWS credentials found. Test artifacts not uploaded to S3.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const s3Folder = `${BUILD_ID}-${BRANCH}-${BUILD_TAG}`.replace(/\./g, '-');
|
||||
const uploadPath = path.resolve(__dirname, `../../${MOCHAWESOME_REPORT_DIR}`);
|
||||
const filesToUpload = await getFiles(uploadPath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
async.eachOfLimit(
|
||||
filesToUpload,
|
||||
10,
|
||||
async.asyncify(async (file) => {
|
||||
const Key = file.replace(uploadPath, s3Folder);
|
||||
const contentType = mime.lookup(file);
|
||||
const charset = mime.charset(contentType);
|
||||
|
||||
return new Promise((res, rej) => {
|
||||
s3.upload(
|
||||
{
|
||||
Key,
|
||||
Bucket: AWS_S3_BUCKET,
|
||||
Body: fs.readFileSync(file),
|
||||
ContentType: `${contentType}${charset ? '; charset=' + charset : ''}`,
|
||||
},
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log('Failed to upload artifact:', file);
|
||||
return rej(new Error(err));
|
||||
}
|
||||
res({success: true});
|
||||
},
|
||||
);
|
||||
});
|
||||
}),
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log('Failed to upload artifacts');
|
||||
return reject(new Error(err));
|
||||
}
|
||||
|
||||
const reportLink = `https://${AWS_S3_BUCKET}.s3.amazonaws.com/${s3Folder}/mochawesome.html`;
|
||||
resolve({success: true, reportLink});
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {saveArtifacts};
|
8
e2e/utils/constants.js
Normal file
8
e2e/utils/constants.js
Normal file
|
@ -0,0 +1,8 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
const MOCHAWESOME_REPORT_DIR = './mochawesome-report';
|
||||
|
||||
module.exports = {
|
||||
MOCHAWESOME_REPORT_DIR,
|
||||
};
|
306
e2e/utils/report.js
Normal file
306
e2e/utils/report.js
Normal file
|
@ -0,0 +1,306 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
/* eslint-disable no-console, camelcase */
|
||||
|
||||
const os = require('os');
|
||||
|
||||
const axios = require('axios');
|
||||
const fse = require('fs-extra');
|
||||
|
||||
const package = require('../../package.json');
|
||||
|
||||
const {MOCHAWESOME_REPORT_DIR} = require('./constants');
|
||||
|
||||
const MAX_FAILED_TITLES = 5;
|
||||
|
||||
let incrementalDuration = 0;
|
||||
|
||||
function getAllTests(results) {
|
||||
const tests = [];
|
||||
results.forEach((result) => {
|
||||
result.tests.forEach((test) => {
|
||||
incrementalDuration += test.duration;
|
||||
tests.push({...test, incrementalDuration});
|
||||
});
|
||||
|
||||
if (result.suites.length > 0) {
|
||||
getAllTests(result.suites).forEach((test) => tests.push(test));
|
||||
}
|
||||
});
|
||||
|
||||
return tests;
|
||||
}
|
||||
|
||||
function generateStatsFieldValue(stats, failedFullTitles) {
|
||||
let statsFieldValue = `
|
||||
| Key | Value |
|
||||
|:---|:---|
|
||||
| Passing Rate | ${stats.passPercent.toFixed(2)}% |
|
||||
| Duration | ${(stats.duration / (60 * 1000)).toFixed(2)} mins |
|
||||
| Suites | ${stats.suites} |
|
||||
| Tests | ${stats.tests} |
|
||||
| :white_check_mark: Passed | ${stats.passes} |
|
||||
| :x: Failed | ${stats.failures} |
|
||||
| :fast_forward: Skipped | ${stats.skipped} |
|
||||
`;
|
||||
|
||||
// If present, add full title of failing tests.
|
||||
// Only show per maximum number of failed titles with the last item as "more..." if failing tests are more than that.
|
||||
let failedTests;
|
||||
if (failedFullTitles && failedFullTitles.length > 0) {
|
||||
const re = /[:'"\\]/gi;
|
||||
const failed = failedFullTitles;
|
||||
if (failed.length > MAX_FAILED_TITLES) {
|
||||
failedTests = failed.slice(0, MAX_FAILED_TITLES - 1).map((f) => `- ${f.replace(re, '')}`).join('\n');
|
||||
failedTests += '\n- more...';
|
||||
} else {
|
||||
failedTests = failed.map((f) => `- ${f.replace(re, '')}`).join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
if (failedTests) {
|
||||
statsFieldValue += '###### Failed Tests:\n' + failedTests;
|
||||
}
|
||||
|
||||
return statsFieldValue;
|
||||
}
|
||||
|
||||
function generateShortSummary(report) {
|
||||
const {results, stats} = report;
|
||||
const tests = getAllTests(results);
|
||||
|
||||
const failedFullTitles = tests.filter((t) => t.fail).map((t) => t.fullTitle);
|
||||
const statsFieldValue = generateStatsFieldValue(stats, failedFullTitles);
|
||||
|
||||
return {
|
||||
stats,
|
||||
statsFieldValue,
|
||||
};
|
||||
}
|
||||
|
||||
function removeOldGeneratedReports() {
|
||||
[
|
||||
'all.json',
|
||||
'summary.json',
|
||||
'mochawesome.html',
|
||||
].forEach((file) => fse.removeSync(`${MOCHAWESOME_REPORT_DIR}/${file}`));
|
||||
}
|
||||
|
||||
function writeJsonToFile(jsonObject, filename, dir) {
|
||||
fse.writeJson(`${dir}/${filename}`, jsonObject).
|
||||
then(() => console.log('Successfully written:', filename)).
|
||||
catch((err) => console.error(err));
|
||||
}
|
||||
|
||||
function readJsonFromFile(file) {
|
||||
try {
|
||||
return fse.readJsonSync(file);
|
||||
} catch (err) {
|
||||
return {err};
|
||||
}
|
||||
}
|
||||
|
||||
function getOS() {
|
||||
switch (process.platform) {
|
||||
case 'darwin':
|
||||
return 'macOS';
|
||||
case 'win32':
|
||||
return 'Windows';
|
||||
case 'linux':
|
||||
return 'Linux';
|
||||
default:
|
||||
return 'Unknown';
|
||||
}
|
||||
}
|
||||
|
||||
function getEnvironmentValues() {
|
||||
return {
|
||||
playwright_version: package.devDependencies.playwright,
|
||||
electron_version: package.devDependencies.electron,
|
||||
os_name: getOS(),
|
||||
os_version: os.release(),
|
||||
node_version: process.version,
|
||||
};
|
||||
}
|
||||
|
||||
const result = [
|
||||
{status: 'Passed', priority: 'none', cutOff: 100, color: '#43A047'},
|
||||
{status: 'Failed', priority: 'low', cutOff: 98, color: '#FFEB3B'},
|
||||
{status: 'Failed', priority: 'medium', cutOff: 95, color: '#FF9800'},
|
||||
{status: 'Failed', priority: 'high', cutOff: 0, color: '#F44336'},
|
||||
];
|
||||
|
||||
function generateTestReport(summary, isUploadedToS3, reportLink, testCycleKey) {
|
||||
const {
|
||||
FULL_REPORT,
|
||||
TEST_CYCLE_LINK_PREFIX,
|
||||
} = process.env;
|
||||
const {statsFieldValue, stats} = summary;
|
||||
const {
|
||||
playwright_version,
|
||||
electron_version,
|
||||
os_name,
|
||||
os_version,
|
||||
node_version,
|
||||
} = getEnvironmentValues();
|
||||
|
||||
let testResult;
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
if (stats.passPercent >= result[i].cutOff) {
|
||||
testResult = result[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const title = generateTitle();
|
||||
const envValue = `playwright@${playwright_version} | node@${node_version} | electron@${electron_version} | ${os_name}@${os_version}`;
|
||||
|
||||
if (FULL_REPORT === 'true') {
|
||||
let reportField;
|
||||
if (isUploadedToS3) {
|
||||
reportField = {
|
||||
short: false,
|
||||
title: 'Test Report',
|
||||
value: `[Link to the report](${reportLink})`,
|
||||
};
|
||||
}
|
||||
|
||||
let testCycleField;
|
||||
if (testCycleKey) {
|
||||
testCycleField = {
|
||||
short: false,
|
||||
title: 'Test Execution',
|
||||
value: `[Recorded test executions](${TEST_CYCLE_LINK_PREFIX}${testCycleKey})`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
username: 'Playwright UI Test',
|
||||
icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
attachments: [{
|
||||
color: testResult.color,
|
||||
author_name: 'Desktop End-to-end Testing',
|
||||
author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
author_link: 'https://www.mattermost.com',
|
||||
title,
|
||||
fields: [
|
||||
{
|
||||
short: false,
|
||||
title: 'Environment',
|
||||
value: envValue,
|
||||
},
|
||||
reportField,
|
||||
testCycleField,
|
||||
{
|
||||
short: false,
|
||||
title: `Key metrics (required support: ${testResult.priority})`,
|
||||
value: statsFieldValue,
|
||||
},
|
||||
],
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
let quickSummary = `${stats.passPercent.toFixed(2)}% (${stats.passes}/${stats.tests}) in ${stats.suites} suites`;
|
||||
if (isUploadedToS3) {
|
||||
quickSummary = `[${quickSummary}](${reportLink})`;
|
||||
}
|
||||
|
||||
let testCycleLink = '';
|
||||
if (testCycleKey) {
|
||||
testCycleLink = testCycleKey ? `| [Recorded test executions](${TEST_CYCLE_LINK_PREFIX}${testCycleKey})` : '';
|
||||
}
|
||||
|
||||
return {
|
||||
username: 'Playwright UI Test',
|
||||
icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
attachments: [{
|
||||
color: testResult.color,
|
||||
author_name: 'Desktop End-to-end Testing',
|
||||
author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
author_link: 'https://www.mattermost.com/',
|
||||
title,
|
||||
text: `${quickSummary} | ${(stats.duration / (60 * 1000)).toFixed(2)} mins ${testCycleLink}\n${envValue}`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
function generateTitle() {
|
||||
const {
|
||||
BRANCH,
|
||||
PULL_REQUEST,
|
||||
RELEASE_VERSION,
|
||||
TYPE,
|
||||
} = process.env;
|
||||
|
||||
let releaseVersion = '';
|
||||
if (RELEASE_VERSION) {
|
||||
releaseVersion = ` for ${RELEASE_VERSION}`;
|
||||
}
|
||||
|
||||
let title;
|
||||
|
||||
switch (TYPE) {
|
||||
case 'PR':
|
||||
title = `E2E for Pull Request Build: [${BRANCH}](${PULL_REQUEST})`;
|
||||
break;
|
||||
case 'RELEASE':
|
||||
title = `E2E for Release Build${releaseVersion}`;
|
||||
break;
|
||||
case 'NIGHTLY':
|
||||
title = 'E2E for Master Nightly Build';
|
||||
break;
|
||||
default:
|
||||
title = 'E2E for Build$';
|
||||
}
|
||||
|
||||
return title;
|
||||
}
|
||||
|
||||
function generateDiagnosticReport(summary, serverInfo) {
|
||||
const {BRANCH, BUILD_ID} = process.env;
|
||||
|
||||
return {
|
||||
username: 'Cypress UI Test',
|
||||
icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
attachments: [{
|
||||
color: '#43A047',
|
||||
author_name: 'Cypress UI Test',
|
||||
author_icon: 'https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png',
|
||||
author_link: 'https://community.mattermost.com/core/channels/ui-test-automation',
|
||||
title: `Cypress UI Test Automation #${BUILD_ID}, **${BRANCH}** branch`,
|
||||
fields: [{
|
||||
short: false,
|
||||
value: `Start: **${summary.stats.start}**\nEnd: **${summary.stats.end}**\nUser ID: **${serverInfo.userId}**\nTeam ID: **${serverInfo.teamId}**`,
|
||||
}],
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async function sendReport(name, url, data) {
|
||||
const requestOptions = {method: 'POST', url, data};
|
||||
|
||||
try {
|
||||
const response = await axios(requestOptions);
|
||||
|
||||
if (response.data) {
|
||||
console.log(`Successfully sent ${name}.`);
|
||||
}
|
||||
return response;
|
||||
} catch (er) {
|
||||
console.log(`Something went wrong while sending ${name}.`, er);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
generateDiagnosticReport,
|
||||
generateShortSummary,
|
||||
generateTestReport,
|
||||
getAllTests,
|
||||
removeOldGeneratedReports,
|
||||
sendReport,
|
||||
readJsonFromFile,
|
||||
writeJsonToFile,
|
||||
};
|
202
e2e/utils/test_cases.js
Normal file
202
e2e/utils/test_cases.js
Normal file
|
@ -0,0 +1,202 @@
|
|||
// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
// See reference: https://support.smartbear.com/tm4j-cloud/api-docs/
|
||||
|
||||
const axios = require('axios');
|
||||
const chalk = require('chalk');
|
||||
|
||||
const {getAllTests} = require('./report');
|
||||
|
||||
const status = {
|
||||
passed: 'Pass',
|
||||
failed: 'Fail',
|
||||
pending: 'Pending',
|
||||
skipped: 'Skip',
|
||||
};
|
||||
|
||||
const environment = {
|
||||
chrome: 'Chrome',
|
||||
firefox: 'Firefox',
|
||||
};
|
||||
|
||||
function getStepStateResult(steps = []) {
|
||||
return steps.reduce((acc, item) => {
|
||||
if (acc[item.state]) {
|
||||
acc[item.state] += 1;
|
||||
} else {
|
||||
acc[item.state] = 1;
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
function getStepStateSummary(steps = []) {
|
||||
const result = getStepStateResult(steps);
|
||||
|
||||
return Object.entries(result).map(([key, value]) => `${value} ${key}`).join(',');
|
||||
}
|
||||
|
||||
function getZEPHYRTestCases(report) {
|
||||
return getAllTests(report.results).
|
||||
filter((item) => /^(MM-T)\w+/g.test(item.title)). // eslint-disable-line wrap-regex
|
||||
map((item) => {
|
||||
return {
|
||||
title: item.title,
|
||||
duration: item.duration,
|
||||
incrementalDuration: item.incrementalDuration,
|
||||
state: item.state,
|
||||
pass: item.pass,
|
||||
fail: item.fail,
|
||||
pending: item.pending,
|
||||
};
|
||||
}).
|
||||
reduce((acc, item) => {
|
||||
// Extract the key to exactly match with "MM-T[0-9]+"
|
||||
const key = item.title.match(/(MM-T\d+)/)[0];
|
||||
|
||||
if (acc[key]) {
|
||||
acc[key].push(item);
|
||||
} else {
|
||||
acc[key] = [item];
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
function saveToEndpoint(url, data) {
|
||||
return axios({
|
||||
method: 'POST',
|
||||
url,
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
Authorization: process.env.ZEPHYR_API_KEY,
|
||||
},
|
||||
data,
|
||||
}).catch((error) => {
|
||||
console.log('Something went wrong:', error.response.data.message);
|
||||
return error.response.data;
|
||||
});
|
||||
}
|
||||
|
||||
async function createTestCycle(startDate, endDate) {
|
||||
const {
|
||||
BRANCH,
|
||||
BUILD_ID,
|
||||
JIRA_PROJECT_KEY,
|
||||
ZEPHYR_CYCLE_NAME,
|
||||
ZEPHYR_FOLDER_ID,
|
||||
} = process.env;
|
||||
|
||||
const testCycle = {
|
||||
projectKey: JIRA_PROJECT_KEY,
|
||||
name: ZEPHYR_CYCLE_NAME ? `${ZEPHYR_CYCLE_NAME} (${BUILD_ID}-${BRANCH})` : `${BUILD_ID}-${BRANCH}`,
|
||||
description: `Cypress automated test with ${BRANCH}`,
|
||||
plannedStartDate: startDate,
|
||||
plannedEndDate: endDate,
|
||||
statusName: 'Done',
|
||||
folderId: ZEPHYR_FOLDER_ID,
|
||||
};
|
||||
|
||||
const response = await saveToEndpoint('https://api.zephyrscale.smartbear.com/v2/testcycles', testCycle);
|
||||
return response.data;
|
||||
}
|
||||
|
||||
async function createTestExecutions(report, testCycle) {
|
||||
const {
|
||||
BROWSER,
|
||||
JIRA_PROJECT_KEY,
|
||||
ZEPHYR_ENVIRONMENT_NAME,
|
||||
} = process.env;
|
||||
|
||||
const testCases = getZEPHYRTestCases(report);
|
||||
const startDate = new Date(report.stats.start);
|
||||
const startTime = startDate.getTime();
|
||||
|
||||
const promises = [];
|
||||
Object.entries(testCases).forEach(([key, steps], index) => {
|
||||
const testScriptResults = steps.
|
||||
sort((a, b) => a.title.localeCompare(b.title)).
|
||||
map((item) => {
|
||||
return {
|
||||
statusName: status[item.state],
|
||||
actualEndDate: new Date(startTime + item.incrementalDuration).toISOString(),
|
||||
actualResult: 'Cypress automated test completed',
|
||||
};
|
||||
});
|
||||
|
||||
const stateResult = getStepStateResult(steps);
|
||||
|
||||
const testExecution = {
|
||||
projectKey: JIRA_PROJECT_KEY,
|
||||
testCaseKey: key,
|
||||
testCycleKey: testCycle.key,
|
||||
statusName: stateResult.passed && stateResult.passed === steps.length ? 'Pass' : 'Fail',
|
||||
testScriptResults,
|
||||
environmentName: ZEPHYR_ENVIRONMENT_NAME || environment[BROWSER] || 'Chrome',
|
||||
actualEndDate: testScriptResults[testScriptResults.length - 1].actualEndDate,
|
||||
executionTime: steps.reduce((acc, prev) => {
|
||||
acc += prev.duration; // eslint-disable-line no-param-reassign
|
||||
return acc;
|
||||
}, 0),
|
||||
comment: `Cypress automated test - ${getStepStateSummary(steps)}`,
|
||||
};
|
||||
|
||||
// Temporarily log to verify cases that were being saved.
|
||||
console.log(index, key); // eslint-disable-line no-console
|
||||
|
||||
promises.push(saveTestExecution(testExecution, index));
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
console.log('Successfully saved test cases into the Test Management System');
|
||||
}
|
||||
|
||||
const saveTestCases = async (allReport) => {
|
||||
const {start, end} = allReport.stats;
|
||||
|
||||
const testCycle = await createTestCycle(start, end);
|
||||
|
||||
await createTestExecutions(allReport, testCycle);
|
||||
};
|
||||
|
||||
const RETRY = [];
|
||||
|
||||
async function saveTestExecution(testExecution, index) {
|
||||
await axios({
|
||||
method: 'POST',
|
||||
url: 'https://api.zephyrscale.smartbear.com/v2/testexecutions',
|
||||
headers: {
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
Authorization: process.env.ZEPHYR_API_KEY,
|
||||
},
|
||||
data: testExecution,
|
||||
}).then(() => {
|
||||
console.log(chalk.green('Success:', index, testExecution.testCaseKey));
|
||||
}).catch((error) => {
|
||||
// Retry on 500 error code / internal server error
|
||||
if (!error.response || error.response.data.errorCode === 500) {
|
||||
if (RETRY[testExecution.testCaseKey]) {
|
||||
RETRY[testExecution.testCaseKey] += 1;
|
||||
} else {
|
||||
RETRY[testExecution.testCaseKey] = 1;
|
||||
}
|
||||
|
||||
saveTestExecution(testExecution, index);
|
||||
console.log(chalk.magenta('Retry:', index, testExecution.testCaseKey, `(${RETRY[testExecution.testCaseKey]}x)`));
|
||||
} else {
|
||||
console.log(chalk.red('Error:', index, testExecution.testCaseKey, error.response.data.message));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createTestCycle,
|
||||
saveTestCases,
|
||||
createTestExecutions,
|
||||
};
|
1709
package-lock.json
generated
1709
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
@ -42,7 +42,8 @@
|
|||
"test:e2e": "cross-env NODE_ENV=test npm-run-all build build-robotjs test:e2e:build test:e2e:run",
|
||||
"test:e2e:nobuild": "cross-env NODE_ENV=test npm-run-all test:e2e:build test:e2e:run",
|
||||
"test:e2e:build": "webpack-cli --config webpack.config.test.js",
|
||||
"test:e2e:run": "electron-mocha -r @babel/register --reporter mocha-circleci-reporter dist/tests/e2e_bundle.js",
|
||||
"test:e2e:run": "electron-mocha -r @babel/register --reporter mochawesome dist/tests/e2e_bundle.js",
|
||||
"test:e2e:send-report": "node ./e2e/save_report.js",
|
||||
"test:unit": "jest",
|
||||
"test:unit-ci": "jest --runInBand",
|
||||
"test:coverage": "jest --coverage",
|
||||
|
@ -136,6 +137,7 @@
|
|||
"@typescript-eslint/eslint-plugin": "5.18.0",
|
||||
"@typescript-eslint/parser": "5.18.0",
|
||||
"7zip-bin": "5.1.1",
|
||||
"aws-sdk": "^2.1190.0",
|
||||
"axios": "0.26.1",
|
||||
"babel-eslint": "10.1.0",
|
||||
"babel-loader": "8.2.4",
|
||||
|
@ -164,11 +166,13 @@
|
|||
"mini-css-extract-plugin": "2.6.0",
|
||||
"mmjstool": "github:mattermost/mattermost-utilities#3b4506b0f6b14fbb402f9f8ef932370e459e3773",
|
||||
"mocha-circleci-reporter": "0.0.3",
|
||||
"mochawesome": "7.1.3",
|
||||
"node-gyp": "9.0.0",
|
||||
"node-loader": "2.0.0",
|
||||
"npm-run-all": "4.1.5",
|
||||
"playwright": "1.23.4",
|
||||
"ps-node": "^0.1.6",
|
||||
"recursive-readdir": "^2.2.2",
|
||||
"robotjs": "0.6.0",
|
||||
"sass-loader": "12.6.0",
|
||||
"shebang-loader": "0.0.1",
|
||||
|
|
Loading…
Reference in a new issue