Skip to content

Commit 384a0cc

Browse files
jugglinmikejugglinmike
authored
Add ability to designate tests as "untestable" (#1405)
* Changes instances of 'unexpected behaviors' wording to 'negative side effects' * Adds an "is assertion untestable" checkbox to test runner
1 parent 6bdd284 commit 384a0cc

37 files changed

+1372
-838
lines changed

‎client/components/CandidateReview/CandidateTestPlanRun/index.jsx

Lines changed: 153 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,12 @@ import createIssueLink, {
2929
import RunHistory from '../../common/RunHistory';
3030
import { useUrlTestIndex } from '../../../hooks/useUrlTestIndex';
3131
import { evaluateAuth } from '../../../utils/evaluateAuth';
32+
import summarizeAssertions from '../../../utils/summarizeAssertions.js';
3233
import NotApprovedModal from '../CandidateModals/NotApprovedModal';
3334
import FailingAssertionsSummaryTable from '../../FailingAssertionsSummary/Table';
3435
import FailingAssertionsSummaryHeading from '../../FailingAssertionsSummary/Heading';
36+
import NegativeSideEffectsSummaryTable from '../../NegativeSideEffectsSummary/Table';
37+
import NegativeSideEffectsSummaryHeading from '../../NegativeSideEffectsSummary/Heading';
3538
import styles from './CandidateTestPlanRun.module.css';
3639
import feedbackStyles from '../FeedbackListItem/FeedbackListItem.module.css';
3740
import testRunStyles from '../../TestRun/TestRun.module.css';
@@ -49,6 +52,8 @@ const vendorReviewStatusMap = {
4952
IN_PROGRESS: 'In Progress',
5053
APPROVED: 'Approved'
5154
};
55+
const FAILING_ASSERTIONS_INDEX = -1;
56+
const NEGATIVE_SIDE_EFFECTS = -2;
5257

5358
const CandidateTestPlanRun = () => {
5459
const { atId, testPlanVersionId } = useParams();
@@ -62,7 +67,7 @@ const CandidateTestPlanRun = () => {
6267
const [viewedTests, setViewedTests] = useState([]);
6368
const [testsLength, setTestsLength] = useState(0);
6469
const [currentTestIndex, setCurrentTestIndex] = useUrlTestIndex({
65-
minTestIndex: -1,
70+
minTestIndex: -2,
6671
maxTestIndex: testsLength
6772
});
6873
const [showTestNavigator, setShowTestNavigator] = useState(true);
@@ -126,7 +131,9 @@ const CandidateTestPlanRun = () => {
126131
skip: !testPlanReport
127132
});
128133

129-
const isSummaryView = currentTestIndex === -1;
134+
const isSummaryView =
135+
currentTestIndex === FAILING_ASSERTIONS_INDEX ||
136+
currentTestIndex === NEGATIVE_SIDE_EFFECTS;
130137

131138
const isLaptopOrLarger = useMediaQuery({
132139
query: '(min-width: 792px)'
@@ -139,7 +146,7 @@ const CandidateTestPlanRun = () => {
139146
}, [data?.testPlanReports]);
140147
const handleTestClick = async index => {
141148
setCurrentTestIndex(index);
142-
if (index === -1) {
149+
if (index < 0) {
143150
// Summary view
144151
setIsFirstTest(false);
145152
setIsLastTest(false);
@@ -403,44 +410,56 @@ const CandidateTestPlanRun = () => {
403410
const fileBugUrl = AtBugTrackerMap[at];
404411

405412
const getHeading = () => {
413+
if (currentTestIndex === FAILING_ASSERTIONS_INDEX) {
414+
return (
415+
<>
416+
<span className={testRunStyles.taskLabel}>
417+
Candidate Test Plan Review
418+
</span>
419+
<FailingAssertionsSummaryHeading
420+
metrics={testPlanReport.metrics}
421+
as="h1"
422+
/>
423+
</>
424+
);
425+
}
426+
427+
if (currentTestIndex === NEGATIVE_SIDE_EFFECTS) {
428+
return (
429+
<>
430+
<span className={testRunStyles.taskLabel}>
431+
Candidate Test Plan Review
432+
</span>
433+
<NegativeSideEffectsSummaryHeading
434+
metrics={testPlanReport.metrics}
435+
as="h1"
436+
/>
437+
</>
438+
);
439+
}
440+
406441
return (
407-
<div className="p-0">
408-
{isSummaryView ? (
409-
<>
410-
<span className={testRunStyles.taskLabel}>
411-
Candidate Test Plan Review
412-
</span>
413-
<FailingAssertionsSummaryHeading
414-
metrics={testPlanReport.metrics}
415-
as="h1"
416-
/>
417-
</>
418-
) : (
419-
<>
420-
<div className="sr-only" aria-live="polite" aria-atomic="true">
421-
Viewing Test {currentTest.title}, Test {currentTest.seq} of{' '}
422-
{tests.length}
423-
{currentTest.seq === tests.length
424-
? 'You are on the last test.'
425-
: ''}
426-
</div>
427-
<span className={testRunStyles.taskLabel}>
428-
Reviewing Test {currentTest.seq} of {tests.length}:
429-
</span>
430-
<h1>
431-
{`${currentTest.seq}. ${currentTest.title}`}{' '}
432-
<span className={styles.using}>using</span> {`${at}`}{' '}
433-
{`${testPlanReport?.latestAtVersionReleasedAt?.name ?? ''}`}
434-
{viewedTests.includes(currentTest.id) && !firstTimeViewing && ' '}
435-
{viewedTests.includes(currentTest.id) && !firstTimeViewing && (
436-
<Badge className={styles.viewedBadge} pill variant="secondary">
437-
Previously Viewed
438-
</Badge>
439-
)}
440-
</h1>
441-
</>
442-
)}
443-
</div>
442+
<>
443+
<div className="sr-only" aria-live="polite" aria-atomic="true">
444+
Viewing Test {currentTest.title}, Test {currentTest.seq} of{' '}
445+
{tests.length}
446+
{currentTest.seq === tests.length ? 'You are on the last test.' : ''}
447+
</div>
448+
<span className={testRunStyles.taskLabel}>
449+
Reviewing Test {currentTest.seq} of {tests.length}:
450+
</span>
451+
<h1>
452+
{`${currentTest.seq}. ${currentTest.title}`}{' '}
453+
<span className={styles.using}>using</span> {`${at}`}{' '}
454+
{`${testPlanReport?.latestAtVersionReleasedAt?.name ?? ''}`}
455+
{viewedTests.includes(currentTest.id) && !firstTimeViewing && ' '}
456+
{viewedTests.includes(currentTest.id) && !firstTimeViewing && (
457+
<Badge className={styles.viewedBadge} pill variant="secondary">
458+
Previously Viewed
459+
</Badge>
460+
)}
461+
</h1>
462+
</>
444463
);
445464
};
446465

@@ -560,93 +579,100 @@ const CandidateTestPlanRun = () => {
560579
};
561580

562581
const getContent = () => {
582+
if (currentTestIndex === FAILING_ASSERTIONS_INDEX) {
583+
return (
584+
<div
585+
className={
586+
failingAssertionsSummaryStyles.failingAssertionsSummaryTableContainer
587+
}
588+
>
589+
<FailingAssertionsSummaryTable
590+
testPlanReport={testPlanReports[0]}
591+
atName={at}
592+
getLinkUrl={assertion => `#${assertion.testIndex + 1}`}
593+
/>
594+
</div>
595+
);
596+
}
597+
if (currentTestIndex === NEGATIVE_SIDE_EFFECTS) {
598+
return (
599+
<div
600+
className={
601+
failingAssertionsSummaryStyles.failingAssertionsSummaryTableContainer
602+
}
603+
>
604+
<NegativeSideEffectsSummaryTable
605+
testPlanReport={testPlanReports[0]}
606+
atName={at}
607+
getLinkUrl={assertion => `#${assertion.testIndex + 1}`}
608+
/>
609+
</div>
610+
);
611+
}
563612
return (
564-
<div className="p-0">
565-
{isSummaryView ? (
566-
<div
567-
className={
568-
failingAssertionsSummaryStyles.failingAssertionsSummaryTableContainer
569-
}
570-
>
571-
<FailingAssertionsSummaryTable
572-
testPlanReport={testPlanReports[0]}
573-
atName={at}
574-
getLinkUrl={assertion => `#${assertion.testIndex + 1}`}
613+
<>
614+
<h1 className="border-0" data-testid="current-test-title">
615+
{currentTest.title}
616+
</h1>
617+
<DisclosureComponent
618+
componentId="candidateReviewRun"
619+
headingLevel="1"
620+
title={[
621+
'Test Instructions',
622+
...testPlanReports.map(
623+
testPlanReport =>
624+
`Test Results for ${testPlanReport.browser.name}`
625+
),
626+
'Run History'
627+
]}
628+
onClick={[
629+
() => setShowInstructions(!showInstructions),
630+
...showBrowserClicks,
631+
() => setShowRunHistory(!showRunHistory)
632+
]}
633+
expanded={[showInstructions, ...showBrowserBools, showRunHistory]}
634+
disclosureContainerView={[
635+
<InstructionsRenderer
636+
customClassNames={
637+
styles.candidateReviewCustomInstructionsRenderer
638+
}
639+
key={`instructions-${currentTest.id}`}
640+
at={testPlanReport.at}
641+
test={currentTest}
642+
testPageUrl={testPlanReport.testPlanVersion.testPageUrl}
643+
testFormatVersion={testPlanVersion.metadata.testFormatVersion}
644+
/>,
645+
...testPlanReports.map(testPlanReport => {
646+
const testResult =
647+
testPlanReport.finalizedTestResults[currentTestIndex];
648+
649+
const assertionsSummary = summarizeAssertions(
650+
getMetrics({
651+
testResult
652+
})
653+
);
654+
655+
return (
656+
<>
657+
<h2 className={styles.testResultsHeader}>
658+
Test Results&nbsp;({assertionsSummary})
659+
</h2>
660+
<TestPlanResultsTable
661+
key={`${testPlanReport.id} + ${testResult.id}`}
662+
test={{ ...currentTest, at: { name: at } }}
663+
testResult={testResult}
664+
/>
665+
</>
666+
);
667+
}),
668+
<RunHistory
669+
key="run-history"
670+
testPlanReports={testPlanReports}
671+
testId={currentTest.id}
575672
/>
576-
</div>
577-
) : (
578-
<>
579-
<h1 className="border-0" data-testid="current-test-title">
580-
{currentTest.title}
581-
</h1>
582-
<DisclosureComponent
583-
componentId="candidateReviewRun"
584-
headingLevel="1"
585-
title={[
586-
'Test Instructions',
587-
...testPlanReports.map(
588-
testPlanReport =>
589-
`Test Results for ${testPlanReport.browser.name}`
590-
),
591-
'Run History'
592-
]}
593-
onClick={[
594-
() => setShowInstructions(!showInstructions),
595-
...showBrowserClicks,
596-
() => setShowRunHistory(!showRunHistory)
597-
]}
598-
expanded={[showInstructions, ...showBrowserBools, showRunHistory]}
599-
disclosureContainerView={[
600-
<InstructionsRenderer
601-
customClassNames={
602-
styles.candidateReviewCustomInstructionsRenderer
603-
}
604-
key={`instructions-${currentTest.id}`}
605-
at={testPlanReport.at}
606-
test={currentTest}
607-
testPageUrl={testPlanReport.testPlanVersion.testPageUrl}
608-
testFormatVersion={testPlanVersion.metadata.testFormatVersion}
609-
/>,
610-
...testPlanReports.map(testPlanReport => {
611-
const testResult =
612-
testPlanReport.finalizedTestResults[currentTestIndex];
613-
614-
const {
615-
assertionsPassedCount,
616-
mustAssertionsFailedCount,
617-
shouldAssertionsFailedCount,
618-
mayAssertionsFailedCount
619-
} = getMetrics({ testResult });
620-
621-
const mustShouldAssertionsFailedCount =
622-
mustAssertionsFailedCount + shouldAssertionsFailedCount;
623-
624-
return (
625-
<>
626-
<h2 className={styles.testResultsHeader}>
627-
Test Results&nbsp;(
628-
{assertionsPassedCount} passed,&nbsp;
629-
{mustShouldAssertionsFailedCount} failed,&nbsp;
630-
{mayAssertionsFailedCount} unsupported)
631-
</h2>
632-
<TestPlanResultsTable
633-
key={`${testPlanReport.id} + ${testResult.id}`}
634-
test={{ ...currentTest, at: { name: at } }}
635-
testResult={testResult}
636-
/>
637-
</>
638-
);
639-
}),
640-
<RunHistory
641-
key="run-history"
642-
testPlanReports={testPlanReports}
643-
testId={currentTest.id}
644-
/>
645-
]}
646-
></DisclosureComponent>
647-
</>
648-
)}
649-
</div>
673+
]}
674+
></DisclosureComponent>
675+
</>
650676
);
651677
};
652678

@@ -668,14 +694,14 @@ const CandidateTestPlanRun = () => {
668694
/>
669695
<Col id="main" as="main" tabIndex="-1">
670696
<Row>
671-
{getHeading()}
697+
<div className="p-0">{getHeading()}</div>
672698
{getTestInfo()}
673699
<Col className="p-0">
674700
<Row xs={1} s={1} md={2}>
675701
<Col className="p-0" md={isLaptopOrLarger ? 9 : 12}>
676702
<Row>{getFeedback()}</Row>
677703
<Row className={styles.candidateResultsContainer}>
678-
{getContent()}
704+
<div className="p-0">{getContent()}</div>
679705
</Row>
680706
<Row>
681707
<ul

‎client/components/NegativeSideEffectsSummary/Heading.jsx

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import React from 'react';
2+
import { TestPlanReportMetricsPropType } from '../common/proptypes';
3+
import PropTypes from 'prop-types';
4+
5+
const NegativeSideEffectsSummaryHeading = ({ metrics, as: Element = 'h2' }) => {
6+
return (
7+
<Element id="negative-side-effects-heading">
8+
Summary of Negative Side Effects (
9+
{metrics.severeImpactFailedAssertionCount} severe,{' '}
10+
{metrics.moderateImpactFailedAssertionCount} moderate)
11+
</Element>
12+
);
13+
};
14+
15+
NegativeSideEffectsSummaryHeading.propTypes = {
16+
metrics: TestPlanReportMetricsPropType.isRequired,
17+
as: PropTypes.elementType
18+
};
19+
20+
export default NegativeSideEffectsSummaryHeading;

0 commit comments

Comments
 (0)