/*
* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
* This source file is part of the Cangjie project, licensed under Apache-2.0
* with Runtime Library Exception.
*
* See https://cangjie-lang.cn/pages/LICENSE for license information.
*/
// The Cangjie API is in Beta. For details on its capabilities and limitations, please refer to the README file.
package std.unittest
import std.unittest.common.PrettyText
import std.unittest.mock.*
import std.collection.*
import std.collection.concurrent.LinkedBlockingQueue
import std.time.DateTime
import std.env.exit
class TestSuiteExecutor {
private let suiteConfiguration: Configuration
private let suiteInfo: TestSuiteReportInfo
TestSuiteExecutor(
private let externalConfiguration: Configuration,
private let suiteId: TestSuiteId,
private let testSuite: TestSuite,
private let filterService: FilterService
) {
this.suiteConfiguration = Configuration.merge(externalConfiguration, testSuite.wholeConfiguration)
this.suiteInfo = TestSuiteReportInfo(tags: this.suiteConfiguration.tags)
}
func execute(): TestSuiteResult {
let result = TestSuiteResult(suiteId, suiteInfo)
let testCases = testSuite.allCasesToBeExecuted(filterService, suiteId)
if (shallNotRunLifecycle(testCases)) {
for (testCase in testCases) {
result.add(tryRun(testCase))
}
result.finish()
return result
}
MockFramework.openSession(MockFramework.initSessionPrefix + testSuite.name, Stateless)
let beforeAllResult = runNonTestBodyStep(suiteId, suiteInfo, BeforeAll) {
testSuite.runBeforeAll()
}
result.add(beforeAllResult)
if (!beforeAllResult.hasFailures()) {
for (testCase in testCases) {
result.add(tryRun(testCase))
}
}
let afterAllResult = runNonTestBodyStep(suiteId, suiteInfo, AfterAll) {
testSuite.runAfterAll()
// collect hanging failures in this step so that they are not lost
Framework.withCurrentContext { ctx =>
for (unattached in Framework.collectUnattachedFailures()) {
ctx.storeCheckResult(unattached)
}
}
}
if (!beforeAllResult.hasFailures()) {
result.add(afterAllResult)
}
// making sure mock session close, but only reporting errors
// if there are no other error during teardown step
try {
MockFramework.closeSession()
} catch (e: Exception) {
// Note: for now we assumes that it should not throw exceptions
}
result.finish()
result
}
private func shallNotRunLifecycle(testCases: Collection<CaseOrBench>): Bool {
suiteConfiguration.skip || externalConfiguration.noRun || testCases.isEmpty() || allSkipped(testCases)
}
private func allSkipped(testCases: Collection<CaseOrBench>): Bool {
testCases |> all<CaseOrBench> { it => it.caseConfiguration.skip }
}
private func tryRun(testOrBench: CaseOrBench): TestCaseResult {
let caseId = TestCaseId(suiteId, testOrBench.name, isBench: testOrBench.isBench)
let caseInfo = TestCaseReportInfo(tags: testOrBench.caseConfiguration.tags)
let caseConfiguration = Configuration.merge(suiteConfiguration, testOrBench.caseConfiguration)
let caseKey = CaseFilterKey(caseId, tags: caseConfiguration.tags, isTopLevel: caseConfiguration.fromTopLevel)
let shouldSkip = suiteConfiguration.skip || caseConfiguration.skip ||
filterService.userFilter.shouldSkipTestCase(caseKey)
caseConfiguration.initRandom()
let caseRunner: (TestCaseResult) -> Unit = if (shouldSkip) {
runSkip
} else if (externalConfiguration.noRun) {
runNoRun
} else {
var testBody = runSingleCase(caseConfiguration, testOrBench.executor)
if (let Some(duration) <- caseConfiguration.timeout) {
// test body decorator if any time constraints are specified
testBody = runConsideringTimeout(caseConfiguration, duration, testBody)
}
testBody
}
runTestCaseStep(caseId, caseInfo, suiteInfo, caseConfiguration, caseRunner)
}
private static func runNonTestBodyStep(
suiteId: TestSuiteId, suiteInfo: TestSuiteReportInfo,
stepKind: LStep, stepBody: () -> Unit
): TestCaseResult {
Framework.onLifecycleStart(suiteId, suiteInfo, stepKind)
let id = TestCaseId(suiteId, stepKind.toString(), isBench: false)
let info = TestCaseReportInfo.empty()
let step = runLifecycleStep(stepKind, suiteId.suiteName, stepBody)
let result = TestCaseResult.fromSingleStep(id, info, step)
result.finish()
Framework.onFinished(result)
return result
}
private static func runLifecycleStep(step: LStep, testCaseName: String, stepBody: () -> Unit): RunStepResult {
Framework.runStepBody(Lifecycle(step), Test(0), testCaseName, stepBody)
}
private static func runTestCaseStep(
caseId: TestCaseId,
caseInfo: TestCaseReportInfo,
suiteInfo: TestSuiteReportInfo,
configuration: Configuration,
caseRunner: (TestCaseResult) -> Unit
): TestCaseResult {
Framework.onTestCaseStarted(caseId, caseInfo, suiteInfo)
let testCaseResult = TestCaseResult(caseId, caseInfo, RenderOptions.fromConfiguration(configuration))
caseRunner(testCaseResult)
testCaseResult.finish()
Framework.onFinished(testCaseResult)
return testCaseResult
}
func runSkip(testCaseResult: TestCaseResult): Unit {
testCaseResult.add(RunStepResult(0, DateTime.now(), StepKind.Skip, Test(0), duration: Duration.Zero))
}
func runNoRun(testCaseResult: TestCaseResult): Unit {
testCaseResult.add(RunStepResult(0, DateTime.now(), StepKind.NoRun, Test(0), duration: Duration.Zero))
}
func runSingleCase(caseConfiguration: Configuration, executor: Executor): (TestCaseResult) -> Unit {
return { result =>
let caseId = result.caseId
let beforeEachResult = runLifecycleStep(BeforeEach, caseId.caseName) {
testSuite.runBeforeEach(caseId.caseName)
}
result.add(beforeEachResult)
if (!beforeEachResult.hasFailures()) {
executor.execute(suiteInfo, result, caseConfiguration)
}
let afterEachResult = runLifecycleStep(AfterEach, caseId.caseName) {
testSuite.runAfterEach(caseId.caseName)
}
if (!result.hasFailures() && afterEachResult.hasFailures()) {
// NOTE: for now these failures are suppressed if there are existing errors
result.add(afterEachResult)
}
}
}
// TODO: as discussed timeout handling should limit whole test case execution
// itself rather than individual run execution
// we cannot properly implement property test that would shrink data that timed out
private static func runConsideringTimeout(
caseConfiguration: Configuration,
duration: Duration,
testBody: (TestCaseResult) -> Unit
): (TestCaseResult) -> Unit {
return { testCaseResult =>
let finishedBeforeTimeoutSignalQueue = LinkedBlockingQueue<Unit>(1)
let testingThread = spawn {
testBody(testCaseResult)
}
let killerThread = spawn {
if (let None <- finishedBeforeTimeoutSignalQueue.remove(duration)) {
onTimeout(caseConfiguration, testCaseResult, testingThread)
}
}
testingThread.get()
finishedBeforeTimeoutSignalQueue.add(())
killerThread.get()
}
}
private static func onTimeout(
caseConfiguration: Configuration,
testCaseResult: TestCaseResult,
testingThread: Future<Unit>
) {
if (TestProcessKind.fromDefaultConfiguration().isWorker) {
exitOnTimeout(testCaseResult)
} else {
// Executed via dynamic test API.
let suiteId = testCaseResult.caseId.suiteId
let info = TestCaseInfo(suiteId.groupName, suiteId.suiteName, testCaseResult.caseId.caseName)
caseConfiguration.timeoutHandler(info)
testingThread.cancel()
Framework.withCurrentContext { ctx: RunContext =>
// It is to complicated to distinquish between cancelled execution and ended by itself.
let msg = "Execution time exceeded specified timeout."
ctx.checkFailed(TimeoutCheckResult(PrettyText(msg)))
}
}
}
private static func exitOnTimeout(testCaseResult: TestCaseResult) {
Framework.withCurrentContext { ctx: RunContext =>
let msg = "Test case ended with timeout."
ctx.storeCheckResult(TimeoutCheckResult(PrettyText(msg)))
}
if (let Some(currentResult) <- Framework.abortCurrentStep()) {
testCaseResult.add(currentResult)
} else if (let Some(prev) <- testCaseResult.subResults.pop()) {
let newFailures = prev.checkResults |> concat(Framework.collectUnattachedFailures()) |> collectArray
let newResult = RunStepResult(prev.checksPassed, prev.startTimestamp, prev.kind, Failure(newFailures),
duration: prev.duration)
testCaseResult.add(newResult)
} else {
// this case is almost nonsensical, but technically it can happen if there is sleep in the first iteration of @Strategy
let failures = Framework.collectUnattachedFailures().toArray()
let newResult = RunStepResult(0, DateTime.now(), UserCode, Failure(failures))
testCaseResult.add(newResult)
}
testCaseResult.finish()
Framework.onFinished(testCaseResult)
exit(EXIT_CODE_ON_TIMEOUT)
}
}
extend TestSuite {
prop wholeConfiguration: Configuration {
get() {
Configuration.merge(this.template?.wholeConfiguration ?? Configuration(), this.suiteConfiguration)
}
}
prop allCasesIterator: Iterable<CaseOrBench> {
get() {
// Better run some basic tests from template before.
chain(template?.allCasesIterator ?? Array<CaseOrBench>().iterator(), cases)
}
}
func allCasesToBeExecuted(filterService: FilterService, suiteId: TestSuiteId): Array<CaseOrBench> {
allCasesIterator |> filter { cob: CaseOrBench =>
let caseKey = CaseFilterKey.fromTestCase(suiteId, cob, config: cob.caseConfiguration)
!filterService.frameworkFilter.shouldSkipTestCase(caseKey)
} |> collectArray
}
func casesCountToBeExecuted(filterService: FilterService, suiteId: TestSuiteId): Int64 {
allCasesToBeExecuted(filterService, suiteId).size
}
func runBeforeEach(caseShortName: String): Unit {
// Run template initializers before
this.template?.runBeforeEach(caseShortName)
for (c in beforeEachs) {
c(caseShortName)
}
}
func runAfterEach(caseShortName: String): Unit {
for (c in afterEachs) {
c(caseShortName)
}
// Run template cleanups after
this.template?.runAfterEach(caseShortName)
}
func runBeforeAll(): Unit {
// Run template initializers before
this.template?.runBeforeAll()
for (c in beforeAlls) {
c()
}
}
func runAfterAll(): Unit {
for (c in afterAlls) {
c()
}
// Run template cleanups after
this.template?.runAfterAll()
}
}
extend Configuration {
prop skip: Bool {
get() {
this.get(KeySkip.skip) ?? false
}
}
}