Reorganise and test std::test

The built-in formatters have been moved to std::test::formatters, and
some dead code has been removed. Tests have been added as well, except
for cases where testing requires functionality not yet present such as
mocking or starting sub OS processes.
parent 7f0cb46b
Pipeline #41303143 passed with stages
in 15 minutes and 50 seconds
......@@ -8,7 +8,7 @@
import std::index::SetIndex
import std::process
import std::test::formatter::Formatter
import std::test::formatter::progress_formatter::ProgressFormatter
import std::test::formatters::ProgressFormatter
## The default number of tests to run concurrently.
let DEFAULT_CONCURRENCY = 32
......
#! Formatter for displaying test results similar to a progress bar.
#!
#! The produced output is inspired by existing test frameworks such as RSpec and
#! ExUnit. Output is colorised using ANSI escape sequences, though this can be
#! disabled if necessary.
import std::ansi
#! Built-in formatters for test suites.
import std::ansi::(self, BOLD, CYAN, GREEN, RED)
import std::conversion::ToString
import std::stdio::stdout
import std::string_buffer::StringBuffer
......@@ -18,6 +14,10 @@ let TIME_IN_SECONDS_THRESHOLD = 1.0
## A formatter that displays tests similar to a progress bar.
##
#! The produced output is inspired by existing test frameworks such as RSpec and
#! ExUnit. Output is colorised using ANSI escape sequences, though this can be
#! disabled if necessary.
##
## Passed tests are displayed as a ".", while failed tests are displayed as "F".
## All output is written to STDOUT.
object ProgressFormatter impl Formatter {
......@@ -135,34 +135,26 @@ object ProgressFormatter impl Formatter {
}
def green(string: ToString) -> String {
@colors.if true: {
ansi.green(string)
}, false: {
string
}
color(string: string, code: GREEN)
}
def red(string: ToString) -> String {
@colors.if true: {
ansi.red(string)
}, false: {
string
}
color(string: string, code: RED)
}
def cyan(string: ToString) -> String {
@colors.if true: {
ansi.cyan(string)
}, false: {
string
}
color(string: string, code: CYAN)
}
def bold(string: ToString) -> String {
color(string: string, code: BOLD)
}
def color(string: ToString, code: String) -> String {
@colors.if true: {
ansi.bold(string)
ansi.wrap(string: string, code: code)
}, false: {
string
string.to_string
}
}
}
......@@ -36,15 +36,6 @@ object RunnerState {
## The configuration details for this runner.
let @configuration = Configuration.new
## A boolean indicating if the runner should terminate after receiving a
## message.
let mut @terminate = False
}
## Returns `True` if the `Runner` this state belongs to should terminate.
def terminate? -> Boolean {
@terminate
}
## Adds a new `Test` to this `Runner`.
......@@ -155,14 +146,18 @@ object RunTests impl Command {
let duration = start_time.elapsed
failed.empty?.if_false {
failed.length.positive?.if_true {
state.formatter.failures(failed)
}
state.formatter.summary(tests: executed, duration: duration)
notify_client(failed.length.positive?)
}
def notify_client(failures: Boolean) {
let finished_message = RunnerFinished
.new(pid: process.current, failures: failed.empty?.not)
.new(pid: process.current, failures: failures)
process.send(pid: @notify, message: finished_message)
}
......@@ -210,13 +205,7 @@ object Runner {
## This method will not return until a command has instructed this Runner to
## terminate.
def run {
let command = @receiver.receive
command.run(@state)
@state.terminate?.if_true {
return
}
@receiver.receive.run(@state)
run
}
......@@ -239,7 +228,7 @@ object Client {
## Sets a configuration option to the given value.
def configure(option: String, value: Dynamic) {
@running.if_true {
running?.if_true {
process.panic(
'Configuration settings can not be changed for a running test suite'
)
......@@ -250,7 +239,7 @@ object Client {
## Adds a new test to the `Runner`.
def add_test(test: Test) {
@running.if_true {
running?.if_true {
process.panic('New tests can not be added to a running test suite')
}
......@@ -259,7 +248,7 @@ object Client {
## Schedules all tests for execution and waits for them to complete.
def run {
@running = True
mark_as_running
@sender.send(RunTests.new(process.current))
......@@ -277,6 +266,14 @@ object Client {
vm.exit(FAILURE_EXIT_STATUS)
}
}
def running? -> Boolean {
@running
}
def mark_as_running {
@running = True
}
}
## Starts a new `Runner` in a separate process.
......
......@@ -7,6 +7,13 @@ import test::std::fs::test_dir
import test::std::fs::test_file
import test::std::fs::test_path
import test::std::fs::test_raw
import test::std::test::test_assert
import test::std::test::test_config
import test::std::test::test_error::(self as test_test_error)
import test::std::test::test_formatters
import test::std::test::test_runner
import test::std::test::test_test
import test::std::test::test_test_group
import test::std::test_ansi
import test::std::test_array
import test::std::test_array_iter
......
import std::process
import std::test
import std::test::assert::(self, PanicResult, PanicTest)
def run_panic_test(block: lambda) -> PanicResult {
let test = PanicTest
.new(owner: process.current, block: block)
let sender = process.channel!(PanicTest) lambda (receiver) {
receiver.receive.run
}
sender.send(test)
process.receive as PanicResult
}
test.group('std::test::assert::PanicResult.panicked?') do (g) {
g.test('Checking if a test panicked') {
assert.false(PanicResult.new(panicked: False).panicked?)
assert.true(PanicResult.new(panicked: True).panicked?)
}
}
test.group('std::test::assert::PanicResult.error') do (g) {
g.test('Obtaining the message of a panic') {
assert.equal(PanicResult.new(panicked: False).error, Nil)
assert.equal(PanicResult.new(panicked: True, error: 'oops').error, 'oops')
}
}
test.group('std::test::assert::PanicTest.run') do (g) {
g.test('Running a panic test that panics') {
let result = run_panic_test lambda { process.panic('oops') }
assert.true(result.panicked?)
assert.equal(result.error, 'oops')
}
g.test('Running a panic test that does not panic') {
let result = run_panic_test lambda { 0 }
assert.false(result.panicked?)
}
}
test.group('std::test::assert.equal') do (g) {
g.test('Comparing two equal values') {
assert.equal(10, 10)
}
g.test('Comparing two different values') {
assert.panic {
assert.equal(10, 20)
}
}
}
test.group('std::test::assert.not_equal') do (g) {
g.test('Comparing two equal values') {
assert.panic {
assert.not_equal(10, 10)
}
}
g.test('Comparing two different values') {
assert.not_equal(10, 20)
}
}
test.group('std::test::assert.greater') do (g) {
g.test('Comparing two equal values') {
assert.panic {
assert.greater(10, 10)
}
}
g.test('Comparing a greater value with a smaller value') {
assert.greater(10, 5)
}
}
test.group('std::test::assert.panic') do (g) {
g.test('Checking if a block panics') {
assert.panic {
process.panic('oops')
}
}
}
test.group('std::test::assert.no_panic') do (g) {
g.test('Checking if a block does not panic') {
assert.no_panic {
10
}
}
}
test.group('std::test::assert.true') do (g) {
g.test('Checking if a value is truthy') {
assert.true(10 == 10)
}
}
test.group('std::test::assert.falsy') do (g) {
g.test('Checking if a value is falsy') {
assert.false(10 == 5)
}
}
import std::mirror
import std::test
import std::test::assert
import std::test::config::(Configuration, DEFAULT_CONCURRENCY)
import std::test::formatter::Formatter
import std::test::formatters::ProgressFormatter
import std::test::test::Test
import std::time::duration::Duration
object DummyFormatter impl Formatter {
def passed(test: Test) {}
def failed(test: Test) {}
def failures(test: Array!(Test)) {}
def summary(tests: Array!(Test), duration: Duration) {}
}
test.group('std::test::config::Configuration.concurrency') do (g) {
g.test('Obtaining the number of concurrent tests') {
assert.equal(Configuration.new.concurrency, DEFAULT_CONCURRENCY)
}
}
test.group('std::test::config::Configuration.formatter') do (g) {
g.test('Obtaining the formatter to use') {
let fmt = Configuration.new.formatter
let fmt_mirror = mirror.reflect_object(fmt)
assert.true(fmt_mirror.instance_of?(ProgressFormatter))
}
}
test.group('std::test::config::Configuration.[]=') do (g) {
g.test('Setting the concurrency level') {
let config = Configuration.new
config['concurrency'] = 2
assert.equal(config.concurrency, 2)
}
g.test('Setting the formatter to use') {
let config = Configuration.new
let fmt = DummyFormatter.new
config['formatter'] = fmt
assert.equal(config.formatter as DummyFormatter, fmt)
}
g.test('Setting an unsupported option') {
assert.panic {
let config = Configuration.new
config['foo'] = 'bar'
}
}
}
import std::debug::CallFrame
import std::test
import std::test::assert
import std::test::error::TestFailure
def test_failure -> TestFailure {
let frame = CallFrame.new(file: 'test.inko', name: 'test', line: 1)
TestFailure.new(message: 'oops', location: frame)
}
test.group('std::test::error::TestFailure.location') do (g) {
g.test('Obtaining the CallFrame of a TestFailure') {
let failure = test_failure
assert.equal(failure.location.file.to_string, 'test.inko')
assert.equal(failure.location.name, 'test')
assert.equal(failure.location.line, 1)
}
}
test.group('std::test::error::TestFailure.to_string') do (g) {
g.test('Converting a TestFailure to a String') {
assert.equal(test_failure.to_string, 'oops')
}
}
test.group('std::test::error::TestFailure.message') do (g) {
g.test('Obtaining the error message of a TestFailure') {
assert.equal(test_failure.message, 'oops')
}
}
import std::debug::CallFrame
import std::process
import std::string_buffer::StringBuffer
import std::test
import std::test::assert
import std::test::error::TestFailure
import std::test::formatters::ProgressFormatter
import std::test::test::Test
import std::time::duration
def example_test -> Test {
let frame = CallFrame.new(file: 'test.inko', name: 'test', line: 1)
let test = Test.new(
name: 'test name',
group_name: 'group name',
runner_pid: process.current,
location: frame,
body: lambda {}
)
}
test.group('std::test::formatters::ProgressFormatter.failure_title') do (g) {
g.test('Generating the title for a failed test') {
let fmt = ProgressFormatter.new
assert.equal(fmt.failure_title(example_test), 'group name test name')
}
}
test.group('std::test::formatters::ProgressFormatter.failure_location') do (g) {
g.test('Generating the failure location description for a failed test') {
let fmt = ProgressFormatter.new
let frame = CallFrame.new(file: 'test.inko', name: 'test', line: 1)
let failure = TestFailure.new(message: 'oops', location: frame)
assert.equal(fmt.failure_location(failure), 'test.inko:1')
}
}
test.group('std::test::formatters::ProgressFormatter.test_suite_duration_unit') do (g) {
g.test('Obtaining the time unit for the duration of a test suite') {
let fmt = ProgressFormatter.new
assert.equal(
fmt.test_suite_duration_unit(duration.from_milliseconds(5)),
'milliseconds'
)
assert.equal(
fmt.test_suite_duration_unit(duration.from_seconds(5)),
'seconds'
)
}
}
test.group('std::test::formatters::ProgressFormatter.test_suite_statistics') do (g) {
g.test('Formatting the test suite statistics') {
let fmt = ProgressFormatter.new(colors: False)
assert.equal(
fmt.test_suite_statistics([example_test]),
'1 tests, 0 failures '
)
}
}
test.group('std::test::formatters::ProgressFormatter.green') do (g) {
g.test('Generating a green string when colors are enabled') {
assert.equal(ProgressFormatter.new.green('hello'), "\e[32mhello\e[0m")
}
g.test('Generating a normal string when colors are disabled') {
assert.equal(ProgressFormatter.new(colors: False).green('hello'), 'hello')
}
g.test('Generating a normal string when colors are disabled using a StringBuffer') {
assert.equal(
ProgressFormatter.new(colors: False).green(StringBuffer.new(['hello'])),
'hello'
)
}
}
test.group('std::test::formatters::ProgressFormatter.red') do (g) {
g.test('Generating a red string when colors are enabled') {
assert.equal(ProgressFormatter.new.red('hello'), "\e[31mhello\e[0m")
}
g.test('Generating a normal string when colors are disabled') {
assert.equal(ProgressFormatter.new(colors: False).red('hello'), 'hello')
}
g.test('Generating a normal string when colors are disabled using a StringBuffer') {
assert.equal(
ProgressFormatter.new(colors: False).red(StringBuffer.new(['hello'])),
'hello'
)
}
}
test.group('std::test::formatters::ProgressFormatter.cyan') do (g) {
g.test('Generating a cyan string when colors are enabled') {
assert.equal(ProgressFormatter.new.cyan('hello'), "\e[36mhello\e[0m")
}
g.test('Generating a normal string when colors are disabled') {
assert.equal(ProgressFormatter.new(colors: False).cyan('hello'), 'hello')
}
g.test('Generating a normal string when colors are disabled using a StringBuffer') {
assert.equal(
ProgressFormatter.new(colors: False).cyan(StringBuffer.new(['hello'])),
'hello'
)
}
}
test.group('std::test::formatters::ProgressFormatter.bold') do (g) {
g.test('Generating a bold string when colors are enabled') {
assert.equal(ProgressFormatter.new.bold('hello'), "\e[1mhello\e[0m")
}
g.test('Generating a normal string when colors are disabled') {
assert.equal(ProgressFormatter.new(colors: False).bold('hello'), 'hello')
}
g.test('Generating a normal string when colors are disabled using a StringBuffer') {
assert.equal(
ProgressFormatter.new(colors: False).bold(StringBuffer.new(['hello'])),
'hello'
)
}
}
import std::debug::CallFrame
import std::mirror
import std::process::(self, Sender)
import std::test
import std::test::assert
import std::test::config::DEFAULT_CONCURRENCY
import std::test::formatters::ProgressFormatter
import std::test::runner::(
AddTest, Client, ConfigureRunner, RunTests, RunnerFinished, RunnerState
)
import std::test::test::Test
import std::time::duration
def example_test -> Test {
let frame = CallFrame.new(file: 'test.inko', name: 'test', line: 1)
Test.new(
name: 'test name',
group_name: 'group name',
runner_pid: process.current,
location: frame,
body: lambda {}
)
}
test.group('std::test::runner::RunnerState.add_test') do (g) {
g.test('Adding a test') {
let state = RunnerState.new
let test = example_test
state.add_test(test)
assert.equal(state.tests, [test])
}
}
test.group('std::test::runner::RunnerState.configuration') do (g) {
g.test('Obtaining the Configuration of a RunnerState') {
let state = RunnerState.new
assert.equal(state.configuration.concurrency, DEFAULT_CONCURRENCY)
}
}
test.group('std::test::runner::RunnerState.concurrency') do (g) {
g.test('Obtaining the concurrency of a RunnerState') {
let state = RunnerState.new
assert.equal(state.concurrency, DEFAULT_CONCURRENCY)
}
}
test.group('std::test::runner::RunnerState.formatter') do (g) {
g.test('Obtaining the formatter of a RunnerState') {
let state = RunnerState.new
let mirror = mirror.reflect_object(state.formatter)
assert.true(mirror.instance_of?(ProgressFormatter))
}
}
test.group('std::test::runner::ConfigureRunner.run') do (g) {
g.test('Configuring a test runner') {
let state = RunnerState.new
let command = ConfigureRunner.new(option: 'concurrency', value: 4)
command.run(state)
assert.equal(state.concurrency, 4)
}
}
test.group('std::test::runner::AddTest.run') do (g) {
g.test('Adding a test to a test suite') {
let state = RunnerState.new
let test = example_test
let command = AddTest.new(test)
command.run(state)
assert.equal(state.tests, [test])
}
}
test.group('std::test::runner::RunnerFinished.pid') do (g) {
g.test('Obtaining the PID of a runner') {
assert.equal(RunnerFinished.new(pid: 1, failures: False).pid, 1)
}
}
test.group('std::test::runner::RunnerFinished.failures?') do (g) {
g.test('Checking if a runner encountered any test failures') {
assert.false(RunnerFinished.new(pid: 1, failures: False).failures?)
assert.true(RunnerFinished.new(pid: 1, failures: True).failures?)
}
}
test.group('std::test::runner::Client.runner_pid') do (g) {
g.test('Obtaining the PID of the test runner') {
let client = Client.new(Sender.new(process.current))
assert.equal(client.runner_pid, process.current)
}
}
test.group('std::test::runner::Client.configure') do (g) {
g.test('Configuring a test runner that is not running') {
let client = Client.new(Sender.new(process.current))
let state = RunnerState.new
client.configure('concurrency', 4)
(process.receive as ConfigureRunner).run(state)
assert.equal(state.concurrency, 4)
}
g.test('Configuring a test runner that is already running') {
assert.panic {
let client = Client.new(Sender.new(process.current))
client.mark_as_running
client.configure('concurrency', 4)
}
}
}
test.group('std::test::runner::Client.add_test') do (g) {
g.test('Adding a test to a test runner that is not running') {
let client = Client.new(Sender.new(process.current))
let state = RunnerState.new
let test = example_test
client.add_test(test)
(process.receive as AddTest).run(state)
assert.equal(state.tests, [test])
}
g.test('Adding a test to a test runner that is running') {
assert.panic {
let client = Client.new(Sender.new(process.current))
client.mark_as_running
client.add_test(example_test)
}
}
}
test.group('std::test::runner::Client.run') do (g) {
g.test('Running a test suite') {
let sender = process.channel!(RunTests) lambda (receiver) {
receiver.receive.notify_client(failures: False)
}
let client = Client.new(sender)
client.run
assert.true(client.running?)
}
}
test.group('std::test::runner::Client.wait_for_tests') do (g) {
g.test('Wait for the tests to finish running') {
let sender = process.channel!(Integer) lambda (receiver) {
let parent = *receiver.receive
let result = RunnerFinished.new(pid: parent, failures: False)
let client = Client.new(Sender.new(parent))
process.send(pid: process.current, message: result)
client.wait_for_tests
process.send(pid: parent, message: True)
}
sender.send(process.current)
# If the client never stops waiting for tests, this receive will time out
# causing the below assertion to fail.
let finished = process.receive(duration.from_seconds(5)) as ?Boolean
assert.true(finished)
}
}
test.group('std::test::runner::Client.running?') do (g) {
g.test('Checking if a test runner is running') {
let client = Client.new(Sender.new(process.current))
assert.false(client.running?)
client.mark_as_running
assert.true(client.running?)
}
}
import std::debug::CallFrame
import std::process
import std::test
import std::test::assert
import std::test::test::Test
def example_test -> Test {
let frame = CallFrame.new(file: 'test.inko', name: 'test', line: 1)
Test.new(
name: 'test name',
group_name: 'group name',
runner_pid: process.current,
location: frame,
body: lambda { process.panic('This is a failure!') }
)
}
def run_example_test -> Test {
let sender = process.channel!(Test) lambda (receiver) {
receiver.receive.run
}
sender.send(example_test)
process.receive as Test
}
test.group('std::test::test::Test.name') do (g) {
g.test('Obtaining the name of the test') {
assert.equal(example_test.name, 'test name')
}
}
test.group('std::test::test::Test.group_name') do (g) {
g.test('Obtaining the name of the test group') {
assert.equal(example_test.group_name, 'group name')
}
}
test.group('std::test::test::Test.location') do (g) {
g.test('Obtaining the CallFrame of a test') {