Clean up scanner tests

This commit is contained in:
Mike Cifelli 2018-07-22 11:10:00 -04:00
parent e60f253552
commit cb0e8a1d15
5 changed files with 72 additions and 103 deletions

View File

@ -2,7 +2,7 @@ package scanner
import error.CriticalLispException import error.CriticalLispException
abstract class LispInputStream: AbstractIterator<Int>() { abstract class LispInputStream : AbstractIterator<Int>() {
abstract fun read(): Int abstract fun read(): Int
abstract fun unreadLastCharacter() abstract fun unreadLastCharacter()

View File

@ -5,15 +5,12 @@ import file.FilePosition
import file.FilePositionTracker import file.FilePositionTracker
import token.Token import token.Token
import token.TokenFactoryImpl import token.TokenFactoryImpl
import util.Characters
import util.Characters.BACKSLASH import util.Characters.BACKSLASH
import util.Characters.DOUBLE_QUOTE import util.Characters.DOUBLE_QUOTE
import util.Characters.EOF
import util.Characters.NEWLINE import util.Characters.NEWLINE
import util.Characters.isIdentifierCharacter import util.Characters.isIdentifierCharacter
import util.Characters.isNumberPrefix import util.Characters.isNumberPrefix
import java.io.InputStream import java.io.InputStream
import java.lang.Character.isDigit
import java.lang.Character.isWhitespace import java.lang.Character.isWhitespace
/** /**
@ -71,10 +68,10 @@ class LispScanner(inputStream: InputStream, fileName: String) {
ComplexTokenTextRetriever(firstDoubleQuote) { true }.retrieveToken() ComplexTokenTextRetriever(firstDoubleQuote) { true }.retrieveToken()
private fun retrieveNumberTokenText(firstCharacter: Char) = private fun retrieveNumberTokenText(firstCharacter: Char) =
ComplexTokenTextRetriever(firstCharacter, Character::isDigit).retrieveToken() ComplexTokenTextRetriever(firstCharacter) { it.isDigit() }.retrieveToken()
private fun retrieveIdentifierTokenText(firstCharacter: Char) = private fun retrieveIdentifierTokenText(firstCharacter: Char) =
ComplexTokenTextRetriever(firstCharacter, Characters::isIdentifierCharacter).retrieveToken() ComplexTokenTextRetriever(firstCharacter) { isIdentifierCharacter(it) }.retrieveToken()
private inner class ComplexTokenTextRetriever(private val firstCharacter: Char, private inner class ComplexTokenTextRetriever(private val firstCharacter: Char,
private val isPartOfToken: (Char) -> Boolean) { private val isPartOfToken: (Char) -> Boolean) {

View File

@ -1,144 +1,127 @@
package scanner package scanner
import file.FilePosition import file.FilePosition
import org.junit.Test import org.assertj.core.api.Assertions.assertThat
import token.Token import org.junit.jupiter.api.Test
import java.io.InputStream
import org.junit.Assert.assertTrue
import testutil.TestUtilities.createInputStreamFromString import testutil.TestUtilities.createInputStreamFromString
class LispScannerLineColumnTest { class LispScannerLineColumnTest {
private class LineColumn { companion object {
const val TEST_FILE = "testFile"
private var line: Int = 0
private var column: Int = 0
fun isEqual(position: FilePosition): Boolean {
return this.line == position.lineNumber && this.column == position.columnNumber
}
companion object {
fun create(line: Int, column: Int): LineColumn {
val lineColumn = LineColumn()
lineColumn.line = line
lineColumn.column = column
return lineColumn
}
}
} }
private data class LineColumn(val line: Int = 0, val column: Int = 0)
private fun assertTokenLineAndColumnsMatch(input: String, expectedLineColumnList: Array<LineColumn>) { private fun assertTokenLineAndColumnsMatch(input: String, expectedLineColumnList: Array<LineColumn>) {
val stringInputStream = createInputStreamFromString(input) val stringInputStream = createInputStreamFromString(input)
val lispScanner = LispScanner(stringInputStream, "testFile") val lispScanner = LispScanner(stringInputStream, TEST_FILE)
for (lineColumn in expectedLineColumnList) { for (lineColumn in expectedLineColumnList) {
val nextToken = lispScanner.nextToken val position = lispScanner.nextToken.position
assertTrue(lineColumn.isEqual(nextToken.position)) val expectedPosition = FilePosition(TEST_FILE, lineColumn.line, lineColumn.column)
assertThat(position).isEqualTo(expectedPosition)
} }
} }
@Test @Test
fun givenNothing_RecordsCorrectEofLocation() { fun `location of EOF`() {
val input = "" val input = ""
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 0)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 0))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenSimpleString_RecordsCorrectLocation() { fun `location of a string`() {
val input = "\"string\"" val input = "\"string\""
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenStringWithTrailingSpace_RecordsCorrectLocation() { fun `location of a string with a trailing space`() {
val input = "\"string\" " val input = "\"string\" "
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenIdentifier_RecordsCorrectLocation() { fun `location of an identifier`() {
val input = "identifier" val input = "identifier"
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenIdentifierWithTrailingSpace_RecordsCorrectLocation() { fun `location of an identifier with a trailing space`() {
val input = "identifier " val input = "identifier "
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenNumber_RecordsCorrectLocation() { fun `location of a number`() {
val input = "123456789" val input = "123456789"
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenNumberWithTrailingSpace_RecordsCorrectLocation() { fun `location of a number with a trailing space`() {
val input = "123456789 " val input = "123456789 "
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenMultipleStrings_RecordsCorrectLocations() { fun `locations of multiple strings`() {
val input = "\"string1\" \n \"string2 \n with newline\" \n \"string3\"" val input = "\"string1\" \n \"string2 \n with newline\" \n \"string3\""
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1), LineColumn.create(2, 2), LineColumn.create(4, 3)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1), LineColumn(2, 2), LineColumn(4, 3))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenQuotedList_RecordsCorrectLocations() { fun `locations of a quoted list`() {
val input = "'(1 2 3 4 5)" val input = "'(1 2 3 4 5)"
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1), val expectedLinesAndColumns = arrayOf(LineColumn(1, 1),
LineColumn.create(1, 2), LineColumn(1, 2),
LineColumn.create(1, 3), LineColumn(1, 3),
LineColumn.create(1, 5), LineColumn(1, 5),
LineColumn.create(1, 7), LineColumn(1, 7),
LineColumn.create(1, 9), LineColumn(1, 9),
LineColumn.create(1, 11), LineColumn(1, 11),
LineColumn.create(1, 12)) LineColumn(1, 12))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenListSpanningMultipleLines_RecordsCorrectLocations() { fun `locations of a list spanning multiple lines`() {
val input = " ( 1 2 \n 3 4 \n5 ) " val input = " ( 1 2 \n 3 4 \n5 ) "
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 2), val expectedLinesAndColumns = arrayOf(LineColumn(1, 2),
LineColumn.create(1, 4), LineColumn(1, 4),
LineColumn.create(1, 6), LineColumn(1, 6),
LineColumn.create(2, 2), LineColumn(2, 2),
LineColumn.create(2, 4), LineColumn(2, 4),
LineColumn.create(3, 1), LineColumn(3, 1),
LineColumn.create(3, 3)) LineColumn(3, 3))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }
@Test @Test
fun givenCommentImmediatelyFollowingNumber_RecordsCorrectLocations() { fun `locations of numbers separated by a comment`() {
val input = "12;comment\n34" val input = "12;comment\n34"
val expectedLinesAndColumns = arrayOf(LineColumn.create(1, 1), LineColumn.create(2, 1)) val expectedLinesAndColumns = arrayOf(LineColumn(1, 1), LineColumn(2, 1))
assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns) assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns)
} }

View File

@ -1,44 +1,36 @@
package scanner package scanner
import file.FilePosition import org.assertj.core.api.Assertions.assertThat
import org.junit.Test import org.junit.Test
import java.io.InputStream
import org.junit.Assert.assertEquals
import testutil.TestUtilities.createInputStreamFromString import testutil.TestUtilities.createInputStreamFromString
class LispScannerTextTest { class LispScannerTextTest {
private fun assertTokenTextMatches(input: String, expectedTextList: Array<String>) { private fun assertTokenTextMatches(input: String, expectedText: Array<String>) {
val lispScanner = createLispScanner(input) val lispScanner = createLispScanner(input)
for (expectedText in expectedTextList) for (expected in expectedText)
assertEquals(expectedText, lispScanner.nextToken.text) assertThat(lispScanner.nextToken.text).isEqualTo(expected)
} }
private fun assertTokenTextMatches(input: String, expectedText: String) { private fun assertTokenTextMatches(input: String, expected: String) {
val lispScanner = createLispScanner(input) val lispScanner = createLispScanner(input)
assertEquals(expectedText, lispScanner.nextToken.text) assertThat(lispScanner.nextToken.text).isEqualTo(expected)
} }
private fun createLispScanner(input: String): LispScanner { private fun createLispScanner(input: String) = LispScanner(createInputStreamFromString(input), "testFile")
val stringInputStream = createInputStreamFromString(input)
return LispScanner(stringInputStream, "testFile")
}
private fun assertInputFileNameMatches(input: String, expectedInputFileName: String) { private fun assertInputFileNameMatches(input: String, expectedInputFileName: String) {
val stringInputStream = createInputStreamFromString(input) val stringInputStream = createInputStreamFromString(input)
val lispScanner = LispScanner(stringInputStream, expectedInputFileName) val lispScanner = LispScanner(stringInputStream, expectedInputFileName)
val (fileName) = lispScanner.nextToken.position val (fileName) = lispScanner.nextToken.position
assertEquals(expectedInputFileName, fileName) assertThat(fileName).isEqualTo(expectedInputFileName)
} }
@Test @Test
fun givenEmptyStream_RecordsCorrectFileName() { fun `an empty stream records the correct file name`() {
val input = "" val input = ""
val expectedFileName = "testFileName" val expectedFileName = "testFileName"
@ -46,7 +38,7 @@ class LispScannerTextTest {
} }
@Test @Test
fun givenParenthesis_RecordsCorrectText() { fun `scan parenthesis`() {
val input = "()" val input = "()"
val expected = arrayOf("(", ")") val expected = arrayOf("(", ")")
@ -54,7 +46,7 @@ class LispScannerTextTest {
} }
@Test @Test
fun givenQuote_RecordsCorrectText() { fun `scan a quote`() {
val input = "'" val input = "'"
val expected = "'" val expected = "'"
@ -62,7 +54,7 @@ class LispScannerTextTest {
} }
@Test @Test
fun givenEof_ReordsCorrectText() { fun `scan EOF`() {
val input = "" val input = ""
val expected = "EOF" val expected = "EOF"
@ -70,28 +62,28 @@ class LispScannerTextTest {
} }
@Test @Test
fun givenIdentifier_RecordsCorrectText() { fun `scan an identifier`() {
val input = "identifier" val input = "identifier"
assertTokenTextMatches(input, input) assertTokenTextMatches(input, input)
} }
@Test @Test
fun givenNumber_RecordsCorrectText() { fun `scan a number`() {
val input = "192837456" val input = "192837456"
assertTokenTextMatches(input, input) assertTokenTextMatches(input, input)
} }
@Test @Test
fun givenString_RecordsCorrectText() { fun `scan a string`() {
val input = "\"String!!! \n More... \"" val input = "\"String!!! \n More... \""
assertTokenTextMatches(input, input) assertTokenTextMatches(input, input)
} }
@Test @Test
fun givenNumberFollowedByComment_RecordsCorrectText() { fun `scan a number followed by a comment`() {
val input = "192837456;comment" val input = "192837456;comment"
val expected = "192837456" val expected = "192837456"
@ -99,7 +91,7 @@ class LispScannerTextTest {
} }
@Test @Test
fun givenIdentifiersWithCommentBetween_RecordsCorrectText() { fun `scan identifiers separated by a comment`() {
val input = "abc123;comment\nabc222" val input = "abc123;comment\nabc222"
val expected = arrayOf("abc123", "abc222") val expected = arrayOf("abc123", "abc222")
@ -107,7 +99,7 @@ class LispScannerTextTest {
} }
@Test @Test
fun givenBackTickExpression_RecordsCorrectText() { fun `scan a back tick expression`() {
val input = "`(list ,a ,@b)" val input = "`(list ,a ,@b)"
val expected = arrayOf("`", "(", "list", ",", "a", ",", "@", "b", ")") val expected = arrayOf("`", "(", "list", ",", "a", ",", "@", "b", ")")

View File

@ -1,8 +1,13 @@
package scanner package scanner
import error.Severity.ERROR
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import org.junit.Before import org.junit.Before
import org.junit.Test import org.junit.Test
import scanner.LispScanner.UnterminatedStringException import scanner.LispScanner.UnterminatedStringException
import testutil.TestUtilities.createInputStreamFromString
import token.AtSign import token.AtSign
import token.Backquote import token.Backquote
import token.Comma import token.Comma
@ -15,16 +20,8 @@ import token.QuotedString
import token.RightParenthesis import token.RightParenthesis
import token.Token import token.Token
import token.TokenFactory.BadCharacterException import token.TokenFactory.BadCharacterException
import java.io.InputStream
import java.util.ArrayList import java.util.ArrayList
import error.Severity.ERROR
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import testutil.TestUtilities.createInputStreamFromString
class LispScannerTypeTest { class LispScannerTypeTest {
private var expectedTypes: MutableList<Class<out Token>>? = null private var expectedTypes: MutableList<Class<out Token>>? = null