From 89bb52ab712f8afd5432da5a99582921bfdec9ef Mon Sep 17 00:00:00 2001 From: Mike Cifelli Date: Sat, 10 Dec 2016 11:57:49 -0500 Subject: [PATCH] Continued refactoring and adding unit tests --- src/{scanner => constructs}/Token.java | 12 +- src/constructs/TokenFactory.java | 11 ++ src/constructs/TokenFactoryImpl.java | 37 ++++ src/error/ErrorManager.java | 25 +-- src/error/LispException.java | 9 + src/file/FilePosition.java | 39 ++++ src/file/FilePositionTracker.java | 32 ++++ src/parser/LispParser.java | 10 +- src/scanner/LispScanner.java | 167 ++++++++---------- src/util/Characters.java | 23 +++ test/scanner/FilePositionTrackerTester.java | 118 +++++++++++++ test/scanner/LispScannerLineColumnTester.java | 44 ++++- test/scanner/LispScannerTextTester.java | 4 +- test/scanner/LispScannerTypeTester.java | 69 ++++++-- 14 files changed, 459 insertions(+), 141 deletions(-) rename src/{scanner => constructs}/Token.java (90%) create mode 100644 src/constructs/TokenFactory.java create mode 100644 src/constructs/TokenFactoryImpl.java create mode 100644 src/error/LispException.java create mode 100644 src/file/FilePosition.java create mode 100644 src/file/FilePositionTracker.java create mode 100644 test/scanner/FilePositionTrackerTester.java diff --git a/src/scanner/Token.java b/src/constructs/Token.java similarity index 90% rename from src/scanner/Token.java rename to src/constructs/Token.java index c7e413b..4416567 100644 --- a/src/scanner/Token.java +++ b/src/constructs/Token.java @@ -4,7 +4,9 @@ * Assignment: Lisp Interpreter Phase 1 - Lexical Analysis */ -package scanner; +package constructs; + +import file.FilePosition; /** * A Token represents a token in Common Lisp. @@ -59,12 +61,12 @@ public class Token { * @param column * the column number that this token is found on */ - public Token(Type type, String text, String fName, int line, int column) { + public Token(Type type, String text, FilePosition position) { this.type = type; this.text = text; - this.fName = fName; - this.line = line; - this.column = column; + this.fName = position.getFileName(); + this.line = position.getLineNumber(); + this.column = position.getColumnNumber(); } /** diff --git a/src/constructs/TokenFactory.java b/src/constructs/TokenFactory.java new file mode 100644 index 0000000..6b6fb3e --- /dev/null +++ b/src/constructs/TokenFactory.java @@ -0,0 +1,11 @@ +package constructs; + +import file.FilePosition; + +public interface TokenFactory { + + Token createToken(String text, FilePosition position); + + Token createEOFToken(FilePosition position); + +} diff --git a/src/constructs/TokenFactoryImpl.java b/src/constructs/TokenFactoryImpl.java new file mode 100644 index 0000000..980339a --- /dev/null +++ b/src/constructs/TokenFactoryImpl.java @@ -0,0 +1,37 @@ +package constructs; + +import static util.Characters.*; + +import file.FilePosition; +import util.Characters; + +public class TokenFactoryImpl implements TokenFactory { + + public Token createToken(String text, FilePosition position) { + char firstCharacter = text.charAt(0); + + switch (firstCharacter) { + case LEFT_PARENTHESIS: + return new Token(Token.Type.LEFT_PAREN, text, position); + case RIGHT_PARENTHESIS: + return new Token(Token.Type.RIGHT_PAREN, text, position); + case SINGLE_QUOTE: + return new Token(Token.Type.QUOTE_MARK, text, position); + case DOUBLE_QUOTE: + return new Token(Token.Type.STRING, text, position); + default: + if (Character.isDigit(firstCharacter)) { + return new Token(Token.Type.NUMBER, text, position); + } else if (Characters.isLegalIdentifierCharacter(firstCharacter)) { + return new Token(Token.Type.IDENTIFIER, text, position); + } + } + + throw new RuntimeException("oh no!"); + } + + public Token createEOFToken(FilePosition position) { + return new Token(Token.Type.EOF, "EOF", position); + } + +} diff --git a/src/error/ErrorManager.java b/src/error/ErrorManager.java index 255c61d..c9a599d 100644 --- a/src/error/ErrorManager.java +++ b/src/error/ErrorManager.java @@ -1,22 +1,12 @@ -/* - * Name: Mike Cifelli - * Course: CIS 443 - Programming Languages - * Assignment: Lisp Interpreter Phase 1 - Lexical Analysis - */ - package error; import java.text.MessageFormat; /** - * ErrorManager is an error handling class for a Lisp interpreter. + * Prints error messages. */ public class ErrorManager { - /** - * The lowest "criticality" level of an error that will cause the currently - * running program to terminate. - */ public static final int CRITICAL_LEVEL = 3; public static final String ANSI_RESET = "\u001B[0m"; @@ -25,16 +15,15 @@ public class ErrorManager { public static final String ANSI_PURPLE = "\u001B[35m"; /** - * Prints out the specified error message to the console and decides - * whether or not to terminate the currently running program. + * Prints out the specified error message to the console and decides whether + * or not to terminate the currently running program. * * @param message - * the error message + * the error message * @param level - * the "criticality" level of the error - * @postcondition - * If level >= CRITICAL_LEVEL the currently running - * program has been terminated. + * the "criticality" level of the error + * @postcondition If level >= CRITICAL_LEVEL the currently + * running program has been terminated. */ public static void generateError(String message, int level) { String color = (level >= CRITICAL_LEVEL) ? ANSI_PURPLE : ANSI_RED; diff --git a/src/error/LispException.java b/src/error/LispException.java new file mode 100644 index 0000000..4b63e9f --- /dev/null +++ b/src/error/LispException.java @@ -0,0 +1,9 @@ +package error; + +public abstract class LispException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public abstract int getSeverity(); + +} diff --git a/src/file/FilePosition.java b/src/file/FilePosition.java new file mode 100644 index 0000000..f8576e7 --- /dev/null +++ b/src/file/FilePosition.java @@ -0,0 +1,39 @@ +package file; + +import java.util.Objects; + +public class FilePosition { + + private String fileName; + private int lineNumber; + private int columnNumber; + + public FilePosition(String fileName) { + this.fileName = fileName; + } + + public String getFileName() { + return fileName; + } + + public int getLineNumber() { + return lineNumber; + } + + public void setLineNumber(int lineNumber) { + this.lineNumber = lineNumber; + } + + public int getColumnNumber() { + return columnNumber; + } + + public void setColumnNumber(int columnNumber) { + this.columnNumber = columnNumber; + } + + public boolean isEqual(FilePosition otherFilePosition) { + return Objects.equals(this.fileName, otherFilePosition.fileName) && (this.lineNumber == otherFilePosition.lineNumber) + && (this.columnNumber == otherFilePosition.columnNumber); + } +} \ No newline at end of file diff --git a/src/file/FilePositionTracker.java b/src/file/FilePositionTracker.java new file mode 100644 index 0000000..eea5156 --- /dev/null +++ b/src/file/FilePositionTracker.java @@ -0,0 +1,32 @@ +package file; + +public class FilePositionTracker { + + private String fileName; + private int lineNumber; + private int columnNumber; + + public FilePositionTracker(String fileName) { + this.fileName = fileName; + this.lineNumber = 1; + this.columnNumber = 0; + } + + public FilePosition getCurrentPosition() { + FilePosition currentPosition = new FilePosition(fileName); + currentPosition.setLineNumber(lineNumber); + currentPosition.setColumnNumber(columnNumber); + + return currentPosition; + } + + public void incrementColumn() { + columnNumber++; + } + + public void incrementLine() { + lineNumber++; + columnNumber = 0; + } + +} \ No newline at end of file diff --git a/src/parser/LispParser.java b/src/parser/LispParser.java index 89dfd60..f93e11a 100644 --- a/src/parser/LispParser.java +++ b/src/parser/LispParser.java @@ -9,6 +9,8 @@ package parser; import scanner.*; import java.io.*; +import constructs.Token; + /** * A LispParser converts a stream of bytes into internal * representations of Lisp S-expressions. When the end of stream has been @@ -57,7 +59,7 @@ public class LispParser { // attempt to read the next token from 'scanner' and store it in // 'nextToken' try { - nextToken = scanner.nextToken(); + nextToken = scanner.getNextToken(); nextTokenStored = true; } catch (Exception e) { // this method should give the illusion of not actually reading @@ -115,7 +117,7 @@ public class LispParser { // the next token has not been stored in 'nextToken' by the 'eof' // method - nextToken = scanner.nextToken(); + nextToken = scanner.getNextToken(); } else { // the 'eof' method has been called and has read in the next token // already to determine if we have reached the end-of-file @@ -145,7 +147,7 @@ public class LispParser { case STRING: return new LispString(nextToken.getText()); case QUOTE_MARK: - nextToken = scanner.nextToken(); + nextToken = scanner.getNextToken(); SExpression arg = sExpr(); return new Cons(new Symbol("QUOTE"), @@ -173,7 +175,7 @@ public class LispParser { // Throws: IOException - Indicates that an I/O error has occurred. // Precondition: 'scanner' is not null. private SExpression sExprTail() throws IOException { - nextToken = scanner.nextToken(); + nextToken = scanner.getNextToken(); // determine the type of 'nextToken' and create the appropriate // S-expression diff --git a/src/scanner/LispScanner.java b/src/scanner/LispScanner.java index c568253..c8058da 100644 --- a/src/scanner/LispScanner.java +++ b/src/scanner/LispScanner.java @@ -1,99 +1,72 @@ package scanner; -import static util.Characters.BACKSLASH; -import static util.Characters.DOUBLE_QUOTE; -import static util.Characters.EOF; -import static util.Characters.HASH; -import static util.Characters.LEFT_PARENTHESIS; -import static util.Characters.LEFT_SQUARE_BRACKET; -import static util.Characters.NEWLINE; -import static util.Characters.PERIOD; -import static util.Characters.RIGHT_PARENTHESIS; -import static util.Characters.RIGHT_SQUARE_BRACKET; -import static util.Characters.SEMICOLON; -import static util.Characters.SINGLE_QUOTE; -import static util.Characters.TICK_MARK; +import static util.Characters.*; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; +import java.text.MessageFormat; + +import constructs.Token; +import constructs.TokenFactory; +import constructs.TokenFactoryImpl; +import error.LispException; +import file.FilePosition; +import file.FilePositionTracker; +import util.Characters; /** * Converts a stream of bytes into a stream of Lisp tokens. */ public class LispScanner { - private static Map illegalIdentifierCharacters = new HashMap<>(); - - static { - illegalIdentifierCharacters.put(DOUBLE_QUOTE, true); - illegalIdentifierCharacters.put(SINGLE_QUOTE, true); - illegalIdentifierCharacters.put(BACKSLASH, true); - illegalIdentifierCharacters.put(TICK_MARK, true); - illegalIdentifierCharacters.put(LEFT_PARENTHESIS, true); - illegalIdentifierCharacters.put(RIGHT_PARENTHESIS, true); - illegalIdentifierCharacters.put(LEFT_SQUARE_BRACKET, true); - illegalIdentifierCharacters.put(RIGHT_SQUARE_BRACKET, true); - illegalIdentifierCharacters.put(HASH, true); - illegalIdentifierCharacters.put(PERIOD, true); - illegalIdentifierCharacters.put(SEMICOLON, true); - } - private InputStream inputStream; - private String inputStreamName; - private int lineNumber; - private int columnNumber; + private FilePositionTracker positionTracker; + private TokenFactory tokenFactory; public LispScanner(InputStream in, String fileName) { this.inputStream = new LispFilterInputStream(new BufferedInputStream(in)); - this.inputStreamName = fileName; - this.lineNumber = 1; - this.columnNumber = 0; + this.positionTracker = new FilePositionTracker(fileName); + this.tokenFactory = new TokenFactoryImpl(); } - public Token nextToken() throws IOException { + public Token getNextToken() throws IOException { int c; while ((c = inputStream.read()) != EOF) { char nextChar = (char) c; + positionTracker.incrementColumn(); - ++columnNumber; + if (Character.isWhitespace(nextChar)) { + if (nextChar == NEWLINE) + positionTracker.incrementLine(); + } else { + FilePosition currentPosition = positionTracker.getCurrentPosition(); + String tokenText = retrieveTokenText(nextChar); - switch (nextChar) { - case NEWLINE: - moveToNewLine(); - break; - case LEFT_PARENTHESIS: - return new Token(Token.Type.LEFT_PAREN, "(", inputStreamName, lineNumber, columnNumber); - case RIGHT_PARENTHESIS: - return new Token(Token.Type.RIGHT_PAREN, ")", inputStreamName, lineNumber, columnNumber); - case SINGLE_QUOTE: - return new Token(Token.Type.QUOTE_MARK, "\'", inputStreamName, lineNumber, columnNumber); - case DOUBLE_QUOTE: - return retrieveString(nextChar); - default: - if (Character.isWhitespace(nextChar)) { - continue; - } else if (Character.isDigit(nextChar)) { - return retrieveNumber(nextChar); - } else if (isLegalIdentifierCharacter(nextChar)) { - return retrieveIdentifier(nextChar); - } else { - throw new RuntimeException("illegal character " + "\'" + nextChar + "\'" + " - line " + lineNumber - + " column " + columnNumber); - } + return tokenFactory.createToken(tokenText, currentPosition); } } - return new Token(Token.Type.EOF, "EOF", inputStreamName, lineNumber, columnNumber); + return tokenFactory.createEOFToken(positionTracker.getCurrentPosition()); } - private Token retrieveString(char firstDoubleQuote) throws IOException { + private String retrieveTokenText(char firstCharacter) throws IOException { + String tokenText = "" + firstCharacter; + + if (firstCharacter == DOUBLE_QUOTE) + tokenText = retrieveString(firstCharacter); + else if (Character.isDigit(firstCharacter)) + tokenText = retrieveNumber(firstCharacter); + else if (Characters.isLegalIdentifierCharacter(firstCharacter)) + tokenText = retrieveIdentifier(firstCharacter); + + return tokenText; + } + + private String retrieveString(char firstDoubleQuote) throws IOException { StringBuilder text = new StringBuilder(); - int startLine = lineNumber; - int startColumn = columnNumber; + FilePosition stringPosition = positionTracker.getCurrentPosition(); char prevChar = firstDoubleQuote; text.append(firstDoubleQuote); @@ -103,18 +76,18 @@ public class LispScanner { while ((c = inputStream.read()) != EOF) { char nextChar = (char) c; - ++columnNumber; + positionTracker.incrementColumn(); text.append(nextChar); switch (nextChar) { case NEWLINE: - moveToNewLine(); + positionTracker.incrementLine(); break; case DOUBLE_QUOTE: if (prevChar != BACKSLASH) { // we have found the terminating double quote - return new Token(Token.Type.STRING, text.toString(), inputStreamName, startLine, startColumn); + return text.toString(); } // this is an escaped double quote @@ -123,15 +96,11 @@ public class LispScanner { prevChar = nextChar; } - // the end of 'inStream' was reached before the terminating double - // quote - - throw new RuntimeException("unterminated quoted string" + " - line " + startLine + " column " + startColumn); + throw new UnterminatedStringException(stringPosition); } - private Token retrieveNumber(char firstDigit) throws IOException { + private String retrieveNumber(char firstDigit) throws IOException { StringBuilder text = new StringBuilder(); - int startColumn = columnNumber; text.append(firstDigit); inputStream.mark(1); @@ -145,27 +114,23 @@ public class LispScanner { // 'nextChar' is a digit in this number text.append(nextChar); - ++columnNumber; + positionTracker.incrementColumn(); } else { // we have reached the end of the number inputStream.reset(); // unread the last character - return new Token(Token.Type.NUMBER, text.toString(), inputStreamName, lineNumber, startColumn); + return text.toString(); } inputStream.mark(1); } - // there are no more bytes to be read from 'inStream' after this number - // token - - return new Token(Token.Type.NUMBER, text.toString(), inputStreamName, lineNumber, startColumn); + return text.toString(); } - private Token retrieveIdentifier(char firstChar) throws IOException { + private String retrieveIdentifier(char firstChar) throws IOException { StringBuilder text = new StringBuilder(); - int startColumn = columnNumber; text.append(firstChar); inputStream.mark(1); @@ -175,35 +140,45 @@ public class LispScanner { while ((c = inputStream.read()) != EOF) { char nextChar = (char) c; - if (isLegalIdentifierCharacter(nextChar)) { + if (Characters.isLegalIdentifierCharacter(nextChar)) { // 'nextChar' is part of the identifier text.append(nextChar); - ++columnNumber; + positionTracker.incrementColumn(); } else { // we have reached the end of this identifier inputStream.reset(); // unread the last character - return new Token(Token.Type.IDENTIFIER, text.toString(), inputStreamName, lineNumber, startColumn); + return text.toString(); } inputStream.mark(1); } - // there are no more bytes to be read from 'inStream' after this - // identifier token - - return new Token(Token.Type.IDENTIFIER, text.toString(), inputStreamName, lineNumber, startColumn); + return text.toString(); } - private void moveToNewLine() { - lineNumber++; - columnNumber = 0; - } + public static class UnterminatedStringException extends LispException { + + private static final long serialVersionUID = 1L; + private FilePosition position; + + public UnterminatedStringException(FilePosition position) { + this.position = position; + } + + @Override + public int getSeverity() { + return 0; + } + + @Override + public String getMessage() { + return MessageFormat.format("unterminated quoted string - line {0}, column {1}", position.getLineNumber(), + position.getColumnNumber()); + } - private boolean isLegalIdentifierCharacter(char c) { - return (!Character.isWhitespace(c)) && (illegalIdentifierCharacters.get(c) == null); } } diff --git a/src/util/Characters.java b/src/util/Characters.java index 9c6c731..c0b0970 100644 --- a/src/util/Characters.java +++ b/src/util/Characters.java @@ -1,5 +1,8 @@ package util; +import java.util.HashMap; +import java.util.Map; + public class Characters { public static final char BACKSLASH = '\\'; @@ -16,5 +19,25 @@ public class Characters { public static final char TICK_MARK = '`'; public static final int EOF = -1; + + public static final Map illegalIdentifierCharacters = new HashMap<>(); + + static { + illegalIdentifierCharacters.put(DOUBLE_QUOTE, true); + illegalIdentifierCharacters.put(SINGLE_QUOTE, true); + illegalIdentifierCharacters.put(BACKSLASH, true); + illegalIdentifierCharacters.put(TICK_MARK, true); + illegalIdentifierCharacters.put(LEFT_PARENTHESIS, true); + illegalIdentifierCharacters.put(RIGHT_PARENTHESIS, true); + illegalIdentifierCharacters.put(LEFT_SQUARE_BRACKET, true); + illegalIdentifierCharacters.put(RIGHT_SQUARE_BRACKET, true); + illegalIdentifierCharacters.put(HASH, true); + illegalIdentifierCharacters.put(PERIOD, true); + illegalIdentifierCharacters.put(SEMICOLON, true); + } + + public static boolean isLegalIdentifierCharacter(char c) { + return (! Character.isWhitespace(c)) && (! illegalIdentifierCharacters.containsKey(c)); + } } diff --git a/test/scanner/FilePositionTrackerTester.java b/test/scanner/FilePositionTrackerTester.java new file mode 100644 index 0000000..b2b5b65 --- /dev/null +++ b/test/scanner/FilePositionTrackerTester.java @@ -0,0 +1,118 @@ +package scanner; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; + +import file.FilePosition; +import file.FilePositionTracker; + +public class FilePositionTrackerTester { + + public static final String FILE_NAME = "testFileName"; + + private FilePositionTracker trackerUnderTest; + + @Before + public void setUp() throws Exception { + trackerUnderTest = new FilePositionTracker(FILE_NAME); + } + + @Test + public void filePositionEquality_CorrectlyReturnsTrue() { + FilePosition positionOne = new FilePosition(FILE_NAME); + positionOne.setLineNumber(5); + positionOne.setColumnNumber(9); + + FilePosition positionTwo = new FilePosition(FILE_NAME); + positionTwo.setLineNumber(5); + positionTwo.setColumnNumber(9); + + assertTrue(positionOne.isEqual(positionTwo)); + } + + @Test + public void filePositionEquality_CorrectlyReturnsFalseWithDifferentLine() { + FilePosition positionOne = new FilePosition(FILE_NAME); + positionOne.setLineNumber(5); + positionOne.setColumnNumber(9); + + FilePosition positionTwo = new FilePosition(FILE_NAME); + positionTwo.setLineNumber(8); + positionTwo.setColumnNumber(9); + + assertFalse(positionOne.isEqual(positionTwo)); + } + + @Test + public void filePositionEquality_CorrectlyReturnsFalseWithDifferentColumn() { + FilePosition positionOne = new FilePosition(FILE_NAME); + positionOne.setLineNumber(5); + positionOne.setColumnNumber(9); + + FilePosition positionTwo = new FilePosition(FILE_NAME); + positionTwo.setLineNumber(5); + positionTwo.setColumnNumber(10); + + assertFalse(positionOne.isEqual(positionTwo)); + } + + @Test + public void filePositionEquality_CorrectlyReturnsFalseWithDifferentFileName() { + FilePosition positionOne = new FilePosition("FileOne"); + positionOne.setLineNumber(5); + positionOne.setColumnNumber(9); + + FilePosition positionTwo = new FilePosition("FileTwo"); + positionTwo.setLineNumber(5); + positionTwo.setColumnNumber(9); + + assertFalse(positionOne.isEqual(positionTwo)); + } + + @Test + public void noMovement_ReturnsInitialPosition() { + FilePosition expectedPosition = new FilePosition(FILE_NAME); + expectedPosition.setLineNumber(1); + expectedPosition.setColumnNumber(0); + + assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); + } + + @Test + public void advanceOneColumn_ReturnsCorrectPosition() { + FilePosition expectedPosition = new FilePosition(FILE_NAME); + expectedPosition.setLineNumber(1); + expectedPosition.setColumnNumber(1); + + trackerUnderTest.incrementColumn(); + + assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); + } + + @Test + public void advanceOneLine_ReturnsCorrectPosition() { + FilePosition expectedPosition = new FilePosition(FILE_NAME); + expectedPosition.setLineNumber(2); + expectedPosition.setColumnNumber(0); + + trackerUnderTest.incrementLine(); + + assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); + } + + @Test + public void advanceOneLine_ResetsColumn() { + FilePosition expectedPosition = new FilePosition(FILE_NAME); + expectedPosition.setLineNumber(2); + expectedPosition.setColumnNumber(0); + + trackerUnderTest.incrementColumn(); + trackerUnderTest.incrementLine(); + + assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); + } + +} diff --git a/test/scanner/LispScannerLineColumnTester.java b/test/scanner/LispScannerLineColumnTester.java index 2c77aa1..7771408 100644 --- a/test/scanner/LispScannerLineColumnTester.java +++ b/test/scanner/LispScannerLineColumnTester.java @@ -8,6 +8,7 @@ import java.io.InputStream; import org.junit.Before; import org.junit.Test; +import constructs.Token; import testutils.TestUtilities; public class LispScannerLineColumnTester { @@ -32,6 +33,47 @@ public class LispScannerLineColumnTester { assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns); } + @Test + public void givenStringWithTrailingSpace_RecordsCorrectLocation() throws IOException { + String input = "\"string\" "; + LineColumn[] expectedLinesAndColumns = { LineColumn.create(1, 1) }; + + assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns); + } + + @Test + public void givenIdentifier_RecordsCorrectLocation() throws IOException { + String input = "identifier"; + LineColumn[] expectedLinesAndColumns = { LineColumn.create(1, 1) }; + + assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns); + } + + @Test + public void givenIdentifierWithTrailingSpace_RecordsCorrectLocation() throws IOException { + String input = "identifier "; + LineColumn[] expectedLinesAndColumns = { LineColumn.create(1, 1) }; + + assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns); + } + + @Test + public void givenNumber_RecordsCorrectLocation() throws IOException { + String input = "123456789"; + LineColumn[] expectedLinesAndColumns = { LineColumn.create(1, 1) }; + + assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns); + } + + @Test + public void givenNumberWithTrailingSpace_RecordsCorrectLocation() throws IOException { + String input = "123456789 "; + LineColumn[] expectedLinesAndColumns = { LineColumn.create(1, 1) }; + + assertTokenLineAndColumnsMatch(input, expectedLinesAndColumns); + } + + @Test public void givenMultipleStrings_RecordsCorrectLocations() throws IOException { String input = "\"string1\" \n \"string2 \n with newline\" \n \"string3\""; @@ -68,7 +110,7 @@ public class LispScannerLineColumnTester { LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream"); for (LineColumn lineColumn : expectedLineColumnList) { - Token nextToken = lispScanner.nextToken(); + Token nextToken = lispScanner.getNextToken(); assertTrue(lineColumn.isEqual(nextToken)); } } diff --git a/test/scanner/LispScannerTextTester.java b/test/scanner/LispScannerTextTester.java index 307cf89..7a4d8c1 100644 --- a/test/scanner/LispScannerTextTester.java +++ b/test/scanner/LispScannerTextTester.java @@ -44,14 +44,14 @@ public class LispScannerTextTester { InputStream stringInputStream = TestUtilities.createInputStreamFromString(input); LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream"); - assertEquals(expectedText, lispScanner.nextToken().getText()); + assertEquals(expectedText, lispScanner.getNextToken().getText()); } private void assertInputFileNameMatches(String input, String expectedInputFileName) throws IOException { InputStream stringInputStream = TestUtilities.createInputStreamFromString(input); LispScanner lispScanner = new LispScanner(stringInputStream, expectedInputFileName); - assertEquals(expectedInputFileName, lispScanner.nextToken().getFName()); + assertEquals(expectedInputFileName, lispScanner.getNextToken().getFName()); } } diff --git a/test/scanner/LispScannerTypeTester.java b/test/scanner/LispScannerTypeTester.java index 2d64090..b0c95dd 100644 --- a/test/scanner/LispScannerTypeTester.java +++ b/test/scanner/LispScannerTypeTester.java @@ -1,19 +1,23 @@ package scanner; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.io.InputStream; import org.junit.Test; -import scanner.Token.Type; +import constructs.Token; +import constructs.Token.Type; +import error.ErrorManager; import testutils.TestUtilities; public class LispScannerTypeTester { @Test - public void givenEmptyFile_returnsCorrectTokenTypes() throws IOException { + public void givenEmptyFile_ReturnsCorrectTypes() throws IOException { String input = ""; Token.Type[] expectedTypes = {}; @@ -29,7 +33,7 @@ public class LispScannerTypeTester { } @Test - public void givenNil_returnsCorrectTokenTypes() throws IOException { + public void givenNil_ReturnsCorrectTypes() throws IOException { String input = "()"; Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.RIGHT_PAREN }; @@ -37,7 +41,7 @@ public class LispScannerTypeTester { } @Test - public void givenListOfNumbers_returnsCorrectTokenTypes() throws IOException { + public void givenListOfNumbers_ReturnsCorrectTypes() throws IOException { String input = "(1 2)"; Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.NUMBER, Type.NUMBER, Type.RIGHT_PAREN }; @@ -45,7 +49,7 @@ public class LispScannerTypeTester { } @Test - public void givenString_returnsCorrectTokenTypes() throws IOException { + public void givenString_ReturnsCorrectTypes() throws IOException { String input = "\"string\""; Token.Type[] expectedTypes = { Type.STRING }; @@ -53,7 +57,7 @@ public class LispScannerTypeTester { } @Test - public void givenStringWithEscapedDoubleQuote_returnsCorrectTokenTypes() throws IOException { + public void givenStringWithEscapedDoubleQuote_ReturnsCorrectTypes() throws IOException { String input = "\"string \n hi \\\" bye\""; Token.Type[] expectedTypes = { Type.STRING }; @@ -61,14 +65,14 @@ public class LispScannerTypeTester { } @Test - public void givenStringWithEscapedDoubleQuoteAndComment_returnsCorrectTokenTypes() throws IOException { + public void givenStringWithEscapedDoubleQuoteAndComment_ReturnsCorrectTypes() throws IOException { String input = "\"string \n hi \\\" ; bye\""; Token.Type[] expectedTypes = { Type.STRING }; assertTokenTypesMatch(input, expectedTypes); } - @Test(expected = RuntimeException.class) + @Test(expected = LispScanner.UnterminatedStringException.class) public void givenUnterminatedString_ThrowsException() throws IOException { String input = "\"oh no!"; Token.Type[] expectedTypes = { Type.STRING }; @@ -76,8 +80,34 @@ public class LispScannerTypeTester { assertTokenTypesMatch(input, expectedTypes); } + @Test() + public void givenUnterminatedString_ExceptionHasCorrectSeverity() throws IOException { + String input = "\"oh no!"; + Token.Type[] expectedTypes = { Type.STRING }; + + try { + assertTokenTypesMatch(input, expectedTypes); + } catch (LispScanner.UnterminatedStringException e) { + assertTrue(e.getSeverity() < ErrorManager.CRITICAL_LEVEL); + } + } + + @Test() + public void givenUnterminatedString_ExceptionContainsMessage() throws IOException { + String input = "\"oh no!"; + Token.Type[] expectedTypes = { Type.STRING }; + + try { + assertTokenTypesMatch(input, expectedTypes); + } catch (LispScanner.UnterminatedStringException e) { + String message = e.getMessage(); + assertNotNull(message); + assertTrue(message.length() > 0); + } + } + @Test - public void givenIdentifier_returnsCorrectTokenTypes() throws IOException { + public void givenIdentifier_ReturnsCorrectTypes() throws IOException { String input = "abcdefgHIJKLMNOP1234"; Token.Type[] expectedTypes = { Type.IDENTIFIER }; @@ -85,7 +115,7 @@ public class LispScannerTypeTester { } @Test - public void givenSingleDigitNumber_returnsCorrectTokenTypes() throws IOException { + public void givenSingleDigitNumber_ReturnsCorrectTypes() throws IOException { String input = "1"; Token.Type[] expectedTypes = { Type.NUMBER }; @@ -93,7 +123,7 @@ public class LispScannerTypeTester { } @Test - public void givenMultipleDigitNumber_returnsCorrectTokenTypes() throws IOException { + public void givenMultipleDigitNumber_ReturnsCorrectTypes() throws IOException { String input = "1234567890"; Token.Type[] expectedTypes = { Type.NUMBER }; @@ -101,7 +131,7 @@ public class LispScannerTypeTester { } @Test - public void givenQuote_returnsCorrectTokenTypes() throws IOException { + public void givenQuote_ReturnsCorrectTypes() throws IOException { String input = "'"; Token.Type[] expectedTypes = { Type.QUOTE_MARK }; @@ -109,7 +139,16 @@ public class LispScannerTypeTester { } @Test - public void givenFunctionCall_returnsCorrectTokenTypes() throws IOException { + public void givenManyTypesWithNoWhitespace_ReturnsCorrectTypes() throws IOException { + String input = "xxx\"hi\"999()'aaa"; + Token.Type[] expectedTypes = { Type.IDENTIFIER, Type.STRING, Type.NUMBER, Type.LEFT_PAREN, Type.RIGHT_PAREN, + Type.QUOTE_MARK, Type.IDENTIFIER }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenFunctionCall_ReturnsCorrectTypes() throws IOException { String input = "(defun myFunction (x)\n (print x))"; Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.IDENTIFIER, Type.IDENTIFIER, Type.LEFT_PAREN, Type.IDENTIFIER, Type.RIGHT_PAREN, Type.LEFT_PAREN, Type.IDENTIFIER, @@ -123,9 +162,9 @@ public class LispScannerTypeTester { LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream"); for (Token.Type expectedType : expectedTypeList) - assertEquals(expectedType, lispScanner.nextToken().getType()); + assertEquals(expectedType, lispScanner.getNextToken().getType()); - assertEquals(Token.Type.EOF, lispScanner.nextToken().getType()); + assertEquals(Token.Type.EOF, lispScanner.getNextToken().getType()); } }