From abdd89737f2bd129afd45583eb4efaf8a4a6872d Mon Sep 17 00:00:00 2001 From: Mike Cifelli Date: Sun, 11 Dec 2016 10:12:59 -0500 Subject: [PATCH] Refactored LispScanner and some unit tests --- src/scanner/LispScanner.java | 125 ++++++++++++++--------- src/util/Characters.java | 4 + test/file/FilePositionTrackerTester.java | 51 ++++----- 3 files changed, 97 insertions(+), 83 deletions(-) diff --git a/src/scanner/LispScanner.java b/src/scanner/LispScanner.java index 5501dfa..e6194e1 100644 --- a/src/scanner/LispScanner.java +++ b/src/scanner/LispScanner.java @@ -1,9 +1,6 @@ package scanner; -import static util.Characters.BACKSLASH; -import static util.Characters.DOUBLE_QUOTE; -import static util.Characters.EOF; -import static util.Characters.NEWLINE; +import static util.Characters.*; import java.io.BufferedInputStream; import java.io.IOException; @@ -39,11 +36,10 @@ public class LispScanner { char currentCharacter = (char) c; positionTracker.incrementColumn(); - if (Character.isWhitespace(currentCharacter)) { - if (currentCharacter == NEWLINE) - positionTracker.incrementLine(); - } else + if (!Character.isWhitespace(currentCharacter)) return createTokenFromCharacter(currentCharacter); + else if (currentCharacter == NEWLINE) + positionTracker.incrementLine(); } return tokenFactory.createEOFToken(positionTracker.getCurrentPosition()); @@ -60,70 +56,100 @@ public class LispScanner { String tokenText = "" + firstCharacter; if (firstCharacter == DOUBLE_QUOTE) - tokenText = retrieveString(firstCharacter); + tokenText = retrieveStringTokenText(firstCharacter); else if (Character.isDigit(firstCharacter)) - tokenText = retrieveNumber(firstCharacter); + tokenText = retrieveNumberTokenText(firstCharacter); else if (Characters.isLegalIdentifierCharacter(firstCharacter)) - tokenText = retrieveIdentifier(firstCharacter); + tokenText = retrieveIdentifierTokenText(firstCharacter); return tokenText; } - private String retrieveString(char firstDoubleQuote) throws IOException { - StringBuilder text = new StringBuilder(); - FilePosition stringPosition = positionTracker.getCurrentPosition(); - char prevChar = firstDoubleQuote; + private String retrieveStringTokenText(char firstDoubleQuote) throws IOException { + ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDoubleQuote, + Characters::isLegalStringCharacter); - text.append(firstDoubleQuote); + return retriever.retrieveToken(); + } - for (int c = inputStream.read(); c != EOF; c = inputStream.read()) { - char nextChar = (char) c; + private String retrieveNumberTokenText(char firstDigit) throws IOException { + ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDigit, Character::isDigit); - positionTracker.incrementColumn(); - text.append(nextChar); + return retriever.retrieveToken(); + } - if (nextChar == NEWLINE) - positionTracker.incrementLine(); - else if ((nextChar == DOUBLE_QUOTE) && (prevChar != BACKSLASH)) - return text.toString(); + private String retrieveIdentifierTokenText(char firstCharacter) throws IOException { + ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstCharacter, + Characters::isLegalIdentifierCharacter); - prevChar = nextChar; + return retriever.retrieveToken(); + } + + public class ComplexTokenTextRetriever { + + Function isPartOfToken; + StringBuilder text; + FilePosition position; + char firstCharacter; + char currentCharacter; + char previousCharacter; + + public ComplexTokenTextRetriever(char firstCharacter, Function isPartOfToken) { + this.isPartOfToken = isPartOfToken; + this.text = new StringBuilder(); + this.position = positionTracker.getCurrentPosition(); + this.firstCharacter = firstCharacter; + this.currentCharacter = firstCharacter; + this.previousCharacter = firstCharacter; } - throw new UnterminatedStringException(stringPosition); - } + public String retrieveToken() throws IOException { + text.append(firstCharacter); + inputStream.mark(1); - private String retrieveNumber(char firstDigit) throws IOException { - return retrieveNumberOrIdentifier(firstDigit, Character::isDigit); - } + for (int c = inputStream.read(); c != EOF; c = inputStream.read()) { + currentCharacter = (char) c; - private String retrieveIdentifier(char firstCharacter) throws IOException { - return retrieveNumberOrIdentifier(firstCharacter, Characters::isLegalIdentifierCharacter); - } + if (!isPartOfToken.apply(currentCharacter)) { + inputStream.reset(); - private String retrieveNumberOrIdentifier(char firstCharacter, Function isPartOfToken) - throws IOException { - StringBuilder text = new StringBuilder(); + return text.toString(); + } - text.append(firstCharacter); - inputStream.mark(1); + addCharacterToToken(); - for (int c = inputStream.read(); c != EOF; c = inputStream.read()) { - char nextChar = (char) c; + if (isStringToken() && isTerminatingDoubleQuote()) + return text.toString(); - if (isPartOfToken.apply(nextChar)) { - text.append(nextChar); - positionTracker.incrementColumn(); - } else { - inputStream.reset(); - - return text.toString(); + previousCharacter = currentCharacter; } - inputStream.mark(1); + return terminateTokenWithEOF(); } - return text.toString(); + private void addCharacterToToken() { + text.append(currentCharacter); + positionTracker.incrementColumn(); + inputStream.mark(1); + + if (currentCharacter == NEWLINE) + positionTracker.incrementLine(); + } + + private boolean isStringToken() { + return firstCharacter == DOUBLE_QUOTE; + } + + private boolean isTerminatingDoubleQuote() { + return (currentCharacter == DOUBLE_QUOTE) && (previousCharacter != BACKSLASH); + } + + private String terminateTokenWithEOF() { + if (isStringToken()) + throw new UnterminatedStringException(position); + + return text.toString(); + } } public static class UnterminatedStringException extends LispException { @@ -145,7 +171,6 @@ public class LispScanner { return MessageFormat.format("unterminated quoted string - line {0}, column {1}", position.getLineNumber(), position.getColumnNumber()); } - } } diff --git a/src/util/Characters.java b/src/util/Characters.java index b6fac14..ec401e2 100644 --- a/src/util/Characters.java +++ b/src/util/Characters.java @@ -39,5 +39,9 @@ public class Characters { public static boolean isLegalIdentifierCharacter(char c) { return (!Character.isWhitespace(c)) && (!illegalIdentifierCharacters.contains(c)); } + + public static boolean isLegalStringCharacter(char c) { + return true; + } } diff --git a/test/file/FilePositionTrackerTester.java b/test/file/FilePositionTrackerTester.java index 5ff5ae4..d75af87 100644 --- a/test/file/FilePositionTrackerTester.java +++ b/test/file/FilePositionTrackerTester.java @@ -19,39 +19,24 @@ public class FilePositionTrackerTester { @Test public void filePositionEquality_CorrectlyReturnsTrue() { - FilePosition positionOne = new FilePosition(FILE_NAME); - positionOne.setLineNumber(5); - positionOne.setColumnNumber(9); - - FilePosition positionTwo = new FilePosition(FILE_NAME); - positionTwo.setLineNumber(5); - positionTwo.setColumnNumber(9); + FilePosition positionOne = createFilePosition(5, 9); + FilePosition positionTwo = createFilePosition(5, 9); assertTrue(positionOne.isEqual(positionTwo)); } @Test public void filePositionEquality_CorrectlyReturnsFalseWithDifferentLine() { - FilePosition positionOne = new FilePosition(FILE_NAME); - positionOne.setLineNumber(5); - positionOne.setColumnNumber(9); - - FilePosition positionTwo = new FilePosition(FILE_NAME); - positionTwo.setLineNumber(8); - positionTwo.setColumnNumber(9); + FilePosition positionOne = createFilePosition(5, 9); + FilePosition positionTwo = createFilePosition(8, 9); assertFalse(positionOne.isEqual(positionTwo)); } @Test public void filePositionEquality_CorrectlyReturnsFalseWithDifferentColumn() { - FilePosition positionOne = new FilePosition(FILE_NAME); - positionOne.setLineNumber(5); - positionOne.setColumnNumber(9); - - FilePosition positionTwo = new FilePosition(FILE_NAME); - positionTwo.setLineNumber(5); - positionTwo.setColumnNumber(10); + FilePosition positionOne = createFilePosition(5, 9); + FilePosition positionTwo = createFilePosition(5, 10); assertFalse(positionOne.isEqual(positionTwo)); } @@ -71,18 +56,14 @@ public class FilePositionTrackerTester { @Test public void noMovement_ReturnsInitialPosition() { - FilePosition expectedPosition = new FilePosition(FILE_NAME); - expectedPosition.setLineNumber(1); - expectedPosition.setColumnNumber(0); + FilePosition expectedPosition = createFilePosition(1, 0); assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); } @Test public void advanceOneColumn_ReturnsCorrectPosition() { - FilePosition expectedPosition = new FilePosition(FILE_NAME); - expectedPosition.setLineNumber(1); - expectedPosition.setColumnNumber(1); + FilePosition expectedPosition = createFilePosition(1, 1); trackerUnderTest.incrementColumn(); @@ -91,9 +72,7 @@ public class FilePositionTrackerTester { @Test public void advanceOneLine_ReturnsCorrectPosition() { - FilePosition expectedPosition = new FilePosition(FILE_NAME); - expectedPosition.setLineNumber(2); - expectedPosition.setColumnNumber(0); + FilePosition expectedPosition = createFilePosition(2, 0); trackerUnderTest.incrementLine(); @@ -102,9 +81,7 @@ public class FilePositionTrackerTester { @Test public void advanceOneLine_ResetsColumn() { - FilePosition expectedPosition = new FilePosition(FILE_NAME); - expectedPosition.setLineNumber(2); - expectedPosition.setColumnNumber(0); + FilePosition expectedPosition = createFilePosition(2, 0); trackerUnderTest.incrementColumn(); trackerUnderTest.incrementLine(); @@ -112,4 +89,12 @@ public class FilePositionTrackerTester { assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); } + private FilePosition createFilePosition(int lineNumber, int columnNumber) { + FilePosition position = new FilePosition(FILE_NAME); + position.setLineNumber(lineNumber); + position.setColumnNumber(columnNumber); + + return position; + } + }