mike
/
transcendental-lisp
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects Releases Wiki Activity

Refactored LispScanner and some unit tests

This commit is contained in:
Mike Cifelli 2016-12-11 10:12:59 -05:00
parent 0d406c3e36
commit abdd89737f
3 changed files with 97 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
src/scanner/LispScanner.java
View File

@ -1,9 +1,6 @@
package scanner;
import static util.Characters.BACKSLASH;
import static util.Characters.DOUBLE_QUOTE;
import static util.Characters.EOF;
import static util.Characters.NEWLINE;
import static util.Characters.*;
import java.io.BufferedInputStream;
import java.io.IOException;
@ -39,11 +36,10 @@ public class LispScanner {
char currentCharacter = (char) c;
positionTracker.incrementColumn();
if (Character.isWhitespace(currentCharacter)) {
if (currentCharacter == NEWLINE)
positionTracker.incrementLine();
} else
if (!Character.isWhitespace(currentCharacter))
return createTokenFromCharacter(currentCharacter);
else if (currentCharacter == NEWLINE)
positionTracker.incrementLine();
}
return tokenFactory.createEOFToken(positionTracker.getCurrentPosition());
@ -60,71 +56,101 @@ public class LispScanner {
String tokenText = "" + firstCharacter;
if (firstCharacter == DOUBLE_QUOTE)
tokenText = retrieveString(firstCharacter);
tokenText = retrieveStringTokenText(firstCharacter);
else if (Character.isDigit(firstCharacter))
tokenText = retrieveNumber(firstCharacter);
tokenText = retrieveNumberTokenText(firstCharacter);
else if (Characters.isLegalIdentifierCharacter(firstCharacter))
tokenText = retrieveIdentifier(firstCharacter);
tokenText = retrieveIdentifierTokenText(firstCharacter);
return tokenText;
}
private String retrieveString(char firstDoubleQuote) throws IOException {
StringBuilder text = new StringBuilder();
FilePosition stringPosition = positionTracker.getCurrentPosition();
char prevChar = firstDoubleQuote;
private String retrieveStringTokenText(char firstDoubleQuote) throws IOException {
ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDoubleQuote,
Characters::isLegalStringCharacter);
text.append(firstDoubleQuote);
for (int c = inputStream.read(); c != EOF; c = inputStream.read()) {
char nextChar = (char) c;
positionTracker.incrementColumn();
text.append(nextChar);
if (nextChar == NEWLINE)
positionTracker.incrementLine();
else if ((nextChar == DOUBLE_QUOTE) && (prevChar != BACKSLASH))
return text.toString();
prevChar = nextChar;
return retriever.retrieveToken();
}
throw new UnterminatedStringException(stringPosition);
private String retrieveNumberTokenText(char firstDigit) throws IOException {
ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDigit, Character::isDigit);
return retriever.retrieveToken();
}
private String retrieveNumber(char firstDigit) throws IOException {
return retrieveNumberOrIdentifier(firstDigit, Character::isDigit);
private String retrieveIdentifierTokenText(char firstCharacter) throws IOException {
ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstCharacter,
Characters::isLegalIdentifierCharacter);
return retriever.retrieveToken();
}
private String retrieveIdentifier(char firstCharacter) throws IOException {
return retrieveNumberOrIdentifier(firstCharacter, Characters::isLegalIdentifierCharacter);
public class ComplexTokenTextRetriever {
Function<Character, Boolean> isPartOfToken;
StringBuilder text;
FilePosition position;
char firstCharacter;
char currentCharacter;
char previousCharacter;
public ComplexTokenTextRetriever(char firstCharacter, Function<Character, Boolean> isPartOfToken) {
this.isPartOfToken = isPartOfToken;
this.text = new StringBuilder();
this.position = positionTracker.getCurrentPosition();
this.firstCharacter = firstCharacter;
this.currentCharacter = firstCharacter;
this.previousCharacter = firstCharacter;
}
private String retrieveNumberOrIdentifier(char firstCharacter, Function<Character, Boolean> isPartOfToken)
throws IOException {
StringBuilder text = new StringBuilder();
public String retrieveToken() throws IOException {
text.append(firstCharacter);
inputStream.mark(1);
for (int c = inputStream.read(); c != EOF; c = inputStream.read()) {
char nextChar = (char) c;
currentCharacter = (char) c;
if (isPartOfToken.apply(nextChar)) {
text.append(nextChar);
positionTracker.incrementColumn();
} else {
if (!isPartOfToken.apply(currentCharacter)) {
inputStream.reset();
return text.toString();
}
inputStream.mark(1);
addCharacterToToken();
if (isStringToken() && isTerminatingDoubleQuote())
return text.toString();
previousCharacter = currentCharacter;
}
return terminateTokenWithEOF();
}
private void addCharacterToToken() {
text.append(currentCharacter);
positionTracker.incrementColumn();
inputStream.mark(1);
if (currentCharacter == NEWLINE)
positionTracker.incrementLine();
}
private boolean isStringToken() {
return firstCharacter == DOUBLE_QUOTE;
}
private boolean isTerminatingDoubleQuote() {
return (currentCharacter == DOUBLE_QUOTE) && (previousCharacter != BACKSLASH);
}
private String terminateTokenWithEOF() {
if (isStringToken())
throw new UnterminatedStringException(position);
return text.toString();
}
}
public static class UnterminatedStringException extends LispException {
@ -145,7 +171,6 @@ public class LispScanner {
return MessageFormat.format("unterminated quoted string - line {0}, column {1}", position.getLineNumber(),
position.getColumnNumber());
}
}
}

4
src/util/Characters.java
View File

@ -40,4 +40,8 @@ public class Characters {
return (!Character.isWhitespace(c)) && (!illegalIdentifierCharacters.contains(c));
}
public static boolean isLegalStringCharacter(char c) {
return true;
}
}

51
test/file/FilePositionTrackerTester.java
View File

@ -19,39 +19,24 @@ public class FilePositionTrackerTester {
@Test
public void filePositionEquality_CorrectlyReturnsTrue() {
FilePosition positionOne = new FilePosition(FILE_NAME);
positionOne.setLineNumber(5);
positionOne.setColumnNumber(9);
FilePosition positionTwo = new FilePosition(FILE_NAME);
positionTwo.setLineNumber(5);
positionTwo.setColumnNumber(9);
FilePosition positionOne = createFilePosition(5, 9);
FilePosition positionTwo = createFilePosition(5, 9);
assertTrue(positionOne.isEqual(positionTwo));
}
@Test
public void filePositionEquality_CorrectlyReturnsFalseWithDifferentLine() {
FilePosition positionOne = new FilePosition(FILE_NAME);
positionOne.setLineNumber(5);
positionOne.setColumnNumber(9);
FilePosition positionTwo = new FilePosition(FILE_NAME);
positionTwo.setLineNumber(8);
positionTwo.setColumnNumber(9);
FilePosition positionOne = createFilePosition(5, 9);
FilePosition positionTwo = createFilePosition(8, 9);
assertFalse(positionOne.isEqual(positionTwo));
}
@Test
public void filePositionEquality_CorrectlyReturnsFalseWithDifferentColumn() {
FilePosition positionOne = new FilePosition(FILE_NAME);
positionOne.setLineNumber(5);
positionOne.setColumnNumber(9);
FilePosition positionTwo = new FilePosition(FILE_NAME);
positionTwo.setLineNumber(5);
positionTwo.setColumnNumber(10);
FilePosition positionOne = createFilePosition(5, 9);
FilePosition positionTwo = createFilePosition(5, 10);
assertFalse(positionOne.isEqual(positionTwo));
}
@ -71,18 +56,14 @@ public class FilePositionTrackerTester {
@Test
public void noMovement_ReturnsInitialPosition() {
FilePosition expectedPosition = new FilePosition(FILE_NAME);
expectedPosition.setLineNumber(1);
expectedPosition.setColumnNumber(0);
FilePosition expectedPosition = createFilePosition(1, 0);
assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition()));
}
@Test
public void advanceOneColumn_ReturnsCorrectPosition() {
FilePosition expectedPosition = new FilePosition(FILE_NAME);
expectedPosition.setLineNumber(1);
expectedPosition.setColumnNumber(1);
FilePosition expectedPosition = createFilePosition(1, 1);
trackerUnderTest.incrementColumn();
@ -91,9 +72,7 @@ public class FilePositionTrackerTester {
@Test
public void advanceOneLine_ReturnsCorrectPosition() {
FilePosition expectedPosition = new FilePosition(FILE_NAME);
expectedPosition.setLineNumber(2);
expectedPosition.setColumnNumber(0);
FilePosition expectedPosition = createFilePosition(2, 0);
trackerUnderTest.incrementLine();
@ -102,9 +81,7 @@ public class FilePositionTrackerTester {
@Test
public void advanceOneLine_ResetsColumn() {
FilePosition expectedPosition = new FilePosition(FILE_NAME);
expectedPosition.setLineNumber(2);
expectedPosition.setColumnNumber(0);
FilePosition expectedPosition = createFilePosition(2, 0);
trackerUnderTest.incrementColumn();
trackerUnderTest.incrementLine();
@ -112,4 +89,12 @@ public class FilePositionTrackerTester {
assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition()));
}
private FilePosition createFilePosition(int lineNumber, int columnNumber) {
FilePosition position = new FilePosition(FILE_NAME);
position.setLineNumber(lineNumber);
position.setColumnNumber(columnNumber);
return position;
}
}