mike
/
transcendental-lisp
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects Releases Wiki Activity

Refactored LispScanner and some unit tests

This commit is contained in:
Mike Cifelli 2016-12-11 10:12:59 -05:00
parent 0d406c3e36
commit abdd89737f
3 changed files with 97 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
src/scanner/LispScanner.java
View File

@ -1,9 +1,6 @@
package scanner; package scanner;
import static util.Characters.BACKSLASH; import static util.Characters.*;
import static util.Characters.DOUBLE_QUOTE;
import static util.Characters.EOF;
import static util.Characters.NEWLINE;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.IOException; import java.io.IOException;
@ -39,11 +36,10 @@ public class LispScanner {
char currentCharacter = (char) c; char currentCharacter = (char) c;
positionTracker.incrementColumn(); positionTracker.incrementColumn();
if (Character.isWhitespace(currentCharacter)) { if (!Character.isWhitespace(currentCharacter))
if (currentCharacter == NEWLINE)
positionTracker.incrementLine();
} else
return createTokenFromCharacter(currentCharacter); return createTokenFromCharacter(currentCharacter);
else if (currentCharacter == NEWLINE)
positionTracker.incrementLine();
} }
return tokenFactory.createEOFToken(positionTracker.getCurrentPosition()); return tokenFactory.createEOFToken(positionTracker.getCurrentPosition());
@ -60,71 +56,101 @@ public class LispScanner {
String tokenText = "" + firstCharacter; String tokenText = "" + firstCharacter;
if (firstCharacter == DOUBLE_QUOTE) if (firstCharacter == DOUBLE_QUOTE)
tokenText = retrieveString(firstCharacter); tokenText = retrieveStringTokenText(firstCharacter);
else if (Character.isDigit(firstCharacter)) else if (Character.isDigit(firstCharacter))
tokenText = retrieveNumber(firstCharacter); tokenText = retrieveNumberTokenText(firstCharacter);
else if (Characters.isLegalIdentifierCharacter(firstCharacter)) else if (Characters.isLegalIdentifierCharacter(firstCharacter))
tokenText = retrieveIdentifier(firstCharacter); tokenText = retrieveIdentifierTokenText(firstCharacter);
return tokenText; return tokenText;
} }
private String retrieveString(char firstDoubleQuote) throws IOException { private String retrieveStringTokenText(char firstDoubleQuote) throws IOException {
StringBuilder text = new StringBuilder(); ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDoubleQuote,
FilePosition stringPosition = positionTracker.getCurrentPosition(); Characters::isLegalStringCharacter);
char prevChar = firstDoubleQuote;
text.append(firstDoubleQuote); return retriever.retrieveToken();
for (int c = inputStream.read(); c != EOF; c = inputStream.read()) {
char nextChar = (char) c;
positionTracker.incrementColumn();
text.append(nextChar);
if (nextChar == NEWLINE)
positionTracker.incrementLine();
else if ((nextChar == DOUBLE_QUOTE) && (prevChar != BACKSLASH))
return text.toString();
prevChar = nextChar;
} }
throw new UnterminatedStringException(stringPosition); private String retrieveNumberTokenText(char firstDigit) throws IOException {
ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDigit, Character::isDigit);
return retriever.retrieveToken();
} }
private String retrieveNumber(char firstDigit) throws IOException { private String retrieveIdentifierTokenText(char firstCharacter) throws IOException {
return retrieveNumberOrIdentifier(firstDigit, Character::isDigit); ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstCharacter,
Characters::isLegalIdentifierCharacter);
return retriever.retrieveToken();
} }
private String retrieveIdentifier(char firstCharacter) throws IOException { public class ComplexTokenTextRetriever {
return retrieveNumberOrIdentifier(firstCharacter, Characters::isLegalIdentifierCharacter);
Function<Character, Boolean> isPartOfToken;
StringBuilder text;
FilePosition position;
char firstCharacter;
char currentCharacter;
char previousCharacter;
public ComplexTokenTextRetriever(char firstCharacter, Function<Character, Boolean> isPartOfToken) {
this.isPartOfToken = isPartOfToken;
this.text = new StringBuilder();
this.position = positionTracker.getCurrentPosition();
this.firstCharacter = firstCharacter;
this.currentCharacter = firstCharacter;
this.previousCharacter = firstCharacter;
} }
private String retrieveNumberOrIdentifier(char firstCharacter, Function<Character, Boolean> isPartOfToken) public String retrieveToken() throws IOException {
throws IOException {
StringBuilder text = new StringBuilder();
text.append(firstCharacter); text.append(firstCharacter);
inputStream.mark(1); inputStream.mark(1);
for (int c = inputStream.read(); c != EOF; c = inputStream.read()) { for (int c = inputStream.read(); c != EOF; c = inputStream.read()) {
char nextChar = (char) c; currentCharacter = (char) c;
if (isPartOfToken.apply(nextChar)) { if (!isPartOfToken.apply(currentCharacter)) {
text.append(nextChar);
positionTracker.incrementColumn();
} else {
inputStream.reset(); inputStream.reset();
return text.toString(); return text.toString();
} }
inputStream.mark(1); addCharacterToToken();
if (isStringToken() && isTerminatingDoubleQuote())
return text.toString();
previousCharacter = currentCharacter;
} }
return terminateTokenWithEOF();
}
private void addCharacterToToken() {
text.append(currentCharacter);
positionTracker.incrementColumn();
inputStream.mark(1);
if (currentCharacter == NEWLINE)
positionTracker.incrementLine();
}
private boolean isStringToken() {
return firstCharacter == DOUBLE_QUOTE;
}
private boolean isTerminatingDoubleQuote() {
return (currentCharacter == DOUBLE_QUOTE) && (previousCharacter != BACKSLASH);
}
private String terminateTokenWithEOF() {
if (isStringToken())
throw new UnterminatedStringException(position);
return text.toString(); return text.toString();
} }
}
public static class UnterminatedStringException extends LispException { public static class UnterminatedStringException extends LispException {
@ -145,7 +171,6 @@ public class LispScanner {
return MessageFormat.format("unterminated quoted string - line {0}, column {1}", position.getLineNumber(), return MessageFormat.format("unterminated quoted string - line {0}, column {1}", position.getLineNumber(),
position.getColumnNumber()); position.getColumnNumber());
} }
} }
} }

4
src/util/Characters.java
View File

@ -40,4 +40,8 @@ public class Characters {
return (!Character.isWhitespace(c)) && (!illegalIdentifierCharacters.contains(c)); return (!Character.isWhitespace(c)) && (!illegalIdentifierCharacters.contains(c));
} }
public static boolean isLegalStringCharacter(char c) {
return true;
}
} }

51
test/file/FilePositionTrackerTester.java
View File

@ -19,39 +19,24 @@ public class FilePositionTrackerTester {
@Test @Test
public void filePositionEquality_CorrectlyReturnsTrue() { public void filePositionEquality_CorrectlyReturnsTrue() {
FilePosition positionOne = new FilePosition(FILE_NAME); FilePosition positionOne = createFilePosition(5, 9);
positionOne.setLineNumber(5); FilePosition positionTwo = createFilePosition(5, 9);
positionOne.setColumnNumber(9);
FilePosition positionTwo = new FilePosition(FILE_NAME);
positionTwo.setLineNumber(5);
positionTwo.setColumnNumber(9);
assertTrue(positionOne.isEqual(positionTwo)); assertTrue(positionOne.isEqual(positionTwo));
} }
@Test @Test
public void filePositionEquality_CorrectlyReturnsFalseWithDifferentLine() { public void filePositionEquality_CorrectlyReturnsFalseWithDifferentLine() {
FilePosition positionOne = new FilePosition(FILE_NAME); FilePosition positionOne = createFilePosition(5, 9);
positionOne.setLineNumber(5); FilePosition positionTwo = createFilePosition(8, 9);
positionOne.setColumnNumber(9);
FilePosition positionTwo = new FilePosition(FILE_NAME);
positionTwo.setLineNumber(8);
positionTwo.setColumnNumber(9);
assertFalse(positionOne.isEqual(positionTwo)); assertFalse(positionOne.isEqual(positionTwo));
} }
@Test @Test
public void filePositionEquality_CorrectlyReturnsFalseWithDifferentColumn() { public void filePositionEquality_CorrectlyReturnsFalseWithDifferentColumn() {
FilePosition positionOne = new FilePosition(FILE_NAME); FilePosition positionOne = createFilePosition(5, 9);
positionOne.setLineNumber(5); FilePosition positionTwo = createFilePosition(5, 10);
positionOne.setColumnNumber(9);
FilePosition positionTwo = new FilePosition(FILE_NAME);
positionTwo.setLineNumber(5);
positionTwo.setColumnNumber(10);
assertFalse(positionOne.isEqual(positionTwo)); assertFalse(positionOne.isEqual(positionTwo));
} }
@ -71,18 +56,14 @@ public class FilePositionTrackerTester {
@Test @Test
public void noMovement_ReturnsInitialPosition() { public void noMovement_ReturnsInitialPosition() {
FilePosition expectedPosition = new FilePosition(FILE_NAME); FilePosition expectedPosition = createFilePosition(1, 0);
expectedPosition.setLineNumber(1);
expectedPosition.setColumnNumber(0);
assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition()));
} }
@Test @Test
public void advanceOneColumn_ReturnsCorrectPosition() { public void advanceOneColumn_ReturnsCorrectPosition() {
FilePosition expectedPosition = new FilePosition(FILE_NAME); FilePosition expectedPosition = createFilePosition(1, 1);
expectedPosition.setLineNumber(1);
expectedPosition.setColumnNumber(1);
trackerUnderTest.incrementColumn(); trackerUnderTest.incrementColumn();
@ -91,9 +72,7 @@ public class FilePositionTrackerTester {
@Test @Test
public void advanceOneLine_ReturnsCorrectPosition() { public void advanceOneLine_ReturnsCorrectPosition() {
FilePosition expectedPosition = new FilePosition(FILE_NAME); FilePosition expectedPosition = createFilePosition(2, 0);
expectedPosition.setLineNumber(2);
expectedPosition.setColumnNumber(0);
trackerUnderTest.incrementLine(); trackerUnderTest.incrementLine();
@ -102,9 +81,7 @@ public class FilePositionTrackerTester {
@Test @Test
public void advanceOneLine_ResetsColumn() { public void advanceOneLine_ResetsColumn() {
FilePosition expectedPosition = new FilePosition(FILE_NAME); FilePosition expectedPosition = createFilePosition(2, 0);
expectedPosition.setLineNumber(2);
expectedPosition.setColumnNumber(0);
trackerUnderTest.incrementColumn(); trackerUnderTest.incrementColumn();
trackerUnderTest.incrementLine(); trackerUnderTest.incrementLine();
@ -112,4 +89,12 @@ public class FilePositionTrackerTester {
assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition())); assertTrue(expectedPosition.isEqual(trackerUnderTest.getCurrentPosition()));
} }
private FilePosition createFilePosition(int lineNumber, int columnNumber) {
FilePosition position = new FilePosition(FILE_NAME);
position.setLineNumber(lineNumber);
position.setColumnNumber(columnNumber);
return position;
}
} }