diff --git a/src/parser/LispParser.java b/src/parser/LispParser.java index 9a7733f..89dfd60 100644 --- a/src/parser/LispParser.java +++ b/src/parser/LispParser.java @@ -126,7 +126,7 @@ public class LispParser { return sExpr(); } - // sExpr ::= NUMBER | IDENTIFIER | RESERVED | STRING | QUOTE_MARK sExpr | + // sExpr ::= NUMBER | IDENTIFIER | STRING | QUOTE_MARK sExpr | // LEFT_PAREN sExprTail // // Returns: an S-expression that matches the rules given above @@ -141,7 +141,6 @@ public class LispParser { case NUMBER: return new LispNumber(nextToken.getText()); case IDENTIFIER: - case RESERVED: return new Symbol(nextToken.getText()); case STRING: return new LispString(nextToken.getText()); diff --git a/src/scanner/LispFilterInputStream.java b/src/scanner/LispFilterInputStream.java index 18b0794..a752fbe 100644 --- a/src/scanner/LispFilterInputStream.java +++ b/src/scanner/LispFilterInputStream.java @@ -9,44 +9,55 @@ import java.io.IOException; */ public class LispFilterInputStream extends FilterInputStream { - private boolean inQuote; + private boolean inQuotedString; + private int previousCharacter; private int nextCharacter; - public LispFilterInputStream(InputStream in) { - super(in); - inQuote = false; + public LispFilterInputStream(InputStream underlyingInputStream) { + super(underlyingInputStream); + + inQuotedString = false; + previousCharacter = 0; nextCharacter = 0; } @Override public int read() throws IOException { - nextCharacter = super.read(); + readNextCharacter(); - if (haveEnteredComment()) { + if (haveEnteredComment()) consumeAllBytesInComment(); - } else if (haveEncounteredStringBoundary()) { - inQuote = (!inQuote); - } return nextCharacter; } + private void readNextCharacter() throws IOException { + previousCharacter = nextCharacter; + nextCharacter = super.read(); + + indicateEncounterWithStringBoundary(); + } + + private void indicateEncounterWithStringBoundary() { + if (haveEncounteredStringBoundary()) + inQuotedString = !inQuotedString; + } + + private boolean haveEncounteredStringBoundary() { + return (previousCharacter != '\\') && (nextCharacter == '\"'); + } + + private boolean haveEnteredComment() { + return (nextCharacter == ';') && (!inQuotedString); + } + private void consumeAllBytesInComment() throws IOException { - while (stillInComment()) { + while (stillInComment()) nextCharacter = super.read(); - } } private boolean stillInComment() { return (nextCharacter != '\n') && (nextCharacter != -1); } - private boolean haveEnteredComment() { - return (nextCharacter == ';') && (!inQuote); - } - - private boolean haveEncounteredStringBoundary() { - return nextCharacter == '\"'; - } - } diff --git a/src/scanner/Token.java b/src/scanner/Token.java index c410102..57d2601 100644 --- a/src/scanner/Token.java +++ b/src/scanner/Token.java @@ -31,9 +31,6 @@ public class Token { /** A number token */ NUMBER, - /** A reserved word token */ - RESERVED, - /** An identifier token */ IDENTIFIER, diff --git a/src/scanner/package.html b/src/scanner/package.html index 6be4f13..5a0c31e 100644 --- a/src/scanner/package.html +++ b/src/scanner/package.html @@ -1,4 +1,3 @@ - Provides the classes necessary to perform a lexical analysis of the Lisp - programming language. + Provides the classes necessary to perform a lexical analysis of the Lisp programming language. diff --git a/test/scanner/LispFilterInputStreamTester.java b/test/scanner/LispFilterInputStreamTester.java index 5776ad2..a24a384 100644 --- a/test/scanner/LispFilterInputStreamTester.java +++ b/test/scanner/LispFilterInputStreamTester.java @@ -2,20 +2,28 @@ package scanner; import static org.junit.Assert.assertEquals; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import org.junit.Before; import org.junit.Test; +import testutils.TestUtilities; + public class LispFilterInputStreamTester { + private StringBuilder stringBuilder; + + @Before + public void setUp() { + stringBuilder = new StringBuilder(); + } + @Test public void oneComment_Removed() throws IOException { String input = ";comment"; String expectedResult = ""; + assertEquals(expectedResult, getLispFilterInputStreamResult(input)); } @@ -23,6 +31,7 @@ public class LispFilterInputStreamTester { public void multipleComments_Removed() throws IOException { String input = ";comment1\n;comment2\n;comment3"; String expectedResult = "\n\n"; + assertEquals(expectedResult, getLispFilterInputStreamResult(input)); } @@ -30,6 +39,7 @@ public class LispFilterInputStreamTester { public void nil_NotRemoved() throws IOException { String input = "()"; String expectedResult = "()"; + assertEquals(expectedResult, getLispFilterInputStreamResult(input)); } @@ -37,18 +47,28 @@ public class LispFilterInputStreamTester { public void interiorComment_Removed() throws IOException { String input = "(;this is a comment\n)"; String expectedResult = "(\n)"; + assertEquals(expectedResult, getLispFilterInputStreamResult(input)); } @Test public void commentInString_NotRemoved() throws IOException { String input = "\"string;this should remain\""; + assertEquals(input, getLispFilterInputStreamResult(input)); } @Test public void commentInStringWithNewline_NotRemoved() throws IOException { String input = "\"string;this should\n remain\""; + + assertEquals(input, getLispFilterInputStreamResult(input)); + } + + @Test + public void commentInStringWithEscapedDoubleQuote_NotRemoved() throws IOException { + String input = "\"string \\\" ;this should remain\""; + assertEquals(input, getLispFilterInputStreamResult(input)); } @@ -56,22 +76,18 @@ public class LispFilterInputStreamTester { public void manyCommentsWithStatements_OnlyCommentsRemoved() throws IOException { String input = ";first comment \n '(1 2 3) \n ;second comment \n (defun add1 (x) (+ x 1)) ;third comment"; String expectedResult = "\n '(1 2 3) \n \n (defun add1 (x) (+ x 1)) "; + assertEquals(expectedResult, getLispFilterInputStreamResult(input)); } private String getLispFilterInputStreamResult(String inputString) throws IOException { - InputStream stringInputStream = createInputStreamFromString(inputString); + InputStream stringInputStream = TestUtilities.createInputStreamFromString(inputString); LispFilterInputStream lispFilterInputStream = new LispFilterInputStream(stringInputStream); return readInputStreamIntoString(lispFilterInputStream); } - private InputStream createInputStreamFromString(String string) { - return new ByteArrayInputStream(string.getBytes()); - } - private String readInputStreamIntoString(InputStream inputStream) throws IOException { - StringBuilder stringBuilder = new StringBuilder(); int c = inputStream.read(); while (c != -1) { diff --git a/test/scanner/LispScannerTester.java b/test/scanner/LispScannerTester.java new file mode 100644 index 0000000..a0da5df --- /dev/null +++ b/test/scanner/LispScannerTester.java @@ -0,0 +1,99 @@ +package scanner; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.InputStream; + +import org.junit.Test; + +import scanner.Token.Type; +import testutils.TestUtilities; + +public class LispScannerTester { + + @Test + public void givenEmptyFile_returnsCorrectTokenTypes() throws IOException { + String input = ""; + Token.Type[] expectedTypes = {}; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenNil_returnsCorrectTokenTypes() throws IOException { + String input = "()"; + Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.RIGHT_PAREN }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenListOfNumbers_returnsCorrectTokenTypes() throws IOException { + String input = "(1 2)"; + Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.NUMBER, Type.NUMBER, Type.RIGHT_PAREN }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenString_returnsCorrectTokenTypes() throws IOException { + String input = "\"string\""; + Token.Type[] expectedTypes = { Type.STRING }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenStringWithEscapedDoubleQuote_returnsCorrectTokenTypes() throws IOException { + String input = "\"string \n hi \\\" bye\""; + Token.Type[] expectedTypes = { Type.STRING }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenStringWithEscapedDoubleQuoteAndComment_returnsCorrectTokenTypes() throws IOException { + String input = "\"string \n hi \\\" ; bye\""; + Token.Type[] expectedTypes = { Type.STRING }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenIdentifier_returnsCorrectTokenTypes() throws IOException { + String input = "abcdefgHIJKLMNOP1234"; + Token.Type[] expectedTypes = { Type.IDENTIFIER }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenQuote_returnsCorrectTokenTypes() throws IOException { + String input = "'"; + Token.Type[] expectedTypes = { Type.QUOTE_MARK }; + + assertTokenTypesMatch(input, expectedTypes); + } + + @Test + public void givenFunctionCall_returnsCorrectTokenTypes() throws IOException { + String input = "(defun myFunction (x)\n (print x))"; + Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.IDENTIFIER, Type.IDENTIFIER, Type.LEFT_PAREN, + Type.IDENTIFIER, Type.RIGHT_PAREN, Type.LEFT_PAREN, Type.IDENTIFIER, + Type.IDENTIFIER, Type.RIGHT_PAREN, Type.RIGHT_PAREN }; + + assertTokenTypesMatch(input, expectedTypes); + } + + private void assertTokenTypesMatch(String input, Token.Type[] expectedTypeList) throws IOException { + InputStream stringInputStream = TestUtilities.createInputStreamFromString(input); + LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream"); + + for (Token.Type expectedType : expectedTypeList) + assertEquals(expectedType, lispScanner.nextToken().getType()); + + assertEquals(Token.Type.EOF, lispScanner.nextToken().getType()); + } + +} diff --git a/test/testutils/TestUtilities.java b/test/testutils/TestUtilities.java new file mode 100644 index 0000000..6d8f896 --- /dev/null +++ b/test/testutils/TestUtilities.java @@ -0,0 +1,12 @@ +package testutils; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +public class TestUtilities { + + public static InputStream createInputStreamFromString(String string) { + return new ByteArrayInputStream(string.getBytes()); + } + +}