Continued refactoring the LispParser and writing unit tests

.externalToolBuilders/Ant_Builder.launch

@@ -1,5 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?><launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
-<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
 <stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AFTER_CLEAN_TARGETS" value="jar,"/>
 <stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AUTO_TARGETS" value="jar,"/>
 <stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_CLEAN_TARGETS" value="clean,"/>

src/parser/LispParser.java

@@ -3,9 +3,8 @@ package parser;
 import java.io.InputStream;
 
 import error.LispException;
-import parser.MalformedSExpressionException.*;
 import scanner.LispScanner;
-import sexpression.*;
+import sexpression.SExpression;
 import token.Eof;
 import token.Token;
 
@@ -17,11 +16,13 @@ public class LispParser {
 
     private LispScanner scanner;
     private Token nextToken;
+    private LispException delayedException;
     private boolean isNextTokenStored;
 
     public LispParser(InputStream inputStream, String fileName) {
         scanner = new LispScanner(inputStream, fileName);
         nextToken = null;
+        delayedException = null;
         isNextTokenStored = false;
     }
 
@@ -37,59 +38,27 @@ public class LispParser {
             nextToken = scanner.getNextToken();
             isNextTokenStored = true;
         } catch (LispException e) {
-            // This method should give the illusion of not actually reading
+            delayedException = e;
-            // a token, so we ignore any exception. It will be thrown when we
-            // return the next sExpression.
         }
     }
 
     public SExpression getNextSExpression() {
+        throwDelayedExceptionIfNecessary();
+
         if (!isNextTokenStored)
             nextToken = scanner.getNextToken();
-        else
-            isNextTokenStored = false;
 
-        return sExpr();
+        isNextTokenStored = false;
+
+        return nextToken.sExpr(scanner::getNextToken);
     }
 
-    // sExpr ::= NUMBER | IDENTIFIER | STRING | QUOTE_MARK sExpr |
+    private void throwDelayedExceptionIfNecessary() {
-    // LEFT_PAREN sExprTail
+        if (delayedException != null) {
-    private SExpression sExpr() {
+            LispException exceptionToThrow = delayedException;
-        switch (nextToken.getType()) {
+            delayedException = null;
-        case NUMBER:
-            return new LispNumber(nextToken.getText());
-        case IDENTIFIER:
-            return new Symbol(nextToken.getText());
-        case STRING:
-            return new LispString(nextToken.getText());
-        case QUOTE_MARK:
-            nextToken = scanner.getNextToken();
-            SExpression arg = sExpr();
 
-            return new Cons(new Symbol("QUOTE"), new Cons(arg, Nil.getUniqueInstance()));
+            throw exceptionToThrow;
-        case LEFT_PAREN:
-            return sExprTail();
-        case RIGHT_PAREN:
-            throw new StartsWithRightParenthesisException(nextToken);
-        case EOF:
-            throw new EofEncounteredException(nextToken);
-        default:
-            throw new UnrecognizedTokenException(nextToken);
-        }
-    }
-
-    // sExprTail ::= RIGHT_PAREN | sExpr sExprTail
-    private SExpression sExprTail() {
-        nextToken = scanner.getNextToken();
-
-        switch (nextToken.getType()) {
-        case RIGHT_PAREN:
-            return Nil.getUniqueInstance();
-        default:
-            SExpression car = sExpr();
-            SExpression cdr = sExprTail();
-
-            return new Cons(car, cdr);
         }
     }
 

src/sexpression/MalformedSExpressionException.java

@@ -1,4 +1,4 @@
-package parser;
+package sexpression;
 
 import java.text.MessageFormat;
 

src/token/Eof.java

@@ -1,7 +1,10 @@
 package token;
 
-import file.FilePosition;
+import java.util.function.Supplier;
 
+import file.FilePosition;
+import sexpression.MalformedSExpressionException.EofEncounteredException;
+import sexpression.SExpression;
 
 public class Eof extends Token {
 
@@ -10,8 +13,8 @@ public class Eof extends Token {
     }
 
     @Override
-    public Type getType() {
+    public SExpression sExpr(Supplier<Token> getNextToken) {
-        return Type.EOF;
+        throw new EofEncounteredException(this);
     }
 
 }

src/token/Identifier.java

@@ -1,7 +1,10 @@
 package token;
 
-import file.FilePosition;
+import java.util.function.Supplier;
 
+import file.FilePosition;
+import sexpression.SExpression;
+import sexpression.Symbol;
 
 public class Identifier extends Token {
 
@@ -10,8 +13,8 @@ public class Identifier extends Token {
     }
 
     @Override
-    public Type getType() {
+    public SExpression sExpr(Supplier<Token> getNextToken) {
-        return Type.IDENTIFIER;
+        return new Symbol(getText());
     }
 
 }

src/token/LeftParenthesis.java

@@ -1,6 +1,9 @@
 package token;
 
+import java.util.function.Supplier;
+
 import file.FilePosition;
+import sexpression.SExpression;
 
 public class LeftParenthesis extends Token {
 
@@ -9,8 +12,10 @@ public class LeftParenthesis extends Token {
     }
 
     @Override
-    public Type getType() {
+    public SExpression sExpr(Supplier<Token> getNextToken) {
-        return Type.LEFT_PAREN;
+        Token nextToken = getNextToken.get();
+
+        return nextToken.sExprTail(getNextToken);
     }
 
 }

src/token/Number.java

@@ -1,7 +1,10 @@
 package token;
 
-import file.FilePosition;
+import java.util.function.Supplier;
 
+import file.FilePosition;
+import sexpression.LispNumber;
+import sexpression.SExpression;
 
 public class Number extends Token {
 
@@ -10,8 +13,8 @@ public class Number extends Token {
     }
 
     @Override
-    public Type getType() {
+    public SExpression sExpr(Supplier<Token> getNextToken) {
-        return Type.NUMBER;
+        return new LispNumber(getText());
     }
 
 }

src/token/QuoteMark.java

@@ -1,7 +1,9 @@
 package token;
 
-import file.FilePosition;
+import java.util.function.Supplier;
 
+import file.FilePosition;
+import sexpression.*;
 
 public class QuoteMark extends Token {
 
@@ -10,8 +12,11 @@ public class QuoteMark extends Token {
     }
 
     @Override
-    public Type getType() {
+    public SExpression sExpr(Supplier<Token> getNextToken) {
-        return Type.QUOTE_MARK;
+        Token nextToken = getNextToken.get();
+        SExpression argument = nextToken.sExpr(getNextToken);
+
+        return new Cons(new Symbol("QUOTE"), new Cons(argument, Nil.getUniqueInstance()));
     }
 
 }

src/token/QuotedString.java

@@ -1,7 +1,10 @@
 package token;
 
-import file.FilePosition;
+import java.util.function.Supplier;
 
+import file.FilePosition;
+import sexpression.LispString;
+import sexpression.SExpression;
 
 public class QuotedString extends Token {
 
@@ -10,8 +13,8 @@ public class QuotedString extends Token {
     }
 
     @Override
-    public Type getType() {
+    public SExpression sExpr(Supplier<Token> getNextToken) {
-        return Type.STRING;
+        return new LispString(getText());
     }
 
 }

src/token/RightParenthesis.java

@@ -1,7 +1,11 @@
 package token;
 
-import file.FilePosition;
+import java.util.function.Supplier;
 
+import file.FilePosition;
+import sexpression.MalformedSExpressionException.StartsWithRightParenthesisException;
+import sexpression.Nil;
+import sexpression.SExpression;
 
 public class RightParenthesis extends Token {
 
@@ -10,8 +14,13 @@ public class RightParenthesis extends Token {
     }
 
     @Override
-    public Type getType() {
+    public SExpression sExpr(Supplier<Token> getNextToken) {
-        return Type.RIGHT_PAREN;
+        throw new StartsWithRightParenthesisException(this);
+    }
+
+    @Override
+    public SExpression sExprTail(Supplier<Token> getNextToken) {
+        return Nil.getUniqueInstance();
     }
 
 }

src/token/Token.java

@@ -1,44 +1,52 @@
 package token;
 
+import java.util.function.Supplier;
+
 import file.FilePosition;
+import sexpression.Cons;
+import sexpression.SExpression;
 
 /**
  * A token in Lisp.
  */
 public abstract class Token {
 
-    public enum Type {
-        LEFT_PAREN, RIGHT_PAREN, STRING, QUOTE_MARK, NUMBER, IDENTIFIER, EOF
-    }
-
     private String text;
-    private String fileName;
+    private FilePosition position;
-    private int line;
-    private int column;
 
     public Token(String text, FilePosition position) {
         this.text = text;
-        this.fileName = position.getFileName();
+        this.position = position;
-        this.line = position.getLineNumber();
-        this.column = position.getColumnNumber();
     }
 
-    public abstract Type getType();
-
     public String getText() {
         return text;
     }
 
     public String getFileName() {
-        return fileName;
+        return position.getFileName();
     }
 
     public int getLine() {
-        return line;
+        return position.getLineNumber();
     }
 
     public int getColumn() {
-        return column;
+        return position.getColumnNumber();
+    }
+
+    // sExpr ::= NUMBER | IDENTIFIER | STRING | QUOTE_MARK sExpr |
+    // LEFT_PAREN sExprTail
+    public abstract SExpression sExpr(Supplier<Token> getNextToken);
+
+    // sExprTail ::= RIGHT_PAREN | sExpr sExprTail
+    public SExpression sExprTail(Supplier<Token> getNextToken) {
+        SExpression car = sExpr(getNextToken);
+
+        Token nextToken = getNextToken.get();
+        SExpression cdr = nextToken.sExprTail(getNextToken);
+
+        return new Cons(car, cdr);
     }
 
 }

src/util/Characters.java

@@ -3,7 +3,9 @@ package util;
 import java.util.HashSet;
 import java.util.Set;
 
-public class Characters {
+public final class Characters {
+
+    private Characters() {}
 
     public static final int EOF = -1;
 

test/parser/LispParserTester.java

@@ -1,20 +1,31 @@
 package parser;
 
 import static org.junit.Assert.*;
+import static testutil.TestUtilities.createIOExceptionThrowingInputStream;
+import static testutil.TestUtilities.createInputStreamFromString;
 
 import java.io.InputStream;
 
 import org.junit.Test;
 
 import error.LispException;
+import scanner.LispInputStream.UncheckedIOException;
+import scanner.LispScanner.UnterminatedStringException;
+import sexpression.MalformedSExpressionException.EofEncounteredException;
+import sexpression.MalformedSExpressionException.StartsWithRightParenthesisException;
 import sexpression.Nil;
 import sexpression.SExpression;
-import testutils.TestUtilities;
+import token.TokenFactory.BadCharacterException;
 
 public class LispParserTester {
 
     private LispParser createLispParser(String input) {
-        InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
+        InputStream stringInputStream = createInputStreamFromString(input);
+        return new LispParser(stringInputStream, "testFile");
+    }
+
+    private LispParser createIOExceptionThrowingLispParser() {
+        InputStream stringInputStream = createIOExceptionThrowingInputStream();
         return new LispParser(stringInputStream, "testFile");
     }
 
@@ -183,6 +194,17 @@ public class LispParserTester {
         assertTrue(parser.isEof());
     }
 
+    @Test
+    public void givenMultipleComplexLists_CreatesCorrectSExpressions() {
+        String input = "(defun f (x) \n (print \n (list \"x is \" x) \n ) \n )"
+                       + "(defun f (x) \n (print \n (list \"x is \" x) \n ) \n )";
+        LispParser parser = createLispParser(input);
+
+        assertList(parser.getNextSExpression());
+        assertList(parser.getNextSExpression());
+        assertTrue(parser.isEof());
+    }
+
     @Test
     public void givenMultipleExpressions_CreatesCorrectSExpressions() {
         String input = "(setf x 2) x \"hi\" () 29";
@@ -208,7 +230,7 @@ public class LispParserTester {
         assertTrue(parser.isEof());
     }
 
-    @Test(expected = LispException.class)
+    @Test(expected = BadCharacterException.class)
     public void givenBadToken_ThrowsException() {
         String input = "[";
         LispParser parser = createLispParser(input);
@@ -216,7 +238,7 @@ public class LispParserTester {
         parser.getNextSExpression();
     }
 
-    @Test(expected = LispException.class)
+    @Test(expected = UnterminatedStringException.class)
     public void givenUnterminatedString_ThrowsException() {
         String input = "\"string";
         LispParser parser = createLispParser(input);
@@ -224,7 +246,7 @@ public class LispParserTester {
         parser.getNextSExpression();
     }
 
-    @Test(expected = LispException.class)
+    @Test(expected = EofEncounteredException.class)
     public void givenUnterminatedList_ThrowsException() {
         String input = "(bad list";
         LispParser parser = createLispParser(input);
@@ -232,7 +254,7 @@ public class LispParserTester {
         parser.getNextSExpression();
     }
 
-    @Test(expected = LispException.class)
+    @Test(expected = StartsWithRightParenthesisException.class)
     public void givenUnmatchedRightParenthesis_ThrowsException() {
         String input = ")";
         LispParser parser = createLispParser(input);
@@ -240,7 +262,7 @@ public class LispParserTester {
         parser.getNextSExpression();
     }
 
-    @Test(expected = LispException.class)
+    @Test(expected = BadCharacterException.class)
     public void givenBadCharacter_ThrowsExceptionAfterEofCalled() {
         String input = "[";
         LispParser parser = createLispParser(input);
@@ -254,7 +276,7 @@ public class LispParserTester {
         parser.getNextSExpression();
     }
 
-    @Test(expected = LispException.class)
+    @Test(expected = BadCharacterException.class)
     public void givenBadCharacterAfterValidToken_ThrowsExceptionAtTheCorrectTime() {
         String input = "id[";
         LispParser parser = createLispParser(input);
@@ -279,9 +301,22 @@ public class LispParserTester {
         try {
             parser.getNextSExpression();
             fail("Expected LispException");
-        } catch (LispException e) {
+        } catch (LispException e) {}
-        }
 
         assertTrue(parser.isEof());
     }
+
+    @Test(expected = UncheckedIOException.class)
+    public void handlesIOExceptionCorrectly() {
+        LispParser parser = createIOExceptionThrowingLispParser();
+
+        try {
+            parser.isEof();
+        } catch (LispException e) {
+            fail("Exception thrown too early");
+        }
+
+        parser.getNextSExpression();
+    }
+
 }

test/scanner/LispCommentRemovingInputStreamTester.java

@@ -1,8 +1,9 @@
 package scanner;
 
 import static org.junit.Assert.*;
+import static testutil.TestUtilities.createIOExceptionThrowingInputStream;
+import static testutil.TestUtilities.createInputStreamFromString;
 
-import java.io.IOException;
 import java.io.InputStream;
 
 import org.junit.Test;
@@ -10,7 +11,6 @@ import org.junit.Test;
 import error.ErrorManager;
 import scanner.LispInputStream.MaximumUnreadsExceededException;
 import scanner.LispInputStream.UncheckedIOException;
-import testutils.TestUtilities;
 
 public class LispCommentRemovingInputStreamTester {
 
@@ -19,7 +19,7 @@ public class LispCommentRemovingInputStreamTester {
     }
 
     private LispInputStream createLispInputStream(String inputString) {
-        InputStream stringInputStream = TestUtilities.createInputStreamFromString(inputString);
+        InputStream stringInputStream = createInputStreamFromString(inputString);
 
         return new LispCommentRemovingInputStream(stringInputStream);
     }
@@ -32,15 +32,6 @@ public class LispCommentRemovingInputStreamTester {
         return charactersRead.toString();
     }
 
-    private InputStream createIOExceptionThrowingInputStream() {
-        return new InputStream() {
-
-            public int read() throws IOException {
-                throw new IOException("test IOException");
-            }
-        };
-    }
-
     @Test
     public void noBytesIn_noBytesOut() {
         String input = "";

test/scanner/LispScannerLineColumnTester.java

@@ -1,12 +1,12 @@
 package scanner;
 
 import static org.junit.Assert.assertTrue;
+import static testutil.TestUtilities.createInputStreamFromString;
 
 import java.io.InputStream;
 
 import org.junit.Test;
 
-import testutils.TestUtilities;
 import token.Token;
 
 public class LispScannerLineColumnTester {
@@ -30,7 +30,7 @@ public class LispScannerLineColumnTester {
     }
 
     private void assertTokenLineAndColumnsMatch(String input, LineColumn[] expectedLineColumnList) {
-        InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
+        InputStream stringInputStream = createInputStreamFromString(input);
         LispScanner lispScanner = new LispScanner(stringInputStream, "testFile");
 
         for (LineColumn lineColumn : expectedLineColumnList) {

test/scanner/LispScannerTextTester.java

@@ -1,13 +1,12 @@
 package scanner;
 
 import static org.junit.Assert.assertEquals;
+import static testutil.TestUtilities.createInputStreamFromString;
 
 import java.io.InputStream;
 
 import org.junit.Test;
 
-import testutils.TestUtilities;
-
 public class LispScannerTextTester {
 
     private void assertTokenTextMatches(String input, String[] expectedTextList) {
@@ -24,13 +23,13 @@ public class LispScannerTextTester {
     }
 
     private LispScanner createLispScanner(String input) {
-        InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
+        InputStream stringInputStream = createInputStreamFromString(input);
 
         return new LispScanner(stringInputStream, "testFile");
     }
 
     private void assertInputFileNameMatches(String input, String expectedInputFileName) {
-        InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
+        InputStream stringInputStream = createInputStreamFromString(input);
         LispScanner lispScanner = new LispScanner(stringInputStream, expectedInputFileName);
 
         assertEquals(expectedInputFileName, lispScanner.getNextToken().getFileName());

test/scanner/LispScannerTypeTester.java

@@ -2,6 +2,7 @@ package scanner;
 
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static testutil.TestUtilities.createInputStreamFromString;
 
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -12,15 +13,8 @@ import org.junit.Test;
 
 import error.ErrorManager;
 import scanner.LispScanner.UnterminatedStringException;
-import testutils.TestUtilities;
+import token.*;
-import token.Eof;
-import token.Identifier;
-import token.LeftParenthesis;
 import token.Number;
-import token.QuoteMark;
-import token.QuotedString;
-import token.RightParenthesis;
-import token.Token;
 import token.TokenFactory.BadCharacterException;
 
 public class LispScannerTypeTester {
@@ -28,7 +22,7 @@ public class LispScannerTypeTester {
     private List<Class<? extends Token>> expectedTypes;
 
     private void assertTokenTypesMatch(String input) {
-        InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
+        InputStream stringInputStream = createInputStreamFromString(input);
         LispScanner lispScanner = new LispScanner(stringInputStream, "testFile");
 
         for (Class<? extends Token> type : expectedTypes)

test/testutil/TestUtilities.java

@@ -0,0 +1,20 @@
+package testutil;
+
+import java.io.*;
+
+public class TestUtilities {
+
+    public static InputStream createInputStreamFromString(String string) {
+        return new ByteArrayInputStream(string.getBytes());
+    }
+    
+    public static InputStream createIOExceptionThrowingInputStream() {
+        return new InputStream() {
+
+            public int read() throws IOException {
+                throw new IOException("test IOException");
+            }
+        };
+    }
+
+}

test/testutils/TestUtilities.java

@@ -1,12 +0,0 @@
-package testutils;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-
-public class TestUtilities {
-
-    public static InputStream createInputStreamFromString(String string) {
-        return new ByteArrayInputStream(string.getBytes());
-    }
-
-}