Added token classes and added unit tests

This commit is contained in:
Mike Cifelli 2016-12-12 10:15:20 -05:00
parent c02ef37f64
commit 6b6f349c29
20 changed files with 346 additions and 185 deletions

View File

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?><launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AFTER_CLEAN_TARGETS" value="jar,"/>
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AUTO_TARGETS" value="jar,"/>
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_CLEAN_TARGETS" value="clean,"/>
@ -7,6 +8,12 @@
<booleanAttribute key="org.eclipse.ant.ui.DEFAULT_VM_INSTALL" value="false"/>
<booleanAttribute key="org.eclipse.ant.uiSET_INPUTHANDLER" value="false"/>
<stringAttribute key="org.eclipse.debug.core.ATTR_REFRESH_SCOPE" value="${workspace}"/>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
<listEntry value="/LispInterpreter"/>
</listAttribute>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
<listEntry value="4"/>
</listAttribute>
<booleanAttribute key="org.eclipse.debug.core.capture_output" value="false"/>
<booleanAttribute key="org.eclipse.debug.ui.ATTR_CONSOLE_OUTPUT_ON" value="false"/>
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/>

View File

@ -1,120 +0,0 @@
/*
* Name: Mike Cifelli
* Course: CIS 443 - Programming Languages
* Assignment: Lisp Interpreter Phase 1 - Lexical Analysis
*/
package constructs;
import file.FilePosition;
/**
* A <code>Token</code> represents a token in Common Lisp.
*/
public class Token {
/**
* An enumeration representing all of the types of tokens found in Common
* Lisp.
*/
public enum Type {
/** A left parenthesis token */
LEFT_PAREN,
/** A right parenthesis token */
RIGHT_PAREN,
/** A quoted string token */
STRING,
/** A quote mark */
QUOTE_MARK,
/** A number token */
NUMBER,
/** An identifier token */
IDENTIFIER,
/** An end-of-file token */
EOF
}
private Type type;
private String text;
private String fName;
private int line;
private int column;
/**
* Create a new token with the specified type, text, file name, line number
* and column number.
*
* @param type
* the type of this token
* @param text
* the text associated with this token
* @param fName
* the name of the file that this token is located in
* @param line
* the line number that this token is found on
* @param column
* the column number that this token is found on
*/
public Token(Type type, String text, FilePosition position) {
this.type = type;
this.text = text;
this.fName = position.getFileName();
this.line = position.getLineNumber();
this.column = position.getColumnNumber();
}
/**
* Accessor method to determine the type of this token.
*
* @return the type of this token
*/
public Type getType() {
return type;
}
/**
* Accessor method to determine the text associated with this token.
*
* @return the text associated with this token
*/
public String getText() {
return text;
}
/**
* Accessor method to determine the name of the file that this token was
* located in.
*
* @return the name of the file that this token was located in
*/
public String getFName() {
return fName;
}
/**
* Accessor method to determine the line number that this token was found
* on.
*
* @return the line number this token was found on
*/
public int getLine() {
return line;
}
/**
* Accessor method to determine the column number that this token was found
* on.
*
* @return the column number this token was found on
*/
public int getColumn() {
return column;
}
}

View File

@ -8,8 +8,8 @@ package parser;
import java.io.InputStream;
import constructs.Token;
import scanner.LispScanner;
import token.Token;
/**
* A <code>LispParser</code> converts a stream of bytes into internal

View File

@ -8,7 +8,7 @@ import java.io.InputStream;
/**
* Removes Lisp comments from an input stream.
*/
public class LispFilterInputStream implements LispInputStream {
public class LispCommentRemovingInputStream implements LispInputStream {
private InputStream underlyingInputStream;
private boolean isInQuotedString;
@ -16,7 +16,7 @@ public class LispFilterInputStream implements LispInputStream {
private int previousCharacter;
private int currentCharacter;
public LispFilterInputStream(InputStream underlyingInputStream) {
public LispCommentRemovingInputStream(InputStream underlyingInputStream) {
this.underlyingInputStream = underlyingInputStream;
this.isInQuotedString = false;
this.rereadLastCharacter = false;

View File

@ -6,12 +6,12 @@ import java.io.InputStream;
import java.text.MessageFormat;
import java.util.function.Function;
import constructs.Token;
import constructs.TokenFactory;
import constructs.TokenFactoryImpl;
import error.LispException;
import file.FilePosition;
import file.FilePositionTracker;
import token.Token;
import token.TokenFactory;
import token.TokenFactoryImpl;
import util.Characters;
/**
@ -23,8 +23,8 @@ public class LispScanner {
private FilePositionTracker positionTracker;
private TokenFactory tokenFactory;
public LispScanner(InputStream in, String fileName) {
this.inputStream = new LispFilterInputStream(in);
public LispScanner(InputStream inputStream, String fileName) {
this.inputStream = new LispCommentRemovingInputStream(inputStream);
this.positionTracker = new FilePositionTracker(fileName);
this.tokenFactory = new TokenFactoryImpl();
}

17
src/token/Eof.java Normal file
View File

@ -0,0 +1,17 @@
package token;
import file.FilePosition;
public class Eof extends Token {
public Eof(String text, FilePosition position) {
super(text, position);
}
@Override
public Type getType() {
return Type.EOF;
}
}

17
src/token/Identifier.java Normal file
View File

@ -0,0 +1,17 @@
package token;
import file.FilePosition;
public class Identifier extends Token {
public Identifier(String text, FilePosition position) {
super(text, position);
}
@Override
public Type getType() {
return Type.IDENTIFIER;
}
}

View File

@ -0,0 +1,16 @@
package token;
import file.FilePosition;
public class LeftParenthesis extends Token {
public LeftParenthesis(String text, FilePosition position) {
super(text, position);
}
@Override
public Type getType() {
return Type.LEFT_PAREN;
}
}

17
src/token/Number.java Normal file
View File

@ -0,0 +1,17 @@
package token;
import file.FilePosition;
public class Number extends Token {
public Number(String text, FilePosition position) {
super(text, position);
}
@Override
public Type getType() {
return Type.NUMBER;
}
}

17
src/token/QuoteMark.java Normal file
View File

@ -0,0 +1,17 @@
package token;
import file.FilePosition;
public class QuoteMark extends Token {
public QuoteMark(String text, FilePosition position) {
super(text, position);
}
@Override
public Type getType() {
return Type.QUOTE_MARK;
}
}

View File

@ -0,0 +1,17 @@
package token;
import file.FilePosition;
public class QuotedString extends Token {
public QuotedString(String text, FilePosition position) {
super(text, position);
}
@Override
public Type getType() {
return Type.STRING;
}
}

View File

@ -0,0 +1,17 @@
package token;
import file.FilePosition;
public class RightParenthesis extends Token {
public RightParenthesis(String text, FilePosition position) {
super(text, position);
}
@Override
public Type getType() {
return Type.RIGHT_PAREN;
}
}

44
src/token/Token.java Normal file
View File

@ -0,0 +1,44 @@
package token;
import file.FilePosition;
/**
* A token in Lisp.
*/
public abstract class Token {
public enum Type {
LEFT_PAREN, RIGHT_PAREN, STRING, QUOTE_MARK, NUMBER, IDENTIFIER, EOF
}
private String text;
private String fileName;
private int line;
private int column;
public Token(String text, FilePosition position) {
this.text = text;
this.fileName = position.getFileName();
this.line = position.getLineNumber();
this.column = position.getColumnNumber();
}
public abstract Type getType();
public String getText() {
return text;
}
public String getFileName() {
return fileName;
}
public int getLine() {
return line;
}
public int getColumn() {
return column;
}
}

View File

@ -1,4 +1,4 @@
package constructs;
package token;
import java.text.MessageFormat;

View File

@ -1,4 +1,4 @@
package constructs;
package token;
import static util.Characters.*;
@ -12,18 +12,18 @@ public class TokenFactoryImpl implements TokenFactory {
switch (firstCharacter) {
case LEFT_PARENTHESIS:
return new Token(Token.Type.LEFT_PAREN, text, position);
return new LeftParenthesis(text, position);
case RIGHT_PARENTHESIS:
return new Token(Token.Type.RIGHT_PAREN, text, position);
return new RightParenthesis(text, position);
case SINGLE_QUOTE:
return new Token(Token.Type.QUOTE_MARK, text, position);
return new QuoteMark(text, position);
case DOUBLE_QUOTE:
return new Token(Token.Type.STRING, text, position);
return new QuotedString(text, position);
default:
if (Character.isDigit(firstCharacter)) {
return new Token(Token.Type.NUMBER, text, position);
return new Number(text, position);
} else if (Characters.isLegalIdentifierCharacter(firstCharacter)) {
return new Token(Token.Type.IDENTIFIER, text, position);
return new Identifier(text, position);
}
}
@ -31,7 +31,7 @@ public class TokenFactoryImpl implements TokenFactory {
}
public Token createEOFToken(FilePosition position) {
return new Token(Token.Type.EOF, "EOF", position);
return new Eof("EOF", position);
}
}

View File

@ -1,15 +1,19 @@
package scanner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.*;
import java.io.IOException;
import java.io.InputStream;
import org.junit.Before;
import org.junit.Test;
import error.ErrorManager;
import scanner.LispInputStream.MaximumUnreadsExceededException;
import scanner.LispInputStream.UncheckedIOException;
import testutils.TestUtilities;
public class LispFilterInputStreamTester {
public class LispCommentRemovingInputStreamTester {
private StringBuilder charactersRead;
@ -22,14 +26,14 @@ public class LispFilterInputStreamTester {
public void noBytesIn_noBytesOut() {
String input = "";
assertEquals(input, getLispFilterInputStreamResult(input));
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
}
@Test
public void oneCharacter_notRemoved() {
String input = "x";
assertEquals(input, getLispFilterInputStreamResult(input));
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
}
@Test
@ -37,7 +41,7 @@ public class LispFilterInputStreamTester {
String input = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "`1234567890-=~!@#$%^&*()_+[]\\',./{}|:\"<>?";
assertEquals(input, getLispFilterInputStreamResult(input));
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
}
@Test
@ -45,7 +49,7 @@ public class LispFilterInputStreamTester {
String input = ";comment";
String expectedResult = "";
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
}
@Test
@ -53,7 +57,7 @@ public class LispFilterInputStreamTester {
String input = ";comment1\n;comment2\n;comment3";
String expectedResult = "\n\n";
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
}
@Test
@ -61,7 +65,7 @@ public class LispFilterInputStreamTester {
String input = "()";
String expectedResult = "()";
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
}
@Test
@ -69,28 +73,28 @@ public class LispFilterInputStreamTester {
String input = "(;this is a comment\n)";
String expectedResult = "(\n)";
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
}
@Test
public void commentInString_NotRemoved() {
String input = "\"string;this should remain\"";
assertEquals(input, getLispFilterInputStreamResult(input));
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
}
@Test
public void commentInStringWithNewline_NotRemoved() {
String input = "\"string;this should\n remain\"";
assertEquals(input, getLispFilterInputStreamResult(input));
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
}
@Test
public void commentInStringWithEscapedDoubleQuote_NotRemoved() {
String input = "\"string \\\" ;this should remain\"";
assertEquals(input, getLispFilterInputStreamResult(input));
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
}
@Test
@ -98,7 +102,7 @@ public class LispFilterInputStreamTester {
String input = ";first comment \n '(1 2 3) \n ;second comment \n (defun add1 (x) (+ x 1)) ;third comment";
String expectedResult = "\n '(1 2 3) \n \n (defun add1 (x) (+ x 1)) ";
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
}
@Test
@ -142,16 +146,6 @@ public class LispFilterInputStreamTester {
assertEquals(expectedResult, lispInputStream.read());
}
@Test(expected = LispInputStream.MaximumUnreadsExceededException.class)
public void callUnreadMultipleTimes_ThrowsException() {
String input = "abc";
LispInputStream lispInputStream = createLispInputStream(input);
lispInputStream.read();
lispInputStream.unreadLastCharacter();
lispInputStream.unreadLastCharacter();
}
@Test
public void unreadNewlineInStringAfterComment_ReturnsNewline() {
String input = "a;123\n";
@ -165,14 +159,73 @@ public class LispFilterInputStreamTester {
assertEquals(expectedResult, lispInputStream.read());
}
private String getLispFilterInputStreamResult(String inputString) {
@Test(expected = MaximumUnreadsExceededException.class)
public void callUnreadMultipleTimes_ThrowsException() {
String input = "abc";
LispInputStream lispInputStream = createLispInputStream(input);
lispInputStream.read();
lispInputStream.unreadLastCharacter();
lispInputStream.unreadLastCharacter();
}
@Test()
public void callUnreadMultipleTimes_ExceptionHasCorrectSeverity() {
String input = "abc";
LispInputStream lispInputStream = createLispInputStream(input);
lispInputStream.read();
lispInputStream.unreadLastCharacter();
try {
lispInputStream.unreadLastCharacter();
} catch (MaximumUnreadsExceededException e) {
assertTrue(e.getSeverity() >= ErrorManager.CRITICAL_LEVEL);
}
}
@Test(expected = UncheckedIOException.class)
public void underlyingInputStreamThrowsIOException_ConvertsToUncheckedIOException() {
InputStream ioExceptionThrowingInputStream = createIOExceptionThrowingInputStream();
LispInputStream lispInputStream = new LispCommentRemovingInputStream(ioExceptionThrowingInputStream);
lispInputStream.read();
}
@Test()
public void underlyingInputStreamThrowsIOException_ExceptionHasCorrectSeverity() {
InputStream ioExceptionThrowingInputStream = createIOExceptionThrowingInputStream();
LispInputStream lispInputStream = new LispCommentRemovingInputStream(ioExceptionThrowingInputStream);
try {
lispInputStream.read();
} catch (UncheckedIOException e) {
assertTrue(e.getSeverity() >= ErrorManager.CRITICAL_LEVEL);
}
}
@Test()
public void underlyingInputStreamThrowsIOException_ExceptionHasErrorMessage() {
InputStream ioExceptionThrowingInputStream = createIOExceptionThrowingInputStream();
LispInputStream lispInputStream = new LispCommentRemovingInputStream(ioExceptionThrowingInputStream);
try {
lispInputStream.read();
} catch (UncheckedIOException e) {
String message = e.getMessage();
assertNotNull(message);
assertTrue(message.length() >= 0);
}
}
private String getLispCommentRemovingInputStreamResult(String inputString) {
return readInputStreamIntoString(createLispInputStream(inputString));
}
private LispInputStream createLispInputStream(String inputString) {
InputStream stringInputStream = TestUtilities.createInputStreamFromString(inputString);
return new LispFilterInputStream(stringInputStream);
return new LispCommentRemovingInputStream(stringInputStream);
}
private String readInputStreamIntoString(LispInputStream inputStream) {
@ -186,4 +239,13 @@ public class LispFilterInputStreamTester {
return charactersRead.toString();
}
private InputStream createIOExceptionThrowingInputStream() {
return new InputStream() {
public int read() throws IOException {
throw new IOException("test IOException");
}
};
}
}

View File

@ -7,8 +7,8 @@ import java.io.InputStream;
import org.junit.Before;
import org.junit.Test;
import constructs.Token;
import testutils.TestUtilities;
import token.Token;
public class LispScannerLineColumnTester {
@ -137,7 +137,6 @@ public class LispScannerLineColumnTester {
public boolean isEqual(Token token) {
return (this.line == token.getLine()) && (this.column == token.getColumn());
}
}
}

View File

@ -7,9 +7,42 @@ import java.io.InputStream;
import org.junit.Test;
import testutils.TestUtilities;
import token.Token;
public class LispScannerTextTester {
@Test
public void givenEmptyStream_RecordsCorrectFileName() {
String input = "";
String expectedFileName = "testFileName";
assertInputFileNameMatches(input, expectedFileName);
}
@Test
public void givenParenthesis_RecordsCorrectText() {
String input = "()";
String[] expected = { "(", ")" };
assertTokenTextMatches(input, expected);
}
@Test
public void givenQuote_RecordsCorrectText() {
String input = "'";
String expected = "'";
assertTokenTextMatches(input, expected);
}
@Test
public void givenEOF_ReordsCorrectText() {
String input = "";
String expected = "EOF";
assertTokenTextMatches(input, expected);
}
@Test
public void givenIdentifier_RecordsCorrectText() {
String input = "identifier";
@ -31,14 +64,6 @@ public class LispScannerTextTester {
assertTokenTextMatches(input, input);
}
@Test
public void givenEmptyStream_RecordsCorrectFileName() {
String input = "";
String expectedFileName = "testFileName";
assertInputFileNameMatches(input, expectedFileName);
}
@Test
public void givenNumberFollowedByComment_RecordsCorrectText() {
String input = "192837456;comment";
@ -50,11 +75,19 @@ public class LispScannerTextTester {
@Test
public void givenIdentifiersWithCommentBetween_RecordsCorrectText() {
String input = "abc123;comment\nabc222";
String expected = "abc123";
String[] expected = { "abc123", "abc222" };
assertTokenTextMatches(input, expected);
}
private void assertTokenTextMatches(String input, String[] expectedTextList) {
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream");
for (String expectedText : expectedTextList)
assertEquals(expectedText, lispScanner.getNextToken().getText());
}
private void assertTokenTextMatches(String input, String expectedText) {
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream");
@ -66,7 +99,7 @@ public class LispScannerTextTester {
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
LispScanner lispScanner = new LispScanner(stringInputStream, expectedInputFileName);
assertEquals(expectedInputFileName, lispScanner.getNextToken().getFName());
assertEquals(expectedInputFileName, lispScanner.getNextToken().getFileName());
}
}

View File

@ -6,11 +6,13 @@ import java.io.InputStream;
import org.junit.Test;
import constructs.Token;
import constructs.Token.Type;
import constructs.TokenFactory;
import error.ErrorManager;
import scanner.LispScanner.UnterminatedStringException;
import testutils.TestUtilities;
import token.Token;
import token.Token.Type;
import token.TokenFactory;
import token.TokenFactory.BadCharacterException;
public class LispScannerTypeTester {
@ -22,7 +24,7 @@ public class LispScannerTypeTester {
assertTokenTypesMatch(input, expectedTypes);
}
@Test(expected = TokenFactory.BadCharacterException.class)
@Test(expected = BadCharacterException.class)
public void givenBadCharacter_ThrowsException() {
String input = "[";
Token.Type[] expectedTypes = {};
@ -37,11 +39,25 @@ public class LispScannerTypeTester {
try {
assertTokenTypesMatch(input, expectedTypes);
} catch (TokenFactory.BadCharacterException e) {
} catch (BadCharacterException e) {
assertTrue(e.getSeverity() < ErrorManager.CRITICAL_LEVEL);
}
}
@Test
public void givenBadCharacter_ExceptionContainsMessage() {
String input = "abc\ndef[";
Token.Type[] expectedTypes = { Type.IDENTIFIER, Type.IDENTIFIER };
try {
assertTokenTypesMatch(input, expectedTypes);
} catch (BadCharacterException e) {
String message = e.getMessage();
assertNotNull(message);
assertTrue(message.length() > 0);
}
}
@Test
public void givenNil_ReturnsCorrectTypes() {
String input = "()";
@ -82,7 +98,7 @@ public class LispScannerTypeTester {
assertTokenTypesMatch(input, expectedTypes);
}
@Test(expected = LispScanner.UnterminatedStringException.class)
@Test(expected = UnterminatedStringException.class)
public void givenUnterminatedString_ThrowsException() {
String input = "\"oh no!";
Token.Type[] expectedTypes = { Type.STRING };

View File

@ -1,12 +1,14 @@
package constructs;
package token;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import org.junit.Before;
import org.junit.Test;
import file.FilePosition;
import token.TokenFactory.BadCharacterException;
public class TokenFactoryTester {
private TokenFactory tokenFactory;
@ -61,7 +63,7 @@ public class TokenFactoryTester {
assertEquals(Token.Type.STRING, tokenFactory.createToken(text, testPosition).getType());
}
@Test(expected = TokenFactory.BadCharacterException.class)
@Test(expected = BadCharacterException.class)
public void testBadCharacter() {
String text = "[abc]";
tokenFactory.createToken(text, testPosition);