Added token classes and added unit tests
This commit is contained in:
parent
c02ef37f64
commit
6b6f349c29
|
@ -1,4 +1,5 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?><launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
|
<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
|
||||||
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AFTER_CLEAN_TARGETS" value="jar,"/>
|
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AFTER_CLEAN_TARGETS" value="jar,"/>
|
||||||
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AUTO_TARGETS" value="jar,"/>
|
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_AUTO_TARGETS" value="jar,"/>
|
||||||
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_CLEAN_TARGETS" value="clean,"/>
|
<stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_CLEAN_TARGETS" value="clean,"/>
|
||||||
|
@ -7,6 +8,12 @@
|
||||||
<booleanAttribute key="org.eclipse.ant.ui.DEFAULT_VM_INSTALL" value="false"/>
|
<booleanAttribute key="org.eclipse.ant.ui.DEFAULT_VM_INSTALL" value="false"/>
|
||||||
<booleanAttribute key="org.eclipse.ant.uiSET_INPUTHANDLER" value="false"/>
|
<booleanAttribute key="org.eclipse.ant.uiSET_INPUTHANDLER" value="false"/>
|
||||||
<stringAttribute key="org.eclipse.debug.core.ATTR_REFRESH_SCOPE" value="${workspace}"/>
|
<stringAttribute key="org.eclipse.debug.core.ATTR_REFRESH_SCOPE" value="${workspace}"/>
|
||||||
|
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
|
||||||
|
<listEntry value="/LispInterpreter"/>
|
||||||
|
</listAttribute>
|
||||||
|
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
|
||||||
|
<listEntry value="4"/>
|
||||||
|
</listAttribute>
|
||||||
<booleanAttribute key="org.eclipse.debug.core.capture_output" value="false"/>
|
<booleanAttribute key="org.eclipse.debug.core.capture_output" value="false"/>
|
||||||
<booleanAttribute key="org.eclipse.debug.ui.ATTR_CONSOLE_OUTPUT_ON" value="false"/>
|
<booleanAttribute key="org.eclipse.debug.ui.ATTR_CONSOLE_OUTPUT_ON" value="false"/>
|
||||||
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/>
|
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/>
|
||||||
|
|
|
@ -1,120 +0,0 @@
|
||||||
/*
|
|
||||||
* Name: Mike Cifelli
|
|
||||||
* Course: CIS 443 - Programming Languages
|
|
||||||
* Assignment: Lisp Interpreter Phase 1 - Lexical Analysis
|
|
||||||
*/
|
|
||||||
|
|
||||||
package constructs;
|
|
||||||
|
|
||||||
import file.FilePosition;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A <code>Token</code> represents a token in Common Lisp.
|
|
||||||
*/
|
|
||||||
public class Token {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An enumeration representing all of the types of tokens found in Common
|
|
||||||
* Lisp.
|
|
||||||
*/
|
|
||||||
public enum Type {
|
|
||||||
/** A left parenthesis token */
|
|
||||||
LEFT_PAREN,
|
|
||||||
|
|
||||||
/** A right parenthesis token */
|
|
||||||
RIGHT_PAREN,
|
|
||||||
|
|
||||||
/** A quoted string token */
|
|
||||||
STRING,
|
|
||||||
|
|
||||||
/** A quote mark */
|
|
||||||
QUOTE_MARK,
|
|
||||||
|
|
||||||
/** A number token */
|
|
||||||
NUMBER,
|
|
||||||
|
|
||||||
/** An identifier token */
|
|
||||||
IDENTIFIER,
|
|
||||||
|
|
||||||
/** An end-of-file token */
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
private Type type;
|
|
||||||
private String text;
|
|
||||||
private String fName;
|
|
||||||
private int line;
|
|
||||||
private int column;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new token with the specified type, text, file name, line number
|
|
||||||
* and column number.
|
|
||||||
*
|
|
||||||
* @param type
|
|
||||||
* the type of this token
|
|
||||||
* @param text
|
|
||||||
* the text associated with this token
|
|
||||||
* @param fName
|
|
||||||
* the name of the file that this token is located in
|
|
||||||
* @param line
|
|
||||||
* the line number that this token is found on
|
|
||||||
* @param column
|
|
||||||
* the column number that this token is found on
|
|
||||||
*/
|
|
||||||
public Token(Type type, String text, FilePosition position) {
|
|
||||||
this.type = type;
|
|
||||||
this.text = text;
|
|
||||||
this.fName = position.getFileName();
|
|
||||||
this.line = position.getLineNumber();
|
|
||||||
this.column = position.getColumnNumber();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Accessor method to determine the type of this token.
|
|
||||||
*
|
|
||||||
* @return the type of this token
|
|
||||||
*/
|
|
||||||
public Type getType() {
|
|
||||||
return type;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Accessor method to determine the text associated with this token.
|
|
||||||
*
|
|
||||||
* @return the text associated with this token
|
|
||||||
*/
|
|
||||||
public String getText() {
|
|
||||||
return text;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Accessor method to determine the name of the file that this token was
|
|
||||||
* located in.
|
|
||||||
*
|
|
||||||
* @return the name of the file that this token was located in
|
|
||||||
*/
|
|
||||||
public String getFName() {
|
|
||||||
return fName;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Accessor method to determine the line number that this token was found
|
|
||||||
* on.
|
|
||||||
*
|
|
||||||
* @return the line number this token was found on
|
|
||||||
*/
|
|
||||||
public int getLine() {
|
|
||||||
return line;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Accessor method to determine the column number that this token was found
|
|
||||||
* on.
|
|
||||||
*
|
|
||||||
* @return the column number this token was found on
|
|
||||||
*/
|
|
||||||
public int getColumn() {
|
|
||||||
return column;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -8,8 +8,8 @@ package parser;
|
||||||
|
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
|
||||||
import constructs.Token;
|
|
||||||
import scanner.LispScanner;
|
import scanner.LispScanner;
|
||||||
|
import token.Token;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A <code>LispParser</code> converts a stream of bytes into internal
|
* A <code>LispParser</code> converts a stream of bytes into internal
|
||||||
|
|
|
@ -8,7 +8,7 @@ import java.io.InputStream;
|
||||||
/**
|
/**
|
||||||
* Removes Lisp comments from an input stream.
|
* Removes Lisp comments from an input stream.
|
||||||
*/
|
*/
|
||||||
public class LispFilterInputStream implements LispInputStream {
|
public class LispCommentRemovingInputStream implements LispInputStream {
|
||||||
|
|
||||||
private InputStream underlyingInputStream;
|
private InputStream underlyingInputStream;
|
||||||
private boolean isInQuotedString;
|
private boolean isInQuotedString;
|
||||||
|
@ -16,7 +16,7 @@ public class LispFilterInputStream implements LispInputStream {
|
||||||
private int previousCharacter;
|
private int previousCharacter;
|
||||||
private int currentCharacter;
|
private int currentCharacter;
|
||||||
|
|
||||||
public LispFilterInputStream(InputStream underlyingInputStream) {
|
public LispCommentRemovingInputStream(InputStream underlyingInputStream) {
|
||||||
this.underlyingInputStream = underlyingInputStream;
|
this.underlyingInputStream = underlyingInputStream;
|
||||||
this.isInQuotedString = false;
|
this.isInQuotedString = false;
|
||||||
this.rereadLastCharacter = false;
|
this.rereadLastCharacter = false;
|
|
@ -6,12 +6,12 @@ import java.io.InputStream;
|
||||||
import java.text.MessageFormat;
|
import java.text.MessageFormat;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
|
|
||||||
import constructs.Token;
|
|
||||||
import constructs.TokenFactory;
|
|
||||||
import constructs.TokenFactoryImpl;
|
|
||||||
import error.LispException;
|
import error.LispException;
|
||||||
import file.FilePosition;
|
import file.FilePosition;
|
||||||
import file.FilePositionTracker;
|
import file.FilePositionTracker;
|
||||||
|
import token.Token;
|
||||||
|
import token.TokenFactory;
|
||||||
|
import token.TokenFactoryImpl;
|
||||||
import util.Characters;
|
import util.Characters;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -23,8 +23,8 @@ public class LispScanner {
|
||||||
private FilePositionTracker positionTracker;
|
private FilePositionTracker positionTracker;
|
||||||
private TokenFactory tokenFactory;
|
private TokenFactory tokenFactory;
|
||||||
|
|
||||||
public LispScanner(InputStream in, String fileName) {
|
public LispScanner(InputStream inputStream, String fileName) {
|
||||||
this.inputStream = new LispFilterInputStream(in);
|
this.inputStream = new LispCommentRemovingInputStream(inputStream);
|
||||||
this.positionTracker = new FilePositionTracker(fileName);
|
this.positionTracker = new FilePositionTracker(fileName);
|
||||||
this.tokenFactory = new TokenFactoryImpl();
|
this.tokenFactory = new TokenFactoryImpl();
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,17 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
|
||||||
|
public class Eof extends Token {
|
||||||
|
|
||||||
|
public Eof(String text, FilePosition position) {
|
||||||
|
super(text, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Type getType() {
|
||||||
|
return Type.EOF;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,17 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
|
||||||
|
public class Identifier extends Token {
|
||||||
|
|
||||||
|
public Identifier(String text, FilePosition position) {
|
||||||
|
super(text, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Type getType() {
|
||||||
|
return Type.IDENTIFIER;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,16 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
public class LeftParenthesis extends Token {
|
||||||
|
|
||||||
|
public LeftParenthesis(String text, FilePosition position) {
|
||||||
|
super(text, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Type getType() {
|
||||||
|
return Type.LEFT_PAREN;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,17 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
|
||||||
|
public class Number extends Token {
|
||||||
|
|
||||||
|
public Number(String text, FilePosition position) {
|
||||||
|
super(text, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Type getType() {
|
||||||
|
return Type.NUMBER;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,17 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
|
||||||
|
public class QuoteMark extends Token {
|
||||||
|
|
||||||
|
public QuoteMark(String text, FilePosition position) {
|
||||||
|
super(text, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Type getType() {
|
||||||
|
return Type.QUOTE_MARK;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,17 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
|
||||||
|
public class QuotedString extends Token {
|
||||||
|
|
||||||
|
public QuotedString(String text, FilePosition position) {
|
||||||
|
super(text, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Type getType() {
|
||||||
|
return Type.STRING;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,17 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
|
||||||
|
public class RightParenthesis extends Token {
|
||||||
|
|
||||||
|
public RightParenthesis(String text, FilePosition position) {
|
||||||
|
super(text, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Type getType() {
|
||||||
|
return Type.RIGHT_PAREN;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,44 @@
|
||||||
|
package token;
|
||||||
|
|
||||||
|
import file.FilePosition;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A token in Lisp.
|
||||||
|
*/
|
||||||
|
public abstract class Token {
|
||||||
|
|
||||||
|
public enum Type {
|
||||||
|
LEFT_PAREN, RIGHT_PAREN, STRING, QUOTE_MARK, NUMBER, IDENTIFIER, EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
private String text;
|
||||||
|
private String fileName;
|
||||||
|
private int line;
|
||||||
|
private int column;
|
||||||
|
|
||||||
|
public Token(String text, FilePosition position) {
|
||||||
|
this.text = text;
|
||||||
|
this.fileName = position.getFileName();
|
||||||
|
this.line = position.getLineNumber();
|
||||||
|
this.column = position.getColumnNumber();
|
||||||
|
}
|
||||||
|
|
||||||
|
public abstract Type getType();
|
||||||
|
|
||||||
|
public String getText() {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getFileName() {
|
||||||
|
return fileName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getLine() {
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getColumn() {
|
||||||
|
return column;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
package constructs;
|
package token;
|
||||||
|
|
||||||
import java.text.MessageFormat;
|
import java.text.MessageFormat;
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package constructs;
|
package token;
|
||||||
|
|
||||||
import static util.Characters.*;
|
import static util.Characters.*;
|
||||||
|
|
||||||
|
@ -12,18 +12,18 @@ public class TokenFactoryImpl implements TokenFactory {
|
||||||
|
|
||||||
switch (firstCharacter) {
|
switch (firstCharacter) {
|
||||||
case LEFT_PARENTHESIS:
|
case LEFT_PARENTHESIS:
|
||||||
return new Token(Token.Type.LEFT_PAREN, text, position);
|
return new LeftParenthesis(text, position);
|
||||||
case RIGHT_PARENTHESIS:
|
case RIGHT_PARENTHESIS:
|
||||||
return new Token(Token.Type.RIGHT_PAREN, text, position);
|
return new RightParenthesis(text, position);
|
||||||
case SINGLE_QUOTE:
|
case SINGLE_QUOTE:
|
||||||
return new Token(Token.Type.QUOTE_MARK, text, position);
|
return new QuoteMark(text, position);
|
||||||
case DOUBLE_QUOTE:
|
case DOUBLE_QUOTE:
|
||||||
return new Token(Token.Type.STRING, text, position);
|
return new QuotedString(text, position);
|
||||||
default:
|
default:
|
||||||
if (Character.isDigit(firstCharacter)) {
|
if (Character.isDigit(firstCharacter)) {
|
||||||
return new Token(Token.Type.NUMBER, text, position);
|
return new Number(text, position);
|
||||||
} else if (Characters.isLegalIdentifierCharacter(firstCharacter)) {
|
} else if (Characters.isLegalIdentifierCharacter(firstCharacter)) {
|
||||||
return new Token(Token.Type.IDENTIFIER, text, position);
|
return new Identifier(text, position);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -31,7 +31,7 @@ public class TokenFactoryImpl implements TokenFactory {
|
||||||
}
|
}
|
||||||
|
|
||||||
public Token createEOFToken(FilePosition position) {
|
public Token createEOFToken(FilePosition position) {
|
||||||
return new Token(Token.Type.EOF, "EOF", position);
|
return new Eof("EOF", position);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -1,15 +1,19 @@
|
||||||
package scanner;
|
package scanner;
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import error.ErrorManager;
|
||||||
|
import scanner.LispInputStream.MaximumUnreadsExceededException;
|
||||||
|
import scanner.LispInputStream.UncheckedIOException;
|
||||||
import testutils.TestUtilities;
|
import testutils.TestUtilities;
|
||||||
|
|
||||||
public class LispFilterInputStreamTester {
|
public class LispCommentRemovingInputStreamTester {
|
||||||
|
|
||||||
private StringBuilder charactersRead;
|
private StringBuilder charactersRead;
|
||||||
|
|
||||||
|
@ -22,14 +26,14 @@ public class LispFilterInputStreamTester {
|
||||||
public void noBytesIn_noBytesOut() {
|
public void noBytesIn_noBytesOut() {
|
||||||
String input = "";
|
String input = "";
|
||||||
|
|
||||||
assertEquals(input, getLispFilterInputStreamResult(input));
|
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void oneCharacter_notRemoved() {
|
public void oneCharacter_notRemoved() {
|
||||||
String input = "x";
|
String input = "x";
|
||||||
|
|
||||||
assertEquals(input, getLispFilterInputStreamResult(input));
|
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -37,7 +41,7 @@ public class LispFilterInputStreamTester {
|
||||||
String input = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
String input = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||||
+ "`1234567890-=~!@#$%^&*()_+[]\\',./{}|:\"<>?";
|
+ "`1234567890-=~!@#$%^&*()_+[]\\',./{}|:\"<>?";
|
||||||
|
|
||||||
assertEquals(input, getLispFilterInputStreamResult(input));
|
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -45,7 +49,7 @@ public class LispFilterInputStreamTester {
|
||||||
String input = ";comment";
|
String input = ";comment";
|
||||||
String expectedResult = "";
|
String expectedResult = "";
|
||||||
|
|
||||||
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
|
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -53,7 +57,7 @@ public class LispFilterInputStreamTester {
|
||||||
String input = ";comment1\n;comment2\n;comment3";
|
String input = ";comment1\n;comment2\n;comment3";
|
||||||
String expectedResult = "\n\n";
|
String expectedResult = "\n\n";
|
||||||
|
|
||||||
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
|
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -61,7 +65,7 @@ public class LispFilterInputStreamTester {
|
||||||
String input = "()";
|
String input = "()";
|
||||||
String expectedResult = "()";
|
String expectedResult = "()";
|
||||||
|
|
||||||
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
|
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -69,28 +73,28 @@ public class LispFilterInputStreamTester {
|
||||||
String input = "(;this is a comment\n)";
|
String input = "(;this is a comment\n)";
|
||||||
String expectedResult = "(\n)";
|
String expectedResult = "(\n)";
|
||||||
|
|
||||||
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
|
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void commentInString_NotRemoved() {
|
public void commentInString_NotRemoved() {
|
||||||
String input = "\"string;this should remain\"";
|
String input = "\"string;this should remain\"";
|
||||||
|
|
||||||
assertEquals(input, getLispFilterInputStreamResult(input));
|
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void commentInStringWithNewline_NotRemoved() {
|
public void commentInStringWithNewline_NotRemoved() {
|
||||||
String input = "\"string;this should\n remain\"";
|
String input = "\"string;this should\n remain\"";
|
||||||
|
|
||||||
assertEquals(input, getLispFilterInputStreamResult(input));
|
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void commentInStringWithEscapedDoubleQuote_NotRemoved() {
|
public void commentInStringWithEscapedDoubleQuote_NotRemoved() {
|
||||||
String input = "\"string \\\" ;this should remain\"";
|
String input = "\"string \\\" ;this should remain\"";
|
||||||
|
|
||||||
assertEquals(input, getLispFilterInputStreamResult(input));
|
assertEquals(input, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -98,7 +102,7 @@ public class LispFilterInputStreamTester {
|
||||||
String input = ";first comment \n '(1 2 3) \n ;second comment \n (defun add1 (x) (+ x 1)) ;third comment";
|
String input = ";first comment \n '(1 2 3) \n ;second comment \n (defun add1 (x) (+ x 1)) ;third comment";
|
||||||
String expectedResult = "\n '(1 2 3) \n \n (defun add1 (x) (+ x 1)) ";
|
String expectedResult = "\n '(1 2 3) \n \n (defun add1 (x) (+ x 1)) ";
|
||||||
|
|
||||||
assertEquals(expectedResult, getLispFilterInputStreamResult(input));
|
assertEquals(expectedResult, getLispCommentRemovingInputStreamResult(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -142,16 +146,6 @@ public class LispFilterInputStreamTester {
|
||||||
assertEquals(expectedResult, lispInputStream.read());
|
assertEquals(expectedResult, lispInputStream.read());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = LispInputStream.MaximumUnreadsExceededException.class)
|
|
||||||
public void callUnreadMultipleTimes_ThrowsException() {
|
|
||||||
String input = "abc";
|
|
||||||
LispInputStream lispInputStream = createLispInputStream(input);
|
|
||||||
|
|
||||||
lispInputStream.read();
|
|
||||||
lispInputStream.unreadLastCharacter();
|
|
||||||
lispInputStream.unreadLastCharacter();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void unreadNewlineInStringAfterComment_ReturnsNewline() {
|
public void unreadNewlineInStringAfterComment_ReturnsNewline() {
|
||||||
String input = "a;123\n";
|
String input = "a;123\n";
|
||||||
|
@ -165,14 +159,73 @@ public class LispFilterInputStreamTester {
|
||||||
assertEquals(expectedResult, lispInputStream.read());
|
assertEquals(expectedResult, lispInputStream.read());
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getLispFilterInputStreamResult(String inputString) {
|
@Test(expected = MaximumUnreadsExceededException.class)
|
||||||
|
public void callUnreadMultipleTimes_ThrowsException() {
|
||||||
|
String input = "abc";
|
||||||
|
LispInputStream lispInputStream = createLispInputStream(input);
|
||||||
|
|
||||||
|
lispInputStream.read();
|
||||||
|
lispInputStream.unreadLastCharacter();
|
||||||
|
lispInputStream.unreadLastCharacter();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void callUnreadMultipleTimes_ExceptionHasCorrectSeverity() {
|
||||||
|
String input = "abc";
|
||||||
|
LispInputStream lispInputStream = createLispInputStream(input);
|
||||||
|
|
||||||
|
lispInputStream.read();
|
||||||
|
lispInputStream.unreadLastCharacter();
|
||||||
|
|
||||||
|
try {
|
||||||
|
lispInputStream.unreadLastCharacter();
|
||||||
|
} catch (MaximumUnreadsExceededException e) {
|
||||||
|
assertTrue(e.getSeverity() >= ErrorManager.CRITICAL_LEVEL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(expected = UncheckedIOException.class)
|
||||||
|
public void underlyingInputStreamThrowsIOException_ConvertsToUncheckedIOException() {
|
||||||
|
InputStream ioExceptionThrowingInputStream = createIOExceptionThrowingInputStream();
|
||||||
|
LispInputStream lispInputStream = new LispCommentRemovingInputStream(ioExceptionThrowingInputStream);
|
||||||
|
|
||||||
|
lispInputStream.read();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void underlyingInputStreamThrowsIOException_ExceptionHasCorrectSeverity() {
|
||||||
|
InputStream ioExceptionThrowingInputStream = createIOExceptionThrowingInputStream();
|
||||||
|
LispInputStream lispInputStream = new LispCommentRemovingInputStream(ioExceptionThrowingInputStream);
|
||||||
|
|
||||||
|
try {
|
||||||
|
lispInputStream.read();
|
||||||
|
} catch (UncheckedIOException e) {
|
||||||
|
assertTrue(e.getSeverity() >= ErrorManager.CRITICAL_LEVEL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test()
|
||||||
|
public void underlyingInputStreamThrowsIOException_ExceptionHasErrorMessage() {
|
||||||
|
InputStream ioExceptionThrowingInputStream = createIOExceptionThrowingInputStream();
|
||||||
|
LispInputStream lispInputStream = new LispCommentRemovingInputStream(ioExceptionThrowingInputStream);
|
||||||
|
|
||||||
|
try {
|
||||||
|
lispInputStream.read();
|
||||||
|
} catch (UncheckedIOException e) {
|
||||||
|
String message = e.getMessage();
|
||||||
|
assertNotNull(message);
|
||||||
|
assertTrue(message.length() >= 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getLispCommentRemovingInputStreamResult(String inputString) {
|
||||||
return readInputStreamIntoString(createLispInputStream(inputString));
|
return readInputStreamIntoString(createLispInputStream(inputString));
|
||||||
}
|
}
|
||||||
|
|
||||||
private LispInputStream createLispInputStream(String inputString) {
|
private LispInputStream createLispInputStream(String inputString) {
|
||||||
InputStream stringInputStream = TestUtilities.createInputStreamFromString(inputString);
|
InputStream stringInputStream = TestUtilities.createInputStreamFromString(inputString);
|
||||||
|
|
||||||
return new LispFilterInputStream(stringInputStream);
|
return new LispCommentRemovingInputStream(stringInputStream);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String readInputStreamIntoString(LispInputStream inputStream) {
|
private String readInputStreamIntoString(LispInputStream inputStream) {
|
||||||
|
@ -186,4 +239,13 @@ public class LispFilterInputStreamTester {
|
||||||
return charactersRead.toString();
|
return charactersRead.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private InputStream createIOExceptionThrowingInputStream() {
|
||||||
|
return new InputStream() {
|
||||||
|
|
||||||
|
public int read() throws IOException {
|
||||||
|
throw new IOException("test IOException");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -7,8 +7,8 @@ import java.io.InputStream;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import constructs.Token;
|
|
||||||
import testutils.TestUtilities;
|
import testutils.TestUtilities;
|
||||||
|
import token.Token;
|
||||||
|
|
||||||
public class LispScannerLineColumnTester {
|
public class LispScannerLineColumnTester {
|
||||||
|
|
||||||
|
@ -137,7 +137,6 @@ public class LispScannerLineColumnTester {
|
||||||
public boolean isEqual(Token token) {
|
public boolean isEqual(Token token) {
|
||||||
return (this.line == token.getLine()) && (this.column == token.getColumn());
|
return (this.line == token.getLine()) && (this.column == token.getColumn());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,9 +7,42 @@ import java.io.InputStream;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import testutils.TestUtilities;
|
import testutils.TestUtilities;
|
||||||
|
import token.Token;
|
||||||
|
|
||||||
public class LispScannerTextTester {
|
public class LispScannerTextTester {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void givenEmptyStream_RecordsCorrectFileName() {
|
||||||
|
String input = "";
|
||||||
|
String expectedFileName = "testFileName";
|
||||||
|
|
||||||
|
assertInputFileNameMatches(input, expectedFileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void givenParenthesis_RecordsCorrectText() {
|
||||||
|
String input = "()";
|
||||||
|
String[] expected = { "(", ")" };
|
||||||
|
|
||||||
|
assertTokenTextMatches(input, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void givenQuote_RecordsCorrectText() {
|
||||||
|
String input = "'";
|
||||||
|
String expected = "'";
|
||||||
|
|
||||||
|
assertTokenTextMatches(input, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void givenEOF_ReordsCorrectText() {
|
||||||
|
String input = "";
|
||||||
|
String expected = "EOF";
|
||||||
|
|
||||||
|
assertTokenTextMatches(input, expected);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void givenIdentifier_RecordsCorrectText() {
|
public void givenIdentifier_RecordsCorrectText() {
|
||||||
String input = "identifier";
|
String input = "identifier";
|
||||||
|
@ -31,14 +64,6 @@ public class LispScannerTextTester {
|
||||||
assertTokenTextMatches(input, input);
|
assertTokenTextMatches(input, input);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void givenEmptyStream_RecordsCorrectFileName() {
|
|
||||||
String input = "";
|
|
||||||
String expectedFileName = "testFileName";
|
|
||||||
|
|
||||||
assertInputFileNameMatches(input, expectedFileName);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void givenNumberFollowedByComment_RecordsCorrectText() {
|
public void givenNumberFollowedByComment_RecordsCorrectText() {
|
||||||
String input = "192837456;comment";
|
String input = "192837456;comment";
|
||||||
|
@ -50,11 +75,19 @@ public class LispScannerTextTester {
|
||||||
@Test
|
@Test
|
||||||
public void givenIdentifiersWithCommentBetween_RecordsCorrectText() {
|
public void givenIdentifiersWithCommentBetween_RecordsCorrectText() {
|
||||||
String input = "abc123;comment\nabc222";
|
String input = "abc123;comment\nabc222";
|
||||||
String expected = "abc123";
|
String[] expected = { "abc123", "abc222" };
|
||||||
|
|
||||||
assertTokenTextMatches(input, expected);
|
assertTokenTextMatches(input, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void assertTokenTextMatches(String input, String[] expectedTextList) {
|
||||||
|
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
|
||||||
|
LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream");
|
||||||
|
|
||||||
|
for (String expectedText : expectedTextList)
|
||||||
|
assertEquals(expectedText, lispScanner.getNextToken().getText());
|
||||||
|
}
|
||||||
|
|
||||||
private void assertTokenTextMatches(String input, String expectedText) {
|
private void assertTokenTextMatches(String input, String expectedText) {
|
||||||
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
|
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
|
||||||
LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream");
|
LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream");
|
||||||
|
@ -66,7 +99,7 @@ public class LispScannerTextTester {
|
||||||
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
|
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
|
||||||
LispScanner lispScanner = new LispScanner(stringInputStream, expectedInputFileName);
|
LispScanner lispScanner = new LispScanner(stringInputStream, expectedInputFileName);
|
||||||
|
|
||||||
assertEquals(expectedInputFileName, lispScanner.getNextToken().getFName());
|
assertEquals(expectedInputFileName, lispScanner.getNextToken().getFileName());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,11 +6,13 @@ import java.io.InputStream;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import constructs.Token;
|
|
||||||
import constructs.Token.Type;
|
|
||||||
import constructs.TokenFactory;
|
|
||||||
import error.ErrorManager;
|
import error.ErrorManager;
|
||||||
|
import scanner.LispScanner.UnterminatedStringException;
|
||||||
import testutils.TestUtilities;
|
import testutils.TestUtilities;
|
||||||
|
import token.Token;
|
||||||
|
import token.Token.Type;
|
||||||
|
import token.TokenFactory;
|
||||||
|
import token.TokenFactory.BadCharacterException;
|
||||||
|
|
||||||
public class LispScannerTypeTester {
|
public class LispScannerTypeTester {
|
||||||
|
|
||||||
|
@ -22,7 +24,7 @@ public class LispScannerTypeTester {
|
||||||
assertTokenTypesMatch(input, expectedTypes);
|
assertTokenTypesMatch(input, expectedTypes);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = TokenFactory.BadCharacterException.class)
|
@Test(expected = BadCharacterException.class)
|
||||||
public void givenBadCharacter_ThrowsException() {
|
public void givenBadCharacter_ThrowsException() {
|
||||||
String input = "[";
|
String input = "[";
|
||||||
Token.Type[] expectedTypes = {};
|
Token.Type[] expectedTypes = {};
|
||||||
|
@ -37,11 +39,25 @@ public class LispScannerTypeTester {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
assertTokenTypesMatch(input, expectedTypes);
|
assertTokenTypesMatch(input, expectedTypes);
|
||||||
} catch (TokenFactory.BadCharacterException e) {
|
} catch (BadCharacterException e) {
|
||||||
assertTrue(e.getSeverity() < ErrorManager.CRITICAL_LEVEL);
|
assertTrue(e.getSeverity() < ErrorManager.CRITICAL_LEVEL);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void givenBadCharacter_ExceptionContainsMessage() {
|
||||||
|
String input = "abc\ndef[";
|
||||||
|
Token.Type[] expectedTypes = { Type.IDENTIFIER, Type.IDENTIFIER };
|
||||||
|
|
||||||
|
try {
|
||||||
|
assertTokenTypesMatch(input, expectedTypes);
|
||||||
|
} catch (BadCharacterException e) {
|
||||||
|
String message = e.getMessage();
|
||||||
|
assertNotNull(message);
|
||||||
|
assertTrue(message.length() > 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void givenNil_ReturnsCorrectTypes() {
|
public void givenNil_ReturnsCorrectTypes() {
|
||||||
String input = "()";
|
String input = "()";
|
||||||
|
@ -82,7 +98,7 @@ public class LispScannerTypeTester {
|
||||||
assertTokenTypesMatch(input, expectedTypes);
|
assertTokenTypesMatch(input, expectedTypes);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = LispScanner.UnterminatedStringException.class)
|
@Test(expected = UnterminatedStringException.class)
|
||||||
public void givenUnterminatedString_ThrowsException() {
|
public void givenUnterminatedString_ThrowsException() {
|
||||||
String input = "\"oh no!";
|
String input = "\"oh no!";
|
||||||
Token.Type[] expectedTypes = { Type.STRING };
|
Token.Type[] expectedTypes = { Type.STRING };
|
||||||
|
|
|
@ -1,12 +1,14 @@
|
||||||
package constructs;
|
package token;
|
||||||
|
|
||||||
import static org.junit.Assert.*;
|
import static org.junit.Assert.assertEquals;
|
||||||
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import file.FilePosition;
|
import file.FilePosition;
|
||||||
|
|
||||||
|
import token.TokenFactory.BadCharacterException;
|
||||||
|
|
||||||
public class TokenFactoryTester {
|
public class TokenFactoryTester {
|
||||||
|
|
||||||
private TokenFactory tokenFactory;
|
private TokenFactory tokenFactory;
|
||||||
|
@ -61,7 +63,7 @@ public class TokenFactoryTester {
|
||||||
assertEquals(Token.Type.STRING, tokenFactory.createToken(text, testPosition).getType());
|
assertEquals(Token.Type.STRING, tokenFactory.createToken(text, testPosition).getType());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = TokenFactory.BadCharacterException.class)
|
@Test(expected = BadCharacterException.class)
|
||||||
public void testBadCharacter() {
|
public void testBadCharacter() {
|
||||||
String text = "[abc]";
|
String text = "[abc]";
|
||||||
tokenFactory.createToken(text, testPosition);
|
tokenFactory.createToken(text, testPosition);
|
Loading…
Reference in New Issue