mike
/
transcendental-lisp
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects Releases Wiki Activity

Added unit tests for LispScanner and refactored LispFilterInputStream

This commit is contained in:
Mike Cifelli 2016-12-08 15:30:40 -05:00
parent daf35a72fa
commit 702c11a7b9
7 changed files with 167 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/parser/LispParser.java
View File

@ -126,7 +126,7 @@ public class LispParser {
return sExpr(); return sExpr();
} }
// sExpr ::= NUMBER | IDENTIFIER | RESERVED | STRING | QUOTE_MARK sExpr | // sExpr ::= NUMBER | IDENTIFIER | STRING | QUOTE_MARK sExpr |
// LEFT_PAREN sExprTail // LEFT_PAREN sExprTail
// //
// Returns: an S-expression that matches the rules given above // Returns: an S-expression that matches the rules given above
@ -141,7 +141,6 @@ public class LispParser {
case NUMBER: case NUMBER:
return new LispNumber(nextToken.getText()); return new LispNumber(nextToken.getText());
case IDENTIFIER: case IDENTIFIER:
case RESERVED:
return new Symbol(nextToken.getText()); return new Symbol(nextToken.getText());
case STRING: case STRING:
return new LispString(nextToken.getText()); return new LispString(nextToken.getText());

49
src/scanner/LispFilterInputStream.java
View File

@ -9,44 +9,55 @@ import java.io.IOException;
*/ */
public class LispFilterInputStream extends FilterInputStream { public class LispFilterInputStream extends FilterInputStream {
private boolean inQuote; private boolean inQuotedString;
private int previousCharacter;
private int nextCharacter; private int nextCharacter;
public LispFilterInputStream(InputStream in) { public LispFilterInputStream(InputStream underlyingInputStream) {
super(in); super(underlyingInputStream);
inQuote = false;
inQuotedString = false;
previousCharacter = 0;
nextCharacter = 0; nextCharacter = 0;
} }
@Override @Override
public int read() throws IOException { public int read() throws IOException {
nextCharacter = super.read(); readNextCharacter();
if (haveEnteredComment()) { if (haveEnteredComment())
consumeAllBytesInComment(); consumeAllBytesInComment();
} else if (haveEncounteredStringBoundary()) {
inQuote = (!inQuote);
}
return nextCharacter; return nextCharacter;
} }
private void consumeAllBytesInComment() throws IOException { private void readNextCharacter() throws IOException {
while (stillInComment()) { previousCharacter = nextCharacter;
nextCharacter = super.read(); nextCharacter = super.read();
indicateEncounterWithStringBoundary();
} }
private void indicateEncounterWithStringBoundary() {
if (haveEncounteredStringBoundary())
inQuotedString = !inQuotedString;
}
private boolean haveEncounteredStringBoundary() {
return (previousCharacter != '\\') && (nextCharacter == '\"');
}
private boolean haveEnteredComment() {
return (nextCharacter == ';') && (!inQuotedString);
}
private void consumeAllBytesInComment() throws IOException {
while (stillInComment())
nextCharacter = super.read();
} }
private boolean stillInComment() { private boolean stillInComment() {
return (nextCharacter != '\n') && (nextCharacter != -1); return (nextCharacter != '\n') && (nextCharacter != -1);
} }
private boolean haveEnteredComment() {
return (nextCharacter == ';') && (!inQuote);
}
private boolean haveEncounteredStringBoundary() {
return nextCharacter == '\"';
}
} }

3
src/scanner/Token.java
View File

@ -31,9 +31,6 @@ public class Token {
/** A number token */ /** A number token */
NUMBER, NUMBER,
/** A reserved word token */
RESERVED,
/** An identifier token */ /** An identifier token */
IDENTIFIER, IDENTIFIER,

3
src/scanner/package.html
View File

@ -1,4 +1,3 @@
<body> <body>
Provides the classes necessary to perform a lexical analysis of the Lisp Provides the classes necessary to perform a lexical analysis of the Lisp programming language.
programming language.
</body> </body>

32
test/scanner/LispFilterInputStreamTester.java
View File

@ -2,20 +2,28 @@ package scanner;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import testutils.TestUtilities;
public class LispFilterInputStreamTester { public class LispFilterInputStreamTester {
private StringBuilder stringBuilder;
@Before
public void setUp() {
stringBuilder = new StringBuilder();
}
@Test @Test
public void oneComment_Removed() throws IOException { public void oneComment_Removed() throws IOException {
String input = ";comment"; String input = ";comment";
String expectedResult = ""; String expectedResult = "";
assertEquals(expectedResult, getLispFilterInputStreamResult(input)); assertEquals(expectedResult, getLispFilterInputStreamResult(input));
} }
@ -23,6 +31,7 @@ public class LispFilterInputStreamTester {
public void multipleComments_Removed() throws IOException { public void multipleComments_Removed() throws IOException {
String input = ";comment1\n;comment2\n;comment3"; String input = ";comment1\n;comment2\n;comment3";
String expectedResult = "\n\n"; String expectedResult = "\n\n";
assertEquals(expectedResult, getLispFilterInputStreamResult(input)); assertEquals(expectedResult, getLispFilterInputStreamResult(input));
} }
@ -30,6 +39,7 @@ public class LispFilterInputStreamTester {
public void nil_NotRemoved() throws IOException { public void nil_NotRemoved() throws IOException {
String input = "()"; String input = "()";
String expectedResult = "()"; String expectedResult = "()";
assertEquals(expectedResult, getLispFilterInputStreamResult(input)); assertEquals(expectedResult, getLispFilterInputStreamResult(input));
} }
@ -37,18 +47,28 @@ public class LispFilterInputStreamTester {
public void interiorComment_Removed() throws IOException { public void interiorComment_Removed() throws IOException {
String input = "(;this is a comment\n)"; String input = "(;this is a comment\n)";
String expectedResult = "(\n)"; String expectedResult = "(\n)";
assertEquals(expectedResult, getLispFilterInputStreamResult(input)); assertEquals(expectedResult, getLispFilterInputStreamResult(input));
} }
@Test @Test
public void commentInString_NotRemoved() throws IOException { public void commentInString_NotRemoved() throws IOException {
String input = "\"string;this should remain\""; String input = "\"string;this should remain\"";
assertEquals(input, getLispFilterInputStreamResult(input)); assertEquals(input, getLispFilterInputStreamResult(input));
} }
@Test @Test
public void commentInStringWithNewline_NotRemoved() throws IOException { public void commentInStringWithNewline_NotRemoved() throws IOException {
String input = "\"string;this should\n remain\""; String input = "\"string;this should\n remain\"";
assertEquals(input, getLispFilterInputStreamResult(input));
}
@Test
public void commentInStringWithEscapedDoubleQuote_NotRemoved() throws IOException {
String input = "\"string \\\" ;this should remain\"";
assertEquals(input, getLispFilterInputStreamResult(input)); assertEquals(input, getLispFilterInputStreamResult(input));
} }
@ -56,22 +76,18 @@ public class LispFilterInputStreamTester {
public void manyCommentsWithStatements_OnlyCommentsRemoved() throws IOException { public void manyCommentsWithStatements_OnlyCommentsRemoved() throws IOException {
String input = ";first comment \n '(1 2 3) \n ;second comment \n (defun add1 (x) (+ x 1)) ;third comment"; String input = ";first comment \n '(1 2 3) \n ;second comment \n (defun add1 (x) (+ x 1)) ;third comment";
String expectedResult = "\n '(1 2 3) \n \n (defun add1 (x) (+ x 1)) "; String expectedResult = "\n '(1 2 3) \n \n (defun add1 (x) (+ x 1)) ";
assertEquals(expectedResult, getLispFilterInputStreamResult(input)); assertEquals(expectedResult, getLispFilterInputStreamResult(input));
} }
private String getLispFilterInputStreamResult(String inputString) throws IOException { private String getLispFilterInputStreamResult(String inputString) throws IOException {
InputStream stringInputStream = createInputStreamFromString(inputString); InputStream stringInputStream = TestUtilities.createInputStreamFromString(inputString);
LispFilterInputStream lispFilterInputStream = new LispFilterInputStream(stringInputStream); LispFilterInputStream lispFilterInputStream = new LispFilterInputStream(stringInputStream);
return readInputStreamIntoString(lispFilterInputStream); return readInputStreamIntoString(lispFilterInputStream);
} }
private InputStream createInputStreamFromString(String string) {
return new ByteArrayInputStream(string.getBytes());
}
private String readInputStreamIntoString(InputStream inputStream) throws IOException { private String readInputStreamIntoString(InputStream inputStream) throws IOException {
StringBuilder stringBuilder = new StringBuilder();
int c = inputStream.read(); int c = inputStream.read();
while (c != -1) { while (c != -1) {

99
test/scanner/LispScannerTester.java Normal file
View File

@ -0,0 +1,99 @@
package scanner;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.io.InputStream;
import org.junit.Test;
import scanner.Token.Type;
import testutils.TestUtilities;
public class LispScannerTester {
@Test
public void givenEmptyFile_returnsCorrectTokenTypes() throws IOException {
String input = "";
Token.Type[] expectedTypes = {};
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenNil_returnsCorrectTokenTypes() throws IOException {
String input = "()";
Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.RIGHT_PAREN };
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenListOfNumbers_returnsCorrectTokenTypes() throws IOException {
String input = "(1 2)";
Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.NUMBER, Type.NUMBER, Type.RIGHT_PAREN };
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenString_returnsCorrectTokenTypes() throws IOException {
String input = "\"string\"";
Token.Type[] expectedTypes = { Type.STRING };
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenStringWithEscapedDoubleQuote_returnsCorrectTokenTypes() throws IOException {
String input = "\"string \n hi \\\" bye\"";
Token.Type[] expectedTypes = { Type.STRING };
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenStringWithEscapedDoubleQuoteAndComment_returnsCorrectTokenTypes() throws IOException {
String input = "\"string \n hi \\\" ; bye\"";
Token.Type[] expectedTypes = { Type.STRING };
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenIdentifier_returnsCorrectTokenTypes() throws IOException {
String input = "abcdefgHIJKLMNOP1234";
Token.Type[] expectedTypes = { Type.IDENTIFIER };
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenQuote_returnsCorrectTokenTypes() throws IOException {
String input = "'";
Token.Type[] expectedTypes = { Type.QUOTE_MARK };
assertTokenTypesMatch(input, expectedTypes);
}
@Test
public void givenFunctionCall_returnsCorrectTokenTypes() throws IOException {
String input = "(defun myFunction (x)\n (print x))";
Token.Type[] expectedTypes = { Type.LEFT_PAREN, Type.IDENTIFIER, Type.IDENTIFIER, Type.LEFT_PAREN,
Type.IDENTIFIER, Type.RIGHT_PAREN, Type.LEFT_PAREN, Type.IDENTIFIER,
Type.IDENTIFIER, Type.RIGHT_PAREN, Type.RIGHT_PAREN };
assertTokenTypesMatch(input, expectedTypes);
}
private void assertTokenTypesMatch(String input, Token.Type[] expectedTypeList) throws IOException {
InputStream stringInputStream = TestUtilities.createInputStreamFromString(input);
LispScanner lispScanner = new LispScanner(stringInputStream, "stringInputStream");
for (Token.Type expectedType : expectedTypeList)
assertEquals(expectedType, lispScanner.nextToken().getType());
assertEquals(Token.Type.EOF, lispScanner.nextToken().getType());
}
}

12
test/testutils/TestUtilities.java Normal file
View File

@ -0,0 +1,12 @@
package testutils;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
public class TestUtilities {
public static InputStream createInputStreamFromString(String string) {
return new ByteArrayInputStream(string.getBytes());
}
}