Convert streams to kotlin
This commit is contained in:
parent
c5fbab6813
commit
0f34d25e12
|
|
@ -1,78 +0,0 @@
|
|||
package scanner;
|
||||
|
||||
import stream.SafeInputStream;
|
||||
import util.Characters;
|
||||
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* Removes Lisp comments from an input stream.
|
||||
*/
|
||||
public class LispCommentRemovingInputStream implements LispInputStream {
|
||||
|
||||
private SafeInputStream underlyingInputStream;
|
||||
private boolean isInQuotedString;
|
||||
private boolean rereadLastCharacter;
|
||||
private int previousCharacter;
|
||||
private int currentCharacter;
|
||||
|
||||
public LispCommentRemovingInputStream(InputStream underlyingInputStream) {
|
||||
this.underlyingInputStream = new SafeInputStream(underlyingInputStream);
|
||||
this.isInQuotedString = false;
|
||||
this.rereadLastCharacter = false;
|
||||
this.previousCharacter = 0;
|
||||
this.currentCharacter = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() {
|
||||
if (!rereadLastCharacter)
|
||||
return readFromUnderlyingInputStream();
|
||||
|
||||
rereadLastCharacter = false;
|
||||
|
||||
return currentCharacter;
|
||||
}
|
||||
|
||||
private int readFromUnderlyingInputStream() {
|
||||
readNextCharacter();
|
||||
|
||||
if (haveEnteredComment())
|
||||
consumeAllBytesInComment();
|
||||
|
||||
return currentCharacter;
|
||||
}
|
||||
|
||||
private void readNextCharacter() {
|
||||
previousCharacter = currentCharacter;
|
||||
currentCharacter = underlyingInputStream.read();
|
||||
|
||||
if (haveEncounteredStringBoundary())
|
||||
isInQuotedString = !isInQuotedString;
|
||||
}
|
||||
|
||||
private boolean haveEncounteredStringBoundary() {
|
||||
return (previousCharacter != Characters.BACKSLASH) && (currentCharacter == Characters.DOUBLE_QUOTE);
|
||||
}
|
||||
|
||||
private boolean haveEnteredComment() {
|
||||
return (currentCharacter == Characters.SEMICOLON) && (!isInQuotedString);
|
||||
}
|
||||
|
||||
private void consumeAllBytesInComment() {
|
||||
while (stillInComment())
|
||||
currentCharacter = underlyingInputStream.read();
|
||||
}
|
||||
|
||||
private boolean stillInComment() {
|
||||
return (currentCharacter != Characters.NEWLINE) && (currentCharacter != Characters.EOF);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unreadLastCharacter() {
|
||||
if (rereadLastCharacter)
|
||||
throw new MaximumUnreadsExceededException();
|
||||
|
||||
this.rereadLastCharacter = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
package scanner
|
||||
|
||||
import stream.SafeInputStream
|
||||
import util.Characters.BACKSLASH
|
||||
import util.Characters.DOUBLE_QUOTE
|
||||
import util.Characters.EOF
|
||||
import util.Characters.NEWLINE
|
||||
import util.Characters.SEMICOLON
|
||||
import java.io.InputStream
|
||||
|
||||
/**
|
||||
* Removes Lisp comments from an input stream.
|
||||
*/
|
||||
class LispCommentRemovingInputStream(underlyingInputStream: InputStream) : LispInputStream {
|
||||
|
||||
private val underlyingInputStream = SafeInputStream(underlyingInputStream)
|
||||
private var isInQuotedString = false
|
||||
private var rereadLastCharacter = false
|
||||
private var previousCharacter = 0
|
||||
private var currentCharacter = 0
|
||||
|
||||
override fun read(): Int {
|
||||
if (!rereadLastCharacter)
|
||||
return readFromUnderlyingInputStream()
|
||||
|
||||
rereadLastCharacter = false
|
||||
|
||||
return currentCharacter
|
||||
}
|
||||
|
||||
override fun unreadLastCharacter() {
|
||||
if (rereadLastCharacter)
|
||||
throw LispInputStream.MaximumUnreadsExceededException()
|
||||
|
||||
this.rereadLastCharacter = true
|
||||
}
|
||||
|
||||
private fun readFromUnderlyingInputStream(): Int {
|
||||
readNextCharacter()
|
||||
|
||||
if (haveEnteredComment())
|
||||
consumeAllBytesInComment()
|
||||
|
||||
return currentCharacter
|
||||
}
|
||||
|
||||
private fun readNextCharacter() {
|
||||
previousCharacter = currentCharacter
|
||||
currentCharacter = underlyingInputStream.read()
|
||||
|
||||
if (haveEncounteredStringBoundary())
|
||||
isInQuotedString = !isInQuotedString
|
||||
}
|
||||
|
||||
private fun haveEncounteredStringBoundary() =
|
||||
previousCharacter != BACKSLASH.toInt() && currentCharacter == DOUBLE_QUOTE.toInt()
|
||||
|
||||
private fun haveEnteredComment() = currentCharacter == SEMICOLON.toInt() && !isInQuotedString
|
||||
|
||||
private fun consumeAllBytesInComment() {
|
||||
while (stillInComment())
|
||||
currentCharacter = underlyingInputStream.read()
|
||||
}
|
||||
|
||||
private fun stillInComment() = currentCharacter != NEWLINE.toInt() && currentCharacter != EOF
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
package scanner;
|
||||
|
||||
import error.CriticalLispException;
|
||||
|
||||
public interface LispInputStream {
|
||||
|
||||
int read();
|
||||
|
||||
void unreadLastCharacter();
|
||||
|
||||
public static class MaximumUnreadsExceededException extends CriticalLispException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package scanner
|
||||
|
||||
import error.CriticalLispException
|
||||
|
||||
interface LispInputStream {
|
||||
|
||||
fun read(): Int
|
||||
fun unreadLastCharacter()
|
||||
|
||||
class MaximumUnreadsExceededException : CriticalLispException()
|
||||
}
|
||||
|
|
@ -1,181 +0,0 @@
|
|||
package scanner;
|
||||
|
||||
import error.LineColumnException;
|
||||
import file.FilePosition;
|
||||
import file.FilePositionTracker;
|
||||
import token.Token;
|
||||
import token.TokenFactory;
|
||||
import token.TokenFactoryImpl;
|
||||
import util.Characters;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static java.lang.Character.isDigit;
|
||||
import static java.lang.Character.isWhitespace;
|
||||
|
||||
/**
|
||||
* Converts a stream of bytes into a stream of Lisp tokens.
|
||||
*/
|
||||
public class LispScanner {
|
||||
|
||||
private LispInputStream inputStream;
|
||||
private FilePositionTracker positionTracker;
|
||||
private TokenFactory tokenFactory;
|
||||
|
||||
public LispScanner(InputStream inputStream, String fileName) {
|
||||
this.inputStream = new LispCommentRemovingInputStream(inputStream);
|
||||
this.positionTracker = new FilePositionTracker(fileName);
|
||||
this.tokenFactory = new TokenFactoryImpl();
|
||||
}
|
||||
|
||||
public Token getNextToken() {
|
||||
for (int c = inputStream.read(); c != Characters.EOF; c = inputStream.read()) {
|
||||
char currentCharacter = (char) c;
|
||||
positionTracker.incrementColumn();
|
||||
|
||||
if (!isWhitespace(currentCharacter))
|
||||
return createTokenFromCharacter(currentCharacter);
|
||||
else if (currentCharacter == Characters.NEWLINE)
|
||||
positionTracker.incrementLine();
|
||||
}
|
||||
|
||||
return tokenFactory.createEofToken(positionTracker.currentPosition());
|
||||
}
|
||||
|
||||
private Token createTokenFromCharacter(char c) {
|
||||
FilePosition currentPosition = positionTracker.currentPosition();
|
||||
String tokenText = retrieveTokenText(c);
|
||||
|
||||
return tokenFactory.createToken(tokenText, currentPosition);
|
||||
}
|
||||
|
||||
private String retrieveTokenText(char firstCharacter) {
|
||||
String tokenText = "" + firstCharacter;
|
||||
|
||||
if (firstCharacter == Characters.DOUBLE_QUOTE)
|
||||
tokenText = retrieveStringTokenText(firstCharacter);
|
||||
else if (Characters.INSTANCE.isNumberPrefix(firstCharacter))
|
||||
tokenText = retrieveNumberOrIdentifierTokenText(firstCharacter);
|
||||
else if (isDigit(firstCharacter))
|
||||
tokenText = retrieveNumberTokenText(firstCharacter);
|
||||
else if (Characters.INSTANCE.isLegalIdentifierCharacter(firstCharacter))
|
||||
tokenText = retrieveIdentifierTokenText(firstCharacter);
|
||||
|
||||
return tokenText;
|
||||
}
|
||||
|
||||
private String retrieveStringTokenText(char firstDoubleQuote) {
|
||||
ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDoubleQuote,
|
||||
Characters.INSTANCE::isLegalStringCharacter);
|
||||
|
||||
return retriever.retrieveToken();
|
||||
}
|
||||
|
||||
private String retrieveNumberOrIdentifierTokenText(char firstCharacter) {
|
||||
char nextCharacter = (char) inputStream.read();
|
||||
inputStream.unreadLastCharacter();
|
||||
|
||||
if (isDigit(nextCharacter))
|
||||
return retrieveNumberTokenText(firstCharacter);
|
||||
|
||||
return retrieveIdentifierTokenText(firstCharacter);
|
||||
}
|
||||
|
||||
private String retrieveNumberTokenText(char firstCharacter) {
|
||||
ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstCharacter, Character::isDigit);
|
||||
|
||||
return retriever.retrieveToken();
|
||||
}
|
||||
|
||||
private String retrieveIdentifierTokenText(char firstCharacter) {
|
||||
ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstCharacter,
|
||||
Characters.INSTANCE::isLegalIdentifierCharacter);
|
||||
|
||||
return retriever.retrieveToken();
|
||||
}
|
||||
|
||||
private class ComplexTokenTextRetriever {
|
||||
|
||||
Function<Character, Boolean> isPartOfToken;
|
||||
StringBuilder text;
|
||||
FilePosition position;
|
||||
char firstCharacter;
|
||||
char currentCharacter;
|
||||
char previousCharacter;
|
||||
|
||||
public ComplexTokenTextRetriever(char firstCharacter, Function<Character, Boolean> isPartOfToken) {
|
||||
this.isPartOfToken = isPartOfToken;
|
||||
this.text = new StringBuilder();
|
||||
this.position = positionTracker.currentPosition();
|
||||
this.firstCharacter = firstCharacter;
|
||||
this.currentCharacter = firstCharacter;
|
||||
this.previousCharacter = firstCharacter;
|
||||
}
|
||||
|
||||
public String retrieveToken() {
|
||||
text.append(firstCharacter);
|
||||
|
||||
for (int c = inputStream.read(); c != Characters.EOF; c = inputStream.read()) {
|
||||
currentCharacter = (char) c;
|
||||
|
||||
if (!isPartOfToken.apply(currentCharacter)) {
|
||||
inputStream.unreadLastCharacter();
|
||||
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
addCharacterToToken();
|
||||
|
||||
if (isTerminatingCharacter())
|
||||
return text.toString();
|
||||
|
||||
previousCharacter = currentCharacter;
|
||||
}
|
||||
|
||||
return terminateTokenWithEof();
|
||||
}
|
||||
|
||||
private void addCharacterToToken() {
|
||||
text.append(currentCharacter);
|
||||
positionTracker.incrementColumn();
|
||||
|
||||
if (currentCharacter == Characters.NEWLINE)
|
||||
positionTracker.incrementLine();
|
||||
}
|
||||
|
||||
private boolean isTerminatingCharacter() {
|
||||
return isStringToken() && isTerminatingDoubleQuote();
|
||||
}
|
||||
|
||||
private boolean isStringToken() {
|
||||
return firstCharacter == Characters.DOUBLE_QUOTE;
|
||||
}
|
||||
|
||||
private boolean isTerminatingDoubleQuote() {
|
||||
return (currentCharacter == Characters.DOUBLE_QUOTE) &&
|
||||
(previousCharacter != Characters.BACKSLASH);
|
||||
}
|
||||
|
||||
private String terminateTokenWithEof() {
|
||||
if (isStringToken())
|
||||
throw new UnterminatedStringException(position);
|
||||
|
||||
return text.toString();
|
||||
}
|
||||
}
|
||||
|
||||
public static class UnterminatedStringException extends LineColumnException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public UnterminatedStringException(FilePosition position) {
|
||||
super(position);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessagePrefix() {
|
||||
return "unterminated quoted string";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
package scanner
|
||||
|
||||
import error.LineColumnException
|
||||
import file.FilePosition
|
||||
import file.FilePositionTracker
|
||||
import token.Token
|
||||
import token.TokenFactoryImpl
|
||||
import util.Characters
|
||||
import util.Characters.BACKSLASH
|
||||
import util.Characters.DOUBLE_QUOTE
|
||||
import util.Characters.EOF
|
||||
import util.Characters.NEWLINE
|
||||
import java.io.InputStream
|
||||
import java.lang.Character.isDigit
|
||||
import java.lang.Character.isWhitespace
|
||||
|
||||
/**
|
||||
* Converts a stream of bytes into a stream of Lisp tokens.
|
||||
*/
|
||||
class LispScanner(inputStream: InputStream, fileName: String) {
|
||||
|
||||
private val inputStream = LispCommentRemovingInputStream(inputStream)
|
||||
private val positionTracker = FilePositionTracker(fileName)
|
||||
private val tokenFactory = TokenFactoryImpl()
|
||||
|
||||
val nextToken: Token
|
||||
get() {
|
||||
var c = inputStream.read()
|
||||
while (c != EOF) {
|
||||
val currentCharacter = c.toChar()
|
||||
positionTracker.incrementColumn()
|
||||
|
||||
if (!isWhitespace(currentCharacter))
|
||||
return createTokenFromCharacter(currentCharacter)
|
||||
else if (currentCharacter == NEWLINE)
|
||||
positionTracker.incrementLine()
|
||||
|
||||
c = inputStream.read()
|
||||
}
|
||||
|
||||
return tokenFactory.createEofToken(positionTracker.currentPosition())
|
||||
}
|
||||
|
||||
private fun createTokenFromCharacter(c: Char): Token {
|
||||
val currentPosition = positionTracker.currentPosition()
|
||||
val tokenText = retrieveTokenText(c)
|
||||
|
||||
return tokenFactory.createToken(tokenText, currentPosition)
|
||||
}
|
||||
|
||||
private fun retrieveTokenText(firstCharacter: Char): String {
|
||||
var tokenText = "" + firstCharacter
|
||||
|
||||
if (firstCharacter == DOUBLE_QUOTE)
|
||||
tokenText = retrieveStringTokenText(firstCharacter)
|
||||
else if (Characters.isNumberPrefix(firstCharacter))
|
||||
tokenText = retrieveNumberOrIdentifierTokenText(firstCharacter)
|
||||
else if (isDigit(firstCharacter))
|
||||
tokenText = retrieveNumberTokenText(firstCharacter)
|
||||
else if (Characters.isLegalIdentifierCharacter(firstCharacter))
|
||||
tokenText = retrieveIdentifierTokenText(firstCharacter)
|
||||
|
||||
return tokenText
|
||||
}
|
||||
|
||||
private fun retrieveStringTokenText(firstDoubleQuote: Char) =
|
||||
ComplexTokenTextRetriever(firstDoubleQuote, { Characters.isLegalStringCharacter(it) }).retrieveToken()
|
||||
|
||||
private fun retrieveNumberOrIdentifierTokenText(firstCharacter: Char): String {
|
||||
val nextCharacter = inputStream.read().toChar()
|
||||
inputStream.unreadLastCharacter()
|
||||
|
||||
return if (isDigit(nextCharacter))
|
||||
retrieveNumberTokenText(firstCharacter)
|
||||
else
|
||||
retrieveIdentifierTokenText(firstCharacter)
|
||||
}
|
||||
|
||||
private fun retrieveNumberTokenText(firstCharacter: Char) =
|
||||
ComplexTokenTextRetriever(firstCharacter, { Character.isDigit(it) }).retrieveToken()
|
||||
|
||||
private fun retrieveIdentifierTokenText(firstCharacter: Char) =
|
||||
ComplexTokenTextRetriever(firstCharacter, { Characters.isLegalIdentifierCharacter(it) }).retrieveToken()
|
||||
|
||||
private inner class ComplexTokenTextRetriever(internal var firstCharacter: Char,
|
||||
internal var isPartOfToken: (Char) -> Boolean) {
|
||||
internal var text = StringBuilder()
|
||||
internal var position = positionTracker.currentPosition()
|
||||
internal var currentCharacter = firstCharacter
|
||||
internal var previousCharacter = firstCharacter
|
||||
|
||||
private fun isTerminatingCharacter() = isStringToken() && isTerminatingDoubleQuote()
|
||||
private fun isStringToken() = firstCharacter == DOUBLE_QUOTE
|
||||
private fun isTerminatingDoubleQuote() = currentCharacter == DOUBLE_QUOTE && previousCharacter != BACKSLASH
|
||||
|
||||
fun retrieveToken(): String {
|
||||
text.append(firstCharacter)
|
||||
|
||||
var c = inputStream.read()
|
||||
while (c != EOF) {
|
||||
currentCharacter = c.toChar()
|
||||
|
||||
if (!isPartOfToken(currentCharacter)) {
|
||||
inputStream.unreadLastCharacter()
|
||||
|
||||
return text.toString()
|
||||
}
|
||||
|
||||
addCharacterToToken()
|
||||
|
||||
if (isTerminatingCharacter())
|
||||
return text.toString()
|
||||
|
||||
previousCharacter = currentCharacter
|
||||
c = inputStream.read()
|
||||
}
|
||||
|
||||
return terminateTokenWithEof()
|
||||
}
|
||||
|
||||
private fun addCharacterToToken() {
|
||||
text.append(currentCharacter)
|
||||
positionTracker.incrementColumn()
|
||||
|
||||
if (currentCharacter == NEWLINE)
|
||||
positionTracker.incrementLine()
|
||||
}
|
||||
|
||||
private fun terminateTokenWithEof(): String {
|
||||
if (isStringToken())
|
||||
throw UnterminatedStringException(position)
|
||||
|
||||
return text.toString()
|
||||
}
|
||||
}
|
||||
|
||||
class UnterminatedStringException(position: FilePosition) : LineColumnException(position) {
|
||||
override val messagePrefix: String
|
||||
get() = "unterminated quoted string"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
package stream;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
public class SafeInputStream {
|
||||
|
||||
private InputStreamReader underlyingStream;
|
||||
|
||||
public SafeInputStream(InputStream underlyingStream) {
|
||||
this.underlyingStream = new InputStreamReader(underlyingStream, UTF_8);
|
||||
}
|
||||
|
||||
public int read() {
|
||||
try {
|
||||
return underlyingStream.read();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void close() {
|
||||
try {
|
||||
underlyingStream.close();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package stream
|
||||
|
||||
import java.io.IOException
|
||||
import java.io.InputStream
|
||||
import java.io.InputStreamReader
|
||||
|
||||
import java.nio.charset.StandardCharsets.UTF_8
|
||||
|
||||
class SafeInputStream(underlyingStream: InputStream) {
|
||||
|
||||
private val underlyingStream: InputStreamReader = InputStreamReader(underlyingStream, UTF_8)
|
||||
|
||||
fun read(): Int {
|
||||
try {
|
||||
return underlyingStream.read()
|
||||
} catch (e: IOException) {
|
||||
throw UncheckedIOException(e)
|
||||
}
|
||||
}
|
||||
|
||||
fun close() {
|
||||
try {
|
||||
underlyingStream.close()
|
||||
} catch (e: IOException) {
|
||||
throw UncheckedIOException(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
package stream;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
public class SafeOutputStream {
|
||||
|
||||
private OutputStream underlyingStream;
|
||||
|
||||
public SafeOutputStream(OutputStream underlyingStream) {
|
||||
this.underlyingStream = underlyingStream;
|
||||
}
|
||||
|
||||
public void write(byte[] b) {
|
||||
try {
|
||||
underlyingStream.write(b);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void flush() {
|
||||
|
||||
try {
|
||||
underlyingStream.flush();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void close() {
|
||||
try {
|
||||
underlyingStream.close();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package stream
|
||||
|
||||
import java.io.IOException
|
||||
import java.io.OutputStream
|
||||
|
||||
class SafeOutputStream(private val underlyingStream: OutputStream) {
|
||||
|
||||
fun write(b: ByteArray) {
|
||||
try {
|
||||
underlyingStream.write(b)
|
||||
} catch (e: IOException) {
|
||||
throw UncheckedIOException(e)
|
||||
}
|
||||
}
|
||||
|
||||
fun flush() {
|
||||
try {
|
||||
underlyingStream.flush()
|
||||
} catch (e: IOException) {
|
||||
throw UncheckedIOException(e)
|
||||
}
|
||||
}
|
||||
|
||||
fun close() {
|
||||
try {
|
||||
underlyingStream.close()
|
||||
} catch (e: IOException) {
|
||||
throw UncheckedIOException(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
package stream;
|
||||
|
||||
import error.CriticalLispException;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class UncheckedIOException extends CriticalLispException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
private IOException exception;
|
||||
|
||||
public UncheckedIOException(IOException exception) {
|
||||
this.exception = exception;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
return exception.getMessage();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package stream
|
||||
|
||||
import error.CriticalLispException
|
||||
|
||||
import java.io.IOException
|
||||
|
||||
class UncheckedIOException(private val exception: IOException) : CriticalLispException() {
|
||||
|
||||
override val message: String
|
||||
get() = exception.message ?: ""
|
||||
}
|
||||
Loading…
Reference in New Issue