diff --git a/src/main/kotlin/scanner/LispCommentRemovingInputStream.java b/src/main/kotlin/scanner/LispCommentRemovingInputStream.java deleted file mode 100644 index 7aa2fe8..0000000 --- a/src/main/kotlin/scanner/LispCommentRemovingInputStream.java +++ /dev/null @@ -1,78 +0,0 @@ -package scanner; - -import stream.SafeInputStream; -import util.Characters; - -import java.io.InputStream; - -/** - * Removes Lisp comments from an input stream. - */ -public class LispCommentRemovingInputStream implements LispInputStream { - - private SafeInputStream underlyingInputStream; - private boolean isInQuotedString; - private boolean rereadLastCharacter; - private int previousCharacter; - private int currentCharacter; - - public LispCommentRemovingInputStream(InputStream underlyingInputStream) { - this.underlyingInputStream = new SafeInputStream(underlyingInputStream); - this.isInQuotedString = false; - this.rereadLastCharacter = false; - this.previousCharacter = 0; - this.currentCharacter = 0; - } - - @Override - public int read() { - if (!rereadLastCharacter) - return readFromUnderlyingInputStream(); - - rereadLastCharacter = false; - - return currentCharacter; - } - - private int readFromUnderlyingInputStream() { - readNextCharacter(); - - if (haveEnteredComment()) - consumeAllBytesInComment(); - - return currentCharacter; - } - - private void readNextCharacter() { - previousCharacter = currentCharacter; - currentCharacter = underlyingInputStream.read(); - - if (haveEncounteredStringBoundary()) - isInQuotedString = !isInQuotedString; - } - - private boolean haveEncounteredStringBoundary() { - return (previousCharacter != Characters.BACKSLASH) && (currentCharacter == Characters.DOUBLE_QUOTE); - } - - private boolean haveEnteredComment() { - return (currentCharacter == Characters.SEMICOLON) && (!isInQuotedString); - } - - private void consumeAllBytesInComment() { - while (stillInComment()) - currentCharacter = underlyingInputStream.read(); - } - - private boolean stillInComment() { - return (currentCharacter != Characters.NEWLINE) && (currentCharacter != Characters.EOF); - } - - @Override - public void unreadLastCharacter() { - if (rereadLastCharacter) - throw new MaximumUnreadsExceededException(); - - this.rereadLastCharacter = true; - } -} diff --git a/src/main/kotlin/scanner/LispCommentRemovingInputStream.kt b/src/main/kotlin/scanner/LispCommentRemovingInputStream.kt new file mode 100644 index 0000000..7730058 --- /dev/null +++ b/src/main/kotlin/scanner/LispCommentRemovingInputStream.kt @@ -0,0 +1,66 @@ +package scanner + +import stream.SafeInputStream +import util.Characters.BACKSLASH +import util.Characters.DOUBLE_QUOTE +import util.Characters.EOF +import util.Characters.NEWLINE +import util.Characters.SEMICOLON +import java.io.InputStream + +/** + * Removes Lisp comments from an input stream. + */ +class LispCommentRemovingInputStream(underlyingInputStream: InputStream) : LispInputStream { + + private val underlyingInputStream = SafeInputStream(underlyingInputStream) + private var isInQuotedString = false + private var rereadLastCharacter = false + private var previousCharacter = 0 + private var currentCharacter = 0 + + override fun read(): Int { + if (!rereadLastCharacter) + return readFromUnderlyingInputStream() + + rereadLastCharacter = false + + return currentCharacter + } + + override fun unreadLastCharacter() { + if (rereadLastCharacter) + throw LispInputStream.MaximumUnreadsExceededException() + + this.rereadLastCharacter = true + } + + private fun readFromUnderlyingInputStream(): Int { + readNextCharacter() + + if (haveEnteredComment()) + consumeAllBytesInComment() + + return currentCharacter + } + + private fun readNextCharacter() { + previousCharacter = currentCharacter + currentCharacter = underlyingInputStream.read() + + if (haveEncounteredStringBoundary()) + isInQuotedString = !isInQuotedString + } + + private fun haveEncounteredStringBoundary() = + previousCharacter != BACKSLASH.toInt() && currentCharacter == DOUBLE_QUOTE.toInt() + + private fun haveEnteredComment() = currentCharacter == SEMICOLON.toInt() && !isInQuotedString + + private fun consumeAllBytesInComment() { + while (stillInComment()) + currentCharacter = underlyingInputStream.read() + } + + private fun stillInComment() = currentCharacter != NEWLINE.toInt() && currentCharacter != EOF +} diff --git a/src/main/kotlin/scanner/LispInputStream.java b/src/main/kotlin/scanner/LispInputStream.java deleted file mode 100644 index aa856a0..0000000 --- a/src/main/kotlin/scanner/LispInputStream.java +++ /dev/null @@ -1,15 +0,0 @@ -package scanner; - -import error.CriticalLispException; - -public interface LispInputStream { - - int read(); - - void unreadLastCharacter(); - - public static class MaximumUnreadsExceededException extends CriticalLispException { - - private static final long serialVersionUID = 1L; - } -} diff --git a/src/main/kotlin/scanner/LispInputStream.kt b/src/main/kotlin/scanner/LispInputStream.kt new file mode 100644 index 0000000..24ac439 --- /dev/null +++ b/src/main/kotlin/scanner/LispInputStream.kt @@ -0,0 +1,11 @@ +package scanner + +import error.CriticalLispException + +interface LispInputStream { + + fun read(): Int + fun unreadLastCharacter() + + class MaximumUnreadsExceededException : CriticalLispException() +} diff --git a/src/main/kotlin/scanner/LispScanner.java b/src/main/kotlin/scanner/LispScanner.java deleted file mode 100644 index e4030f7..0000000 --- a/src/main/kotlin/scanner/LispScanner.java +++ /dev/null @@ -1,181 +0,0 @@ -package scanner; - -import error.LineColumnException; -import file.FilePosition; -import file.FilePositionTracker; -import token.Token; -import token.TokenFactory; -import token.TokenFactoryImpl; -import util.Characters; - -import java.io.InputStream; -import java.util.function.Function; - -import static java.lang.Character.isDigit; -import static java.lang.Character.isWhitespace; - -/** - * Converts a stream of bytes into a stream of Lisp tokens. - */ -public class LispScanner { - - private LispInputStream inputStream; - private FilePositionTracker positionTracker; - private TokenFactory tokenFactory; - - public LispScanner(InputStream inputStream, String fileName) { - this.inputStream = new LispCommentRemovingInputStream(inputStream); - this.positionTracker = new FilePositionTracker(fileName); - this.tokenFactory = new TokenFactoryImpl(); - } - - public Token getNextToken() { - for (int c = inputStream.read(); c != Characters.EOF; c = inputStream.read()) { - char currentCharacter = (char) c; - positionTracker.incrementColumn(); - - if (!isWhitespace(currentCharacter)) - return createTokenFromCharacter(currentCharacter); - else if (currentCharacter == Characters.NEWLINE) - positionTracker.incrementLine(); - } - - return tokenFactory.createEofToken(positionTracker.currentPosition()); - } - - private Token createTokenFromCharacter(char c) { - FilePosition currentPosition = positionTracker.currentPosition(); - String tokenText = retrieveTokenText(c); - - return tokenFactory.createToken(tokenText, currentPosition); - } - - private String retrieveTokenText(char firstCharacter) { - String tokenText = "" + firstCharacter; - - if (firstCharacter == Characters.DOUBLE_QUOTE) - tokenText = retrieveStringTokenText(firstCharacter); - else if (Characters.INSTANCE.isNumberPrefix(firstCharacter)) - tokenText = retrieveNumberOrIdentifierTokenText(firstCharacter); - else if (isDigit(firstCharacter)) - tokenText = retrieveNumberTokenText(firstCharacter); - else if (Characters.INSTANCE.isLegalIdentifierCharacter(firstCharacter)) - tokenText = retrieveIdentifierTokenText(firstCharacter); - - return tokenText; - } - - private String retrieveStringTokenText(char firstDoubleQuote) { - ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstDoubleQuote, - Characters.INSTANCE::isLegalStringCharacter); - - return retriever.retrieveToken(); - } - - private String retrieveNumberOrIdentifierTokenText(char firstCharacter) { - char nextCharacter = (char) inputStream.read(); - inputStream.unreadLastCharacter(); - - if (isDigit(nextCharacter)) - return retrieveNumberTokenText(firstCharacter); - - return retrieveIdentifierTokenText(firstCharacter); - } - - private String retrieveNumberTokenText(char firstCharacter) { - ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstCharacter, Character::isDigit); - - return retriever.retrieveToken(); - } - - private String retrieveIdentifierTokenText(char firstCharacter) { - ComplexTokenTextRetriever retriever = new ComplexTokenTextRetriever(firstCharacter, - Characters.INSTANCE::isLegalIdentifierCharacter); - - return retriever.retrieveToken(); - } - - private class ComplexTokenTextRetriever { - - Function isPartOfToken; - StringBuilder text; - FilePosition position; - char firstCharacter; - char currentCharacter; - char previousCharacter; - - public ComplexTokenTextRetriever(char firstCharacter, Function isPartOfToken) { - this.isPartOfToken = isPartOfToken; - this.text = new StringBuilder(); - this.position = positionTracker.currentPosition(); - this.firstCharacter = firstCharacter; - this.currentCharacter = firstCharacter; - this.previousCharacter = firstCharacter; - } - - public String retrieveToken() { - text.append(firstCharacter); - - for (int c = inputStream.read(); c != Characters.EOF; c = inputStream.read()) { - currentCharacter = (char) c; - - if (!isPartOfToken.apply(currentCharacter)) { - inputStream.unreadLastCharacter(); - - return text.toString(); - } - - addCharacterToToken(); - - if (isTerminatingCharacter()) - return text.toString(); - - previousCharacter = currentCharacter; - } - - return terminateTokenWithEof(); - } - - private void addCharacterToToken() { - text.append(currentCharacter); - positionTracker.incrementColumn(); - - if (currentCharacter == Characters.NEWLINE) - positionTracker.incrementLine(); - } - - private boolean isTerminatingCharacter() { - return isStringToken() && isTerminatingDoubleQuote(); - } - - private boolean isStringToken() { - return firstCharacter == Characters.DOUBLE_QUOTE; - } - - private boolean isTerminatingDoubleQuote() { - return (currentCharacter == Characters.DOUBLE_QUOTE) && - (previousCharacter != Characters.BACKSLASH); - } - - private String terminateTokenWithEof() { - if (isStringToken()) - throw new UnterminatedStringException(position); - - return text.toString(); - } - } - - public static class UnterminatedStringException extends LineColumnException { - - private static final long serialVersionUID = 1L; - - public UnterminatedStringException(FilePosition position) { - super(position); - } - - @Override - public String getMessagePrefix() { - return "unterminated quoted string"; - } - } -} diff --git a/src/main/kotlin/scanner/LispScanner.kt b/src/main/kotlin/scanner/LispScanner.kt new file mode 100644 index 0000000..e3cce53 --- /dev/null +++ b/src/main/kotlin/scanner/LispScanner.kt @@ -0,0 +1,141 @@ +package scanner + +import error.LineColumnException +import file.FilePosition +import file.FilePositionTracker +import token.Token +import token.TokenFactoryImpl +import util.Characters +import util.Characters.BACKSLASH +import util.Characters.DOUBLE_QUOTE +import util.Characters.EOF +import util.Characters.NEWLINE +import java.io.InputStream +import java.lang.Character.isDigit +import java.lang.Character.isWhitespace + +/** + * Converts a stream of bytes into a stream of Lisp tokens. + */ +class LispScanner(inputStream: InputStream, fileName: String) { + + private val inputStream = LispCommentRemovingInputStream(inputStream) + private val positionTracker = FilePositionTracker(fileName) + private val tokenFactory = TokenFactoryImpl() + + val nextToken: Token + get() { + var c = inputStream.read() + while (c != EOF) { + val currentCharacter = c.toChar() + positionTracker.incrementColumn() + + if (!isWhitespace(currentCharacter)) + return createTokenFromCharacter(currentCharacter) + else if (currentCharacter == NEWLINE) + positionTracker.incrementLine() + + c = inputStream.read() + } + + return tokenFactory.createEofToken(positionTracker.currentPosition()) + } + + private fun createTokenFromCharacter(c: Char): Token { + val currentPosition = positionTracker.currentPosition() + val tokenText = retrieveTokenText(c) + + return tokenFactory.createToken(tokenText, currentPosition) + } + + private fun retrieveTokenText(firstCharacter: Char): String { + var tokenText = "" + firstCharacter + + if (firstCharacter == DOUBLE_QUOTE) + tokenText = retrieveStringTokenText(firstCharacter) + else if (Characters.isNumberPrefix(firstCharacter)) + tokenText = retrieveNumberOrIdentifierTokenText(firstCharacter) + else if (isDigit(firstCharacter)) + tokenText = retrieveNumberTokenText(firstCharacter) + else if (Characters.isLegalIdentifierCharacter(firstCharacter)) + tokenText = retrieveIdentifierTokenText(firstCharacter) + + return tokenText + } + + private fun retrieveStringTokenText(firstDoubleQuote: Char) = + ComplexTokenTextRetriever(firstDoubleQuote, { Characters.isLegalStringCharacter(it) }).retrieveToken() + + private fun retrieveNumberOrIdentifierTokenText(firstCharacter: Char): String { + val nextCharacter = inputStream.read().toChar() + inputStream.unreadLastCharacter() + + return if (isDigit(nextCharacter)) + retrieveNumberTokenText(firstCharacter) + else + retrieveIdentifierTokenText(firstCharacter) + } + + private fun retrieveNumberTokenText(firstCharacter: Char) = + ComplexTokenTextRetriever(firstCharacter, { Character.isDigit(it) }).retrieveToken() + + private fun retrieveIdentifierTokenText(firstCharacter: Char) = + ComplexTokenTextRetriever(firstCharacter, { Characters.isLegalIdentifierCharacter(it) }).retrieveToken() + + private inner class ComplexTokenTextRetriever(internal var firstCharacter: Char, + internal var isPartOfToken: (Char) -> Boolean) { + internal var text = StringBuilder() + internal var position = positionTracker.currentPosition() + internal var currentCharacter = firstCharacter + internal var previousCharacter = firstCharacter + + private fun isTerminatingCharacter() = isStringToken() && isTerminatingDoubleQuote() + private fun isStringToken() = firstCharacter == DOUBLE_QUOTE + private fun isTerminatingDoubleQuote() = currentCharacter == DOUBLE_QUOTE && previousCharacter != BACKSLASH + + fun retrieveToken(): String { + text.append(firstCharacter) + + var c = inputStream.read() + while (c != EOF) { + currentCharacter = c.toChar() + + if (!isPartOfToken(currentCharacter)) { + inputStream.unreadLastCharacter() + + return text.toString() + } + + addCharacterToToken() + + if (isTerminatingCharacter()) + return text.toString() + + previousCharacter = currentCharacter + c = inputStream.read() + } + + return terminateTokenWithEof() + } + + private fun addCharacterToToken() { + text.append(currentCharacter) + positionTracker.incrementColumn() + + if (currentCharacter == NEWLINE) + positionTracker.incrementLine() + } + + private fun terminateTokenWithEof(): String { + if (isStringToken()) + throw UnterminatedStringException(position) + + return text.toString() + } + } + + class UnterminatedStringException(position: FilePosition) : LineColumnException(position) { + override val messagePrefix: String + get() = "unterminated quoted string" + } +} diff --git a/src/main/kotlin/stream/SafeInputStream.java b/src/main/kotlin/stream/SafeInputStream.java deleted file mode 100644 index 5632f8f..0000000 --- a/src/main/kotlin/stream/SafeInputStream.java +++ /dev/null @@ -1,32 +0,0 @@ -package stream; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; - -import static java.nio.charset.StandardCharsets.UTF_8; - -public class SafeInputStream { - - private InputStreamReader underlyingStream; - - public SafeInputStream(InputStream underlyingStream) { - this.underlyingStream = new InputStreamReader(underlyingStream, UTF_8); - } - - public int read() { - try { - return underlyingStream.read(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public void close() { - try { - underlyingStream.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/stream/SafeInputStream.kt b/src/main/kotlin/stream/SafeInputStream.kt new file mode 100644 index 0000000..5e1f49d --- /dev/null +++ b/src/main/kotlin/stream/SafeInputStream.kt @@ -0,0 +1,28 @@ +package stream + +import java.io.IOException +import java.io.InputStream +import java.io.InputStreamReader + +import java.nio.charset.StandardCharsets.UTF_8 + +class SafeInputStream(underlyingStream: InputStream) { + + private val underlyingStream: InputStreamReader = InputStreamReader(underlyingStream, UTF_8) + + fun read(): Int { + try { + return underlyingStream.read() + } catch (e: IOException) { + throw UncheckedIOException(e) + } + } + + fun close() { + try { + underlyingStream.close() + } catch (e: IOException) { + throw UncheckedIOException(e) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/stream/SafeOutputStream.java b/src/main/kotlin/stream/SafeOutputStream.java deleted file mode 100644 index effde5f..0000000 --- a/src/main/kotlin/stream/SafeOutputStream.java +++ /dev/null @@ -1,38 +0,0 @@ -package stream; - -import java.io.IOException; -import java.io.OutputStream; - -public class SafeOutputStream { - - private OutputStream underlyingStream; - - public SafeOutputStream(OutputStream underlyingStream) { - this.underlyingStream = underlyingStream; - } - - public void write(byte[] b) { - try { - underlyingStream.write(b); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public void flush() { - - try { - underlyingStream.flush(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public void close() { - try { - underlyingStream.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/stream/SafeOutputStream.kt b/src/main/kotlin/stream/SafeOutputStream.kt new file mode 100644 index 0000000..93692fe --- /dev/null +++ b/src/main/kotlin/stream/SafeOutputStream.kt @@ -0,0 +1,31 @@ +package stream + +import java.io.IOException +import java.io.OutputStream + +class SafeOutputStream(private val underlyingStream: OutputStream) { + + fun write(b: ByteArray) { + try { + underlyingStream.write(b) + } catch (e: IOException) { + throw UncheckedIOException(e) + } + } + + fun flush() { + try { + underlyingStream.flush() + } catch (e: IOException) { + throw UncheckedIOException(e) + } + } + + fun close() { + try { + underlyingStream.close() + } catch (e: IOException) { + throw UncheckedIOException(e) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/stream/UncheckedIOException.java b/src/main/kotlin/stream/UncheckedIOException.java deleted file mode 100644 index dee1aa6..0000000 --- a/src/main/kotlin/stream/UncheckedIOException.java +++ /dev/null @@ -1,20 +0,0 @@ -package stream; - -import error.CriticalLispException; - -import java.io.IOException; - -public class UncheckedIOException extends CriticalLispException { - - private static final long serialVersionUID = 1L; - private IOException exception; - - public UncheckedIOException(IOException exception) { - this.exception = exception; - } - - @Override - public String getMessage() { - return exception.getMessage(); - } -} \ No newline at end of file diff --git a/src/main/kotlin/stream/UncheckedIOException.kt b/src/main/kotlin/stream/UncheckedIOException.kt new file mode 100644 index 0000000..edf8da0 --- /dev/null +++ b/src/main/kotlin/stream/UncheckedIOException.kt @@ -0,0 +1,11 @@ +package stream + +import error.CriticalLispException + +import java.io.IOException + +class UncheckedIOException(private val exception: IOException) : CriticalLispException() { + + override val message: String + get() = exception.message ?: "" +} \ No newline at end of file