Skip to content
This repository has been archived by the owner on Aug 9, 2024. It is now read-only.

Commit

Permalink
Prototype - .next() method on TokenStream
Browse files Browse the repository at this point in the history
  • Loading branch information
@Laifsyn
Laifsyn committed Apr 29, 2024
1 parent 2c1deff commit fd1db40
Show file tree
Hide file tree
Showing 2 changed files with 201 additions and 53 deletions.
134 changes: 81 additions & 53 deletions src/main/java/com/utp/clsEstructuraDatos/pry3/Tokens/Token.java
View file
Original file line number Diff line number Diff line change
@@ -1,118 +1,146 @@
package com.utp.clsEstructuraDatos.pry3.Tokens;

import java.beans.Expression;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Optional;

public sealed interface Token {
public final static int MAX_CHARS = TokensGroup.MAX_CHARS;

public static Token fromString(String substring) {
substring = substring.trim();
if (substring.isEmpty()) {
return new Token.EMPTY();
}
if (substring.length() > TokensGroup.max_chars) {
return new Token.INVALID(new Exception("Token too long - Max length: " + TokensGroup.max_chars));
}

Token result = TokensGroup.tokens.get(substring);
return result;

}

public String toString();

record AND() implements Token {
public record AND(String inner_string) implements Token {
@Override
public String toString() {
return "&";
}

}

record OR() implements Token {
public record OR(String inner_string) implements Token {
@Override
public String toString() {
return "|";
}
}

record NOT() implements Token {
public record NOT(String inner_string) implements Token {
@Override
public String toString() {
return "~";
}
}

record IMPLICATES() implements Token {
public record IMPLICATES(String inner_string) implements Token {
@Override
public String toString() {
return "->";
}
}

record XAND() implements Token {
public record XAND(String inner_string) implements Token {
@Override
public String toString() {
return "<->";
return inner_string;
}
}

/**
* A token representing an expression. the inner value of the expression can be
* evaluated into a bool.
*/
record EXPRESSION(Optional<String[]> parentheses, Expression expression) implements Token {

public EXPRESSION(Optional<String[]> parentheses, String expression) {
if (parentheses.isPresent()) {
String[] parens = parentheses.get();
if (parens.length != 2) {
throw new IllegalArgumentException("Invalid parentheses: " + Arrays.toString(parens));
}
for (var valid_delimiters : TokensGroup.DELIMITER) {
if (valid_delimiters[0].equals(parens[0]) && valid_delimiters[1].equals(parens[1])) {
this.parentheses = parentheses;
this.expression = expression;
return;
}
}
throw new IllegalArgumentException("Invalid parentheses: " + Arrays.toString(parens)
+ "%nExpected any of the following:" + Arrays.deepToString(TokensGroup.DELIMITER));
} else {
this.parentheses = Optional.empty();
this.expression = expression;
}


// /**
// * A token representing an expression. the inner value of the expression can
// be
// * evaluated into a bool.
// */
// record EXPRESSION(Optional<String[]> parentheses, Expression expression)
// implements Token {

// public EXPRESSION(Optional<String[]> parentheses, String expression) {
// if (parentheses.isPresent()) {
// String[] parens = parentheses.get();
// if (parens.length != 2) {
// throw new IllegalArgumentException("Invalid parentheses: " +
// Arrays.toString(parens));
// }
// for (var valid_delimiters : TokensGroup.DELIMITER) {
// if (valid_delimiters[0].equals(parens[0]) &&
// valid_delimiters[1].equals(parens[1])) {
// this.parentheses = parentheses;
// this.expression = expression;
// return;
// }
// }
// throw new IllegalArgumentException("Invalid parentheses: " +
// Arrays.toString(parens)
// + "%nExpected any of the following:" +
// Arrays.deepToString(TokensGroup.DELIMITER));
// } else {
// this.parentheses = Optional.empty();
// this.expression = expression;
// }
// }

// @Override
// public String toString() {
// String[] parentheses = this.parentheses.orElse(new String[] { "", "" });
// return "EXPRESSION(\"" + parentheses[0] + expression + parentheses[1] +
// "\")";
// }
// }

public record IDENTIFIER(String ident) implements Token {
@Override
public String toString() {
return "IDENTIFIER(" + ident + ")";
}
}

public record OPEN_PAREN(char paren) implements Token {
@Override
public String toString() {
String[] parentheses = this.parentheses.orElse(new String[] { "", "" });
return "EXPRESSION(\"" + parentheses[0] + expression + parentheses[1] + "\")";
return paren + "";
}
}

record IDENTIFIER(String value) implements Token {
public record CLOSE_PAREN(char paren) implements Token {
@Override
public String toString() {
return "IDENTIFIER(" + value + ")";
return paren + "";
}
}

record UNKNOWN(String value) implements Token {
public record UNEXPECTED(String substring) implements Token {
@Override
public String toString() {
return "`" + substring + "` is an invalid token";
}
}

public record EOL(String inner_string) implements Token{
@Override
public String toString() {
return "UNKNOWN(" + value + ")";
return inner_string;
}
}

record INVALID(Exception e) implements Token {
public record EXPECTS(String inner_string, Optional<String> expects_next) implements Token {
@Override
public String toString() {
return e.getMessage();
return "`" + inner_string + "` expects: `" + expects_next.orElseGet(() -> "{unknown}") + "`";
}
}

record EMPTY() implements Token {
public record EMPTY() implements Token {
@Override
public String toString() {
return "{EMPTY}";
Expand All @@ -127,16 +155,16 @@ public String toString() {
// }

class TokensGroup {
static final int max_chars = 3;
static final int MAX_CHARS = 3;
static final HashMap<String, Token> tokens = new HashMap<String, Token>() {
{
put(AND[0], new Token.AND());
put(AND[1], new Token.AND());
put(OR[0], new Token.OR());
put(NOT[0], new Token.NOT());
put(NOT[1], new Token.NOT());
put(IMPLICATES[0], new Token.IMPLICATES());
put(XAND[0], new Token.XAND());
put(AND[0], new Token.AND(AND[0]));
put(AND[1], new Token.AND(AND[1]));
put(OR[0], new Token.OR(OR[0]));
put(NOT[0], new Token.NOT(NOT[0]));
put(NOT[1], new Token.NOT(NOT[1]));
put(IMPLICATES[0], new Token.IMPLICATES(IMPLICATES[0]));
put(XAND[0], new Token.XAND(XAND[0]));

for (var delimiters : DELIMITER) {
assert delimiters.length == 2 : delimiters.toString() + " isn't a valid delimiter - Expects 2 elements";
Expand Down
120 changes: 120 additions & 0 deletions src/main/java/com/utp/clsEstructuraDatos/pry3/Tokens/TokenStream.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package com.utp.clsEstructuraDatos.pry3.Tokens;

import java.util.Optional;

import com.utp.utils.Result;

public class TokenStream {
final String inner_string;
int pointer = 0;

private TokenStream(String string) {
this.inner_string = string;
}

public static Result<TokenStream, IllegalArgumentException> from_string(String string) {
string = string.strip();
if (string.length() > Integer.MAX_VALUE) {
return Result.error(new IllegalArgumentException(
"string too long. Keep it under " + Integer.MAX_VALUE + " characters"));
}
return Result.ok(new TokenStream(string));
}

int next_offset = 1;

Optional<Token> next() {
if (pointer >= inner_string.length()) {
return Optional.empty();
}
// Avanza el puntero por cada espacio en blanco
while (pointer < inner_string.length() && Character.isWhitespace(inner_string.charAt(pointer))) {
pointer++;
}
Token result = Token.fromString(inner_string.substring(pointer, pointer + next_offset));
// Use Arrow syntax
switch (result) {
case Token.EMPTY() -> {
next_offset = 1;
pointer += next_offset;
return next();
}
case Token.AND(String substring) -> {
next_offset = 1;
pointer += substring.length();
return Optional.of(result);
}
case Token.OR(String substring) -> {
next_offset = 1;
pointer += substring.length();
return Optional.of(result);
}
case Token.NOT(String substring) -> {
next_offset = 1;
pointer += substring.length();
return Optional.of(result);
}
case Token.IMPLICATES(String substring) -> {
next_offset = 1;
pointer += substring.length();
return Optional.of(result);
}
case Token.XAND(String substring) -> {
next_offset = 1;
pointer += substring.length();
return Optional.of(result);
}
case Token.CLOSE_PAREN(char _open_paren) -> {
next_offset = 1;
pointer += next_offset;
return Optional.of(result);
}
case Token.OPEN_PAREN(char _close_paren) -> {
next_offset = 1;
pointer += next_offset;
return Optional.of(result);
}
case Token.IDENTIFIER(String substring) -> {
next_offset = 1;
pointer += substring.length();
return Optional.of(result);
}
case Token.EXPECTS(String stored_substring, Optional<String> might_expect) -> {
if (might_expect.isEmpty()) {
// Hay match, pero se desconoce qué sigue. Así que aumentamos el offset, y
// continuamos.
next_offset++;
// pero si el offset se sale de rango, significa que no hay match.
if (pointer + next_offset >= inner_string.length()) {
return Optional.of(new Token.EOL(inner_string.substring(pointer)));
}
return next();
}
String expects_next = might_expect.get();
final int offset = stored_substring.length() + expects_next.length();
final String next_substring = inner_string.substring(pointer, pointer + offset);
final String expects_string = stored_substring + expects_next;
if (!next_substring.equals(expects_string)) {
return Optional.of(new Token.UNEXPECTED(next_substring));
}
next_offset = offset;
return next();
}
case Token.UNEXPECTED(String substring) -> {
next_offset = 1;
// don't advance the pointer
return Optional.of(result);
}
case Token.EOL(String substring) -> {
next_offset = 1;
pointer += substring.length();
return Optional.of(result);
}
// default -> throw new Exception("sadasd");
}

throw new UnsupportedOperationException("Not supported yet.");

}

}

0 comments on commit fd1db40

Please sign in to comment.