From dc13f917b0e11f4551e672c8a173ac4c77677309 Mon Sep 17 00:00:00 2001 From: weckyy702 Date: Mon, 5 Apr 2021 20:19:53 +0200 Subject: [PATCH] added curly counter to tokenizer for basic error checking --- interpreter/Tokenizer.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/interpreter/Tokenizer.py b/interpreter/Tokenizer.py index 4a0d148..05e1fbf 100644 --- a/interpreter/Tokenizer.py +++ b/interpreter/Tokenizer.py @@ -1,5 +1,6 @@ """Tokenizer.py: Definition for Tokenizer class""" +from errors.custom import JavaSyntaxError import logging import re from typing import List, Optional @@ -20,10 +21,13 @@ class Tokenizer: self.source_text = re.sub("(private)|(public)|(protected)", "", self.source_text) - self.type_name_pattern = re.compile('(char)|(int)|(void)|(double)|(Pixel)') #TODO: make this modular + self.type_name_pattern = re.compile('(char)|(int)|(void)|(double)|(boolean)|(Pixel)|(String)') #TODO: make this modular self._filename = file_name + self._left_curly_number=0 + self._right_curly_number=0 + def get_tokens(self) -> List[Token]: tokens = [] @@ -36,9 +40,18 @@ class Tokenizer: if self._handle_comments(char): continue - token = self._get_token(char) - logging.debug(f"found token \"{token}\" on line {self.line_number}") - tokens.append(make_token(token, SourceLocation(self._filename, self.line_number, self.column_number), self.type_name_pattern)) + tag = self._get_token(char) + logging.debug(f"found tag \"{tag}\" on line {self.line_number}") + + if tag == "{": + self._left_curly_number+=1 + elif tag == "}": + self._right_curly_number+=1 + + tokens.append(make_token(tag, SourceLocation(self._filename, self.line_number, self.column_number), self.type_name_pattern)) + + if self._left_curly_number != self._right_curly_number: + raise JavaSyntaxError(f"Ill-formed Java program! Expected equal number of '{'{'}' and '{'}'}' tokens, got {self._left_curly_number} and {self._right_curly_number}") return tokens