plexx-dev/Nassi-Shneiderman-Diagramm-Generator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added curly counter to tokenizer for basic error checking

Browse Source
This commit is contained in:
weckyy702
2021-04-05 20:19:53 +02:00
parent bb67b3dc0a
commit dc13f917b0
1 changed files with 17 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
interpreter/Tokenizer.py
View File

@@ -1,5 +1,6 @@
"""Tokenizer.py: Definition for Tokenizer class"""
from errors.custom import JavaSyntaxError
import logging
import re
from typing import List, Optional
@@ -20,10 +21,13 @@ class Tokenizer:
self.source_text = re.sub("(private)|(public)|(protected)", "", self.source_text)
self.type_name_pattern = re.compile('(char)|(int)|(void)|(double)|(Pixel)') #TODO: make this modular
self.type_name_pattern = re.compile('(char)|(int)|(void)|(double)|(boolean)|(Pixel)|(String)') #TODO: make this modular
self._filename = file_name
self._left_curly_number=0
self._right_curly_number=0
def get_tokens(self) -> List[Token]:
tokens = []
@@ -36,9 +40,18 @@ class Tokenizer:
if self._handle_comments(char):
continue
token = self._get_token(char)
logging.debug(f"found token \"{token}\" on line {self.line_number}")
tokens.append(make_token(token, SourceLocation(self._filename, self.line_number, self.column_number), self.type_name_pattern))
tag = self._get_token(char)
logging.debug(f"found tag \"{tag}\" on line {self.line_number}")
if tag == "{":
self._left_curly_number+=1
elif tag == "}":
self._right_curly_number+=1
tokens.append(make_token(tag, SourceLocation(self._filename, self.line_number, self.column_number), self.type_name_pattern))
if self._left_curly_number != self._right_curly_number:
raise JavaSyntaxError(f"Ill-formed Java program! Expected equal number of '{'{'}' and '{'}'}' tokens, got {self._left_curly_number} and {self._right_curly_number}")
return tokens