# Simple recursive descent parser for SL-LEX format # class to represent a token class Token: # constructor method def __init__(self, line, col, token, lexeme): self.type = token.upper() self.value = lexeme self.lineno = line self.lexerpos = col # python calls the repr method to print something out def __repr__(self): return "Token({}, {}, {}, {})".format( self.lineno, self.lexerpos, self.type, self.value) # Class to read SL-Lex file class TokenReader: def __init__(self, f): """ f - file-like object containing SL-Lex data """ # create a list of all the lines in the file lines = f.readlines() # member variable to track the tokens # __ essentially makes this variable private self.__tokens = [] # simple while loop to read tokens i = 0 while i < len(lines): # first we read a line number lineno = int(lines[i]) colno = int(lines[i+1]) # python doesn't strip newlines tok_type = lines[i+2].strip() if tok_type in ["string", "int", "ident"]: lexeme = lines[i+3].strip() i += 1 else: lexeme = tok_type self.__tokens.append( Token(lineno, colno, tok_type, lexeme)) # increment i by 3 i += 3 # debug check for having read everything assert(i == len(lines)) # turn the token list into an iterator to grab the next # token self.token_stream = iter(self.__tokens) # method to grab the next token or None if we've read them all def token(self): return next(self.token_stream, None) # main program to try this out if __name__ == "__main__": import sys tokens = TokenReader(sys.stdin) t = tokens.token() while t is not None: print(t) t = tokens.token()