Add Token_Stream class as new lexer and ast_link to every token (#61)

Infos:
- I have not considered Markup-Strings yet. They are just normal
Strings.
- I added string representations for some ast objects to better see how
things are connected.
- All ast_links are set in the parser except for import statements as
their names are resolved in the [ast.py](http://ast.py/).
- The import token itself is set to the files own package because it was
easier to set the link at the moment the token first appears. The names
are resolved after the graph was built and set correctly to their own
ast package's object.
- Comments are linked to the files main package. I have put that at the
end of each file's parsing because comments could appear at the top
before the "package ..." clause.

trlc/ast.py

@@ -112,6 +112,10 @@ def __init__(self, location):
         assert isinstance(location, Location)
         self.location = location
 
+    def set_ast_link(self, tok):
+        assert isinstance(tok, Token)
+        tok.ast_link = self
+
     def write_indent(self, indent, message):  # pragma: no cover
         # lobster-exclude: Debugging feature
         assert isinstance(indent, int)
@@ -262,7 +266,9 @@ def resolve_imports(self, mh, stab):
             # We can ignore errors here, because that just means we
             # generate more error later.
             try:
-                self.imports.add(stab.lookup(mh, t_import, Package))
+                a_import = stab.lookup(mh, t_import, Package)
+                self.imports.add(a_import)
+                a_import.set_ast_link(t_import)
             except TRLC_Error:
                 pass
 
@@ -2391,6 +2397,10 @@ def dump(self, indent=0):  # pragma: no cover
         self.write_indent(indent + 1, "Declared_Late: %s" % self.declared_late)
         self.symbols.dump(indent + 1, omit_heading=True)
 
+    def __repr__(self):
+        return "%s<%s>" % (self.__class__.__name__,
+                           self.name)
+
 
 class Composite_Type(Concrete_Type, metaclass=ABCMeta):
     """Abstract base for record and tuple types, as they share some
@@ -2487,6 +2497,11 @@ def dump(self, indent=0):  # pragma: no cover
         self.write_indent(indent + 1, "Optional: %s" % self.optional)
         self.write_indent(indent + 1, "Type: %s" % self.n_typ.name)
 
+    def __repr__(self):
+        return "%s<%s>" % (self.__class__.__name__,
+                           self.member_of.fully_qualified_name() + "." +
+                           self.name)
+
 
 class Record_Type(Composite_Type):
     """A user-defined record type.
@@ -2942,6 +2957,12 @@ def perform_checks(self, mh):
 
         return ok
 
+    def __repr__(self):
+        return "%s<%s>" % (self.__class__.__name__,
+                           self.n_package.name + "." +
+                           self.n_typ.name + "." +
+                           self.name)
+
 
 class Section(Entity):
     # lobster-trace: LRM.Section_Declaration

trlc/lexer.py

@@ -158,7 +158,7 @@ class Token(Token_Base):
         "STRING"     : "string literal",
     }
 
-    def __init__(self, location, kind, value=None):
+    def __init__(self, location, kind, value=None, ast_link=None):
         assert kind in Token.KIND
         if kind in ("COMMENT", "IDENTIFIER",
                     "KEYWORD", "OPERATOR", "STRING"):
@@ -170,6 +170,7 @@ def __init__(self, location, kind, value=None):
         else:
             assert value is None
         super().__init__(location, kind, value)
+        self.ast_link = ast_link
 
     def __repr__(self):
         if self.value is None:
@@ -185,6 +186,7 @@ def __init__(self, mh, content):
         self.mh        = mh
         self.content   = content
         self.length    = len(self.content)
+        self.tokens    = []
 
         self.lexpos  = -3
         self.line_no = 0
@@ -632,6 +634,15 @@ def token(self):
         return Token(sref, kind, value)
 
 
+class Token_Stream(TRLC_Lexer):
+
+    def token(self):
+        tok = super().token()
+        if tok is not None:
+            self.tokens.append(tok)
+        return tok
+
+
 def sanity_test():
     # lobster-exclude: Developer test function
     mh    = Message_Handler()