[107] Use Python 3 type annotations

.gitignore

@@ -19,3 +19,5 @@ docs/AUTHORS.rst
 docs/SUPPORT.rst
 
 docs/_build/*
+.venv/
+.idea/

pythran/analyses/__init__.py

@@ -18,6 +18,7 @@
 from .constant_expressions import ConstantExpressions
 from .dependencies import Dependencies
 from .extended_syntax_check import ExtendedSyntaxCheck
+from .extract_function_type_annotations import ExtractFunctionTypeAnnotations
 from .fixed_size_list import FixedSizeList
 from .global_declarations import GlobalDeclarations
 from .global_effects import GlobalEffects

pythran/analyses/extract_function_type_annotations.py

@@ -0,0 +1,95 @@
+# Copyright whatever. I don't care.
+
+from pythran.passmanager import ModuleAnalysis
+from pythran.tables import MODULES
+from pythran.spec import Spec
+from pythran.typing import List, Set, Tuple, NDArray
+from pythran.log import logger as log
+from pythran import types
+from pythran import spec
+from typing import ClassVar
+
+import gast as ast
+
+import numpy as np
+
+dtypes = {
+    'bool': np.bool,
+    'byte': np.byte,
+    'complex': np.complex,
+    'int': np.int,
+    'float': np.float,
+    'uint8': np.uint8,
+    'uint16': np.uint16,
+    'uint32': np.uint32,
+    'uint64': np.uint64,
+    'uintc': np.uintc,
+    'uintp': np.uintp,
+    'int8': np.int8,
+    'int16': np.int16,
+    'int32': np.int32,
+    'int64': np.int64,
+    'intc': np.intc,
+    'intp': np.intp,
+    'float32': np.float32,
+    'float64': np.float64,
+    'complex64': np.complex64,
+    'complex128': np.complex128,
+    'str': str,
+    'None': type(None),
+    'bytes': bytes,
+}
+
+
+def type_annot_to_spec(node: ast.AST):
+    if isinstance(node, ast.Name):
+        return dtypes[node.id]
+    elif isinstance(node, ast.Slice):
+        start = node.lower.value if node.lower else 0
+        stop = node.upper.value if node.upper else -1
+        step = node.step.value if node.step else 1
+        s = slice(start, stop, step)
+        return s
+    elif isinstance(node, ast.Constant):
+        return node.value
+    elif isinstance(node, ast.Tuple):
+        return tuple(type_annot_to_spec(n) for n in node.elts)
+    elif isinstance(node, ast.Subscript):
+        if isinstance(node.value, ast.Name):
+            tfm = dict(list=List, set=Set, tuple=Tuple, ndarray=NDArray)
+            if node.value.id not in tfm:
+                raise Exception(f"Instantiation of undefined template '{node.value.id}'.")
+            container = tfm[node.value.id]
+            args = type_annot_to_spec(node.slice)
+            if container is NDArray:
+                elt_ty = args[0]
+                if not isinstance(elt_ty, type):
+                    raise Exception(f"Invalid type argument to template type ndarray; first argument is expected to be a type, got: {elt_ty} of type {type(elt_ty)}.")
+                new_args = [elt_ty]
+                for arg in args[1:]:
+                    if isinstance(arg, int):
+                        new_args.append(slice(0, arg, 1))
+                    elif isinstance(arg, slice):
+                        new_args.append(arg)
+                    else:
+                        raise Exception(f"Invalid type argument to template type ndarray; expected slice or int literal, got: {arg.__name__} of type {type(arg)}.")
+                args = tuple(new_args)
+            return container(args)
+        else:
+            raise Exception(f"Expected template type name, got '{node.value}'.")
+    return node
+
+
+class ExtractFunctionTypeAnnotations(ModuleAnalysis):
+    def __init__(self: 'ExtractFunctionTypeAnnotations'):
+        self.result = dict()
+        self.update = False
+        self.functions = set()
+        ModuleAnalysis.__init__(self)
+
+    def run(self, node):
+        super(ModuleAnalysis, self).run(node)
+        return spec.Spec(functions=self.result)
+
+    def visit_FunctionDef(self: 'ExtractFunctionTypeAnnotations', fn: ast.FunctionDef) -> None:
+        self.result[fn.name] = (tuple(type_annot_to_spec(arg.annotation) for arg in fn.args.args),)

pythran/analyses/extract_function_types.py

@@ -0,0 +1,98 @@
+# Copyright whatever. I don't care.
+
+from pythran.passmanager import ModuleAnalysis
+from pythran.tables import MODULES
+from pythran.spec import Spec
+from pythran.typing import List, Set, Tuple, NDArray
+from pythran.log import logger as log
+from pythran import types
+from pythran import spec
+from astpretty import pprint
+from typing import ClassVar
+
+import gast as ast
+
+import numpy as np
+
+dtypes = {
+    'bool': np.bool,
+    'byte': np.byte,
+    'complex': np.complex,
+    'int': np.int,
+    'float': np.float,
+    'uint8': np.uint8,
+    'uint16': np.uint16,
+    'uint32': np.uint32,
+    'uint64': np.uint64,
+    'uintc': np.uintc,
+    'uintp': np.uintp,
+    'int8': np.int8,
+    'int16': np.int16,
+    'int32': np.int32,
+    'int64': np.int64,
+    'intc': np.intc,
+    'intp': np.intp,
+    'float32': np.float32,
+    'float64': np.float64,
+    'float128': np.float128,
+    'complex64': np.complex64,
+    'complex128': np.complex128,
+    'complex256': np.complex256,
+    'str': str,
+    'None': type(None),
+    'bytes': bytes,
+}
+
+
+def type_annot_to_spec(node: ast.AST):
+    if isinstance(node, ast.Name):
+        return dtypes[node.id]
+    elif isinstance(node, ast.Slice):
+        start = node.lower.value if node.lower else 0
+        stop = node.upper.value if node.upper else -1
+        step = node.step.value if node.step else 1
+        s = slice(start, stop, step)
+        return s
+    elif isinstance(node, ast.Constant):
+        return node.value
+    elif isinstance(node, ast.Tuple):
+        return tuple(type_annot_to_spec(n) for n in node.elts)
+    elif isinstance(node, ast.Subscript):
+        if isinstance(node.value, ast.Name):
+            tfm = dict(list=List, set=Set, tuple=Tuple, ndarray=NDArray)
+            if node.value.id not in tfm:
+                raise Exception(f"Instantiation of undefined template '{node.value.id}'.")
+            container = tfm[node.value.id]
+            args = type_annot_to_spec(node.slice)
+            if container is NDArray:
+                elt_ty = args[0]
+                if not isinstance(elt_ty, type):
+                    raise Exception(f"Invalid type argument to template type ndarray; first argument is expected to be a type, got: {elt_ty} of type {type(elt_ty)}.")
+                new_args = [elt_ty]
+                for arg in args[1:]:
+                    if isinstance(arg, int):
+                        new_args.append(slice(0, arg, 1))
+                    elif isinstance(arg, slice):
+                        new_args.append(arg)
+                    else:
+                        raise Exception(f"Invalid type argument to template type ndarray; expected slice or int literal, got: {arg.__name__} of type {type(arg)}.")
+                args = tuple(new_args)
+            return container(args)
+        else:
+            raise Exception(f"Expected template type name, got '{node.value}'.")
+    return node
+
+
+class ExtractFunctionTypeAnnotations(ModuleAnalysis):
+    def __init__(self: 'ExtractFunctionTypeAnnotations'):
+        self.result = dict()
+        self.update = False
+        self.functions = set()
+        ModuleAnalysis.__init__(self)
+
+    def run(self, node):
+        super(ModuleAnalysis, self).run(node)
+        return spec.Spec(functions=self.result)
+
+    def visit_FunctionDef(self: 'ExtractFunctionTypeAnnotations', fn: ast.FunctionDef) -> None:
+        self.result[fn.name] = (tuple(type_annot_to_spec(arg.annotation) for arg in fn.args.args),)

pythran/middlend.py

@@ -1,6 +1,6 @@
 """This module turns a python AST into an optimized, pythran compatible ast."""
 
-from pythran.analyses import ExtendedSyntaxCheck
+from pythran.analyses import ExtendedSyntaxCheck, ExtractFunctionTypeAnnotations
 from pythran.optimizations import (ComprehensionPatterns, ListCompToGenexp,
                                    RemoveDeadFunctions)
 from pythran.transformations import (ExpandBuiltins, ExpandImports,
@@ -12,12 +12,14 @@
                                      UnshadowParameters, RemoveNamedArguments,
                                      ExpandGlobals, NormalizeIsNone,
                                      NormalizeIfElse,
-                                     NormalizeStaticIf, SplitStaticExpression)
+                                     NormalizeStaticIf, SplitStaticExpression,
+                                     RemoveTypeAnnotations)
 
 
 def refine(pm, node, optimizations):
     """ Refine node in place until it matches pythran's expectations. """
     # Sanitize input
+    pm.apply(RemoveTypeAnnotations, node)
     pm.apply(RemoveDeadFunctions, node)
     pm.apply(ExpandGlobals, node)
     pm.apply(ExpandImportAll, node)

pythran/toolchain.py

@@ -3,6 +3,7 @@
 a dynamic library, see __init__.py for exported interfaces.
 '''
 
+from pythran.analyses.extract_function_type_annotations import ExtractFunctionTypeAnnotations
 from pythran.backend import Cxx, Python
 from pythran.config import cfg
 from pythran.cxxgen import PythonModule, Include, Line, Statement
@@ -130,10 +131,21 @@ def generate_cxx(module_name, code, specs=None, optimizations=None,
     a compile error (e.g. due to bad typing)
 
     '''
+    def extract_annotated_specs(module_name, code) -> Spec:
+        pm = PassManager(module_name, module_dir)
+        node, docstrings = frontend.parse(pm, code)
+        return pm.gather(ExtractFunctionTypeAnnotations, node)
+
+    ann_specs = extract_annotated_specs(module_name, code)
+
     if specs:
+        assert set(ann_specs.keys()) == set(specs.keys())
+        for fn, ty in ann_specs.functions.items():
+            specs.functions[fn] = ty
         entry_points = set(specs.keys())
+        print(str(ann_specs.__dict__), str(specs.__dict__))
     else:
-        entry_points = None
+        entry_points = ann_specs
 
     pm, ir, docstrings = front_middle_end(module_name, code, optimizations,
                                           module_dir,
@@ -385,11 +397,37 @@ def compile_pythrancode(module_name, pythrancode, specs=None,
             shutil.move(tmp_file, output_file)
             logger.info("Generated Python source file: " + output_file)
 
+    def extract_annotated_specs(module_name, code) -> Spec:
+        pm = PassManager(module_name, module_dir)
+        node, docstrings = frontend.parse(pm, code)
+        return pm.gather(ExtractFunctionTypeAnnotations, node)
+
+    ann_specs = extract_annotated_specs(module_name, pythrancode)
+
     # Autodetect the Pythran spec if not given as parameter
     from pythran.spec import spec_parser
     if specs is None:
         specs = spec_parser(pythrancode)
 
+    if specs:
+        # if a spec directive could be detected then:
+        # 1. where a name appears empty in the spec directive (eg. #pythran export foo), overwrite the empty
+        #    fn type with the type annotations;
+        # 2. where a name defines a function in the spec directive (eg. #pythran export foo(int8)), verify
+        #    that the types match those in the type annotation.
+        dual_spec_fns = set(specs.functions.keys()) & set(ann_specs.functions.keys())
+        mismatches: list[str] = []
+        for fn in dual_spec_fns:
+            t = specs.functions[fn]
+            u = ann_specs.functions[fn]
+            if not t or t == ():
+                t = u
+            if t != u:
+                mismatches.append(f"Type annotations don't match pythran spec directive for '{fn}': expected {t}; got {u}.")
+        if mismatches:
+            # FIXME: the error should be emitted with the location. Not sure how to do that.
+            raise CompileError(" ".join(mismatches))
+
     # Generate C++, get a PythonModule object
     module, error_checker = generate_cxx(module_name, pythrancode, specs, opts,
                                          module_dir)

pythran/transformations/__init__.py

@@ -31,3 +31,4 @@
 from .remove_nested_functions import RemoveNestedFunctions
 from .unshadow_parameters import UnshadowParameters
 from .remove_named_arguments import RemoveNamedArguments
+from .remove_type_annotations import RemoveTypeAnnotations

pythran/transformations/remove_type_annotations.py

@@ -0,0 +1,37 @@
+""" RemoveLambdas turns lambda into regular functions.  """
+
+from pythran.analyses import GlobalDeclarations, ImportedIds
+from pythran.passmanager import Transformation
+from pythran.tables import MODULES
+from pythran.conversion import mangle
+
+import pythran.metadata as metadata
+
+from copy import copy
+import gast as ast
+
+
+class RemoveTypeAnnotations(Transformation):
+    """
+    Removes type annotations in function and variable
+    definitions.
+
+    TODO: add the example
+    """
+
+    def __init__(self):
+        super(RemoveTypeAnnotations, self).__init__(GlobalDeclarations)
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST:
+        self.generic_visit(node)
+
+        node.returns = None
+        for arg in node.args.args:
+            arg.annotation = None
+
+        return node
+
+    def visit_AnnAssign(self, node: ast.AnnAssign) -> ast.AST:
+        self.generic_visit(node)
+        node.value = None
+        return node

requirements-dev.txt

@@ -0,0 +1,2 @@
+pytest-runner
+astpretty