[mypyc] Detect always defined attributes (#12600)

Use static analysis to find attributes that are always defined. Always defined
attributes don't require checks on each access. This makes them faster and 
also reduces code size.

Attributes defined in the class body and assigned to in all code paths in 
`__init__` are always defined. We need to know all subclasses statically
to determine whether `__init__` always defines an attribute in every case,
including in subclasses.

The analysis looks at `__init__` methods and supports limited inter-procedural
analysis over `super().__init__(...)` calls. Otherwise we rely on intra-procedural
analysis to keep the analysis fast.

As a side effect, `__init__` will now always be called when constructing an object.
This means that `copy.copy` (and others like it) won't be supported for native 
classes unless `__init__` can be called without arguments.

`mypyc/analysis/attrdefined.py` has more details about the algorithm in 
docstrings.

Performance impact to selected benchmarks (with clang):
- richards +28% 
- deltablue +10%
- hexiom +1%

The richards result is probably an outlier.

This will also significantly help with native integers (mypyc/mypyc#837, as tracking 
undefined values would otherwise require extra memory use.

Closes mypyc/mypyc#836.

Author: JukkaL

mypy/copytype.py

@@ -0,0 +1,111 @@
+from typing import Any, cast
+
+from mypy.types import (
+    ProperType, UnboundType, AnyType, NoneType, UninhabitedType, ErasedType, DeletedType,
+    Instance, TypeVarType, ParamSpecType, PartialType, CallableType, TupleType, TypedDictType,
+    LiteralType, UnionType, Overloaded, TypeType, TypeAliasType, UnpackType, Parameters,
+    TypeVarTupleType
+)
+from mypy.type_visitor import TypeVisitor
+
+
+def copy_type(t: ProperType) -> ProperType:
+    """Create a shallow copy of a type.
+
+    This can be used to mutate the copy with truthiness information.
+
+    Classes compiled with mypyc don't support copy.copy(), so we need
+    a custom implementation.
+    """
+    return t.accept(TypeShallowCopier())
+
+
+class TypeShallowCopier(TypeVisitor[ProperType]):
+    def visit_unbound_type(self, t: UnboundType) -> ProperType:
+        return t
+
+    def visit_any(self, t: AnyType) -> ProperType:
+        return self.copy_common(t, AnyType(t.type_of_any, t.source_any, t.missing_import_name))
+
+    def visit_none_type(self, t: NoneType) -> ProperType:
+        return self.copy_common(t, NoneType())
+
+    def visit_uninhabited_type(self, t: UninhabitedType) -> ProperType:
+        dup = UninhabitedType(t.is_noreturn)
+        dup.ambiguous = t.ambiguous
+        return self.copy_common(t, dup)
+
+    def visit_erased_type(self, t: ErasedType) -> ProperType:
+        return self.copy_common(t, ErasedType())
+
+    def visit_deleted_type(self, t: DeletedType) -> ProperType:
+        return self.copy_common(t, DeletedType(t.source))
+
+    def visit_instance(self, t: Instance) -> ProperType:
+        dup = Instance(t.type, t.args, last_known_value=t.last_known_value)
+        dup.invalid = t.invalid
+        return self.copy_common(t, dup)
+
+    def visit_type_var(self, t: TypeVarType) -> ProperType:
+        dup = TypeVarType(
+            t.name,
+            t.fullname,
+            t.id,
+            values=t.values,
+            upper_bound=t.upper_bound,
+            variance=t.variance,
+        )
+        return self.copy_common(t, dup)
+
+    def visit_param_spec(self, t: ParamSpecType) -> ProperType:
+        dup = ParamSpecType(t.name, t.fullname, t.id, t.flavor, t.upper_bound, prefix=t.prefix)
+        return self.copy_common(t, dup)
+
+    def visit_parameters(self, t: Parameters) -> ProperType:
+        dup = Parameters(t.arg_types, t.arg_kinds, t.arg_names,
+                         variables=t.variables,
+                         is_ellipsis_args=t.is_ellipsis_args)
+        return self.copy_common(t, dup)
+
+    def visit_type_var_tuple(self, t: TypeVarTupleType) -> ProperType:
+        dup = TypeVarTupleType(t.name, t.fullname, t.id, t.upper_bound)
+        return self.copy_common(t, dup)
+
+    def visit_unpack_type(self, t: UnpackType) -> ProperType:
+        dup = UnpackType(t.type)
+        return self.copy_common(t, dup)
+
+    def visit_partial_type(self, t: PartialType) -> ProperType:
+        return self.copy_common(t, PartialType(t.type, t.var, t.value_type))
+
+    def visit_callable_type(self, t: CallableType) -> ProperType:
+        return self.copy_common(t, t.copy_modified())
+
+    def visit_tuple_type(self, t: TupleType) -> ProperType:
+        return self.copy_common(t, TupleType(t.items, t.partial_fallback, implicit=t.implicit))
+
+    def visit_typeddict_type(self, t: TypedDictType) -> ProperType:
+        return self.copy_common(t, TypedDictType(t.items, t.required_keys, t.fallback))
+
+    def visit_literal_type(self, t: LiteralType) -> ProperType:
+        return self.copy_common(t, LiteralType(value=t.value, fallback=t.fallback))
+
+    def visit_union_type(self, t: UnionType) -> ProperType:
+        return self.copy_common(t, UnionType(t.items))
+
+    def visit_overloaded(self, t: Overloaded) -> ProperType:
+        return self.copy_common(t, Overloaded(items=t.items))
+
+    def visit_type_type(self, t: TypeType) -> ProperType:
+        # Use cast since the type annotations in TypeType are imprecise.
+        return self.copy_common(t, TypeType(cast(Any, t.item)))
+
+    def visit_type_alias_type(self, t: TypeAliasType) -> ProperType:
+        assert False, "only ProperTypes supported"
+
+    def copy_common(self, t: ProperType, t2: ProperType) -> ProperType:
+        t2.line = t.line
+        t2.column = t.column
+        t2.can_be_false = t.can_be_false
+        t2.can_be_true = t.can_be_true
+        return t2

mypy/moduleinspect.py

@@ -12,19 +12,20 @@
 
 
 class ModuleProperties:
+    # Note that all __init__ args must have default values
     def __init__(self,
-                 name: str,
-                 file: Optional[str],
-                 path: Optional[List[str]],
-                 all: Optional[List[str]],
-                 is_c_module: bool,
-                 subpackages: List[str]) -> None:
+                 name: str = "",
+                 file: Optional[str] = None,
+                 path: Optional[List[str]] = None,
+                 all: Optional[List[str]] = None,
+                 is_c_module: bool = False,
+                 subpackages: Optional[List[str]] = None) -> None:
         self.name = name  # __name__ attribute
         self.file = file  # __file__ attribute
         self.path = path  # __path__ attribute
         self.all = all  # __all__ attribute
         self.is_c_module = is_c_module
-        self.subpackages = subpackages
+        self.subpackages = subpackages or []
 
 
 def is_c_module(module: ModuleType) -> bool:

mypy/nodes.py

@@ -668,16 +668,16 @@ class FuncItem(FuncBase):
     __deletable__ = ('arguments', 'max_pos', 'min_args')
 
     def __init__(self,
-                 arguments: List[Argument],
-                 body: 'Block',
+                 arguments: Optional[List[Argument]] = None,
+                 body: Optional['Block'] = None,
                  typ: 'Optional[mypy.types.FunctionLike]' = None) -> None:
         super().__init__()
-        self.arguments = arguments
-        self.arg_names = [None if arg.pos_only else arg.variable.name for arg in arguments]
+        self.arguments = arguments or []
+        self.arg_names = [None if arg.pos_only else arg.variable.name for arg in self.arguments]
         self.arg_kinds: List[ArgKind] = [arg.kind for arg in self.arguments]
         self.max_pos: int = (
             self.arg_kinds.count(ARG_POS) + self.arg_kinds.count(ARG_OPT))
-        self.body: 'Block' = body
+        self.body: 'Block' = body or Block([])
         self.type = typ
         self.unanalyzed_type = typ
         self.is_overload: bool = False
@@ -725,10 +725,11 @@ class FuncDef(FuncItem, SymbolNode, Statement):
                  'original_def',
                  )
 
+    # Note that all __init__ args must have default values
     def __init__(self,
-                 name: str,              # Function name
-                 arguments: List[Argument],
-                 body: 'Block',
+                 name: str = '',              # Function name
+                 arguments: Optional[List[Argument]] = None,
+                 body: Optional['Block'] = None,
                  typ: 'Optional[mypy.types.FunctionLike]' = None) -> None:
         super().__init__(arguments, body, typ)
         self._name = name

mypy/stubtest.py

@@ -895,7 +895,6 @@ def _resolve_funcitem_from_decorator(dec: nodes.OverloadPart) -> Optional[nodes.
 
     Returns None if we can't figure out what that would be. For convenience, this function also
     accepts FuncItems.
-
     """
     if isinstance(dec, nodes.FuncItem):
         return dec
@@ -917,6 +916,7 @@ def apply_decorator_to_funcitem(
             return func
         if decorator.fullname == "builtins.classmethod":
             assert func.arguments[0].variable.name in ("cls", "metacls")
+            # FuncItem is written so that copy.copy() actually works, even when compiled
             ret = copy.copy(func)
             # Remove the cls argument, since it's not present in inspect.signature of classmethods
             ret.arguments = ret.arguments[1:]

mypy/typeops.py

@@ -14,15 +14,15 @@
     TupleType, Instance, FunctionLike, Type, CallableType, TypeVarLikeType, Overloaded,
     TypeVarType, UninhabitedType, FormalArgument, UnionType, NoneType,
     AnyType, TypeOfAny, TypeType, ProperType, LiteralType, get_proper_type, get_proper_types,
-    copy_type, TypeAliasType, TypeQuery, ParamSpecType, Parameters,
-    ENUM_REMOVED_PROPS
+    TypeAliasType, TypeQuery, ParamSpecType, Parameters, ENUM_REMOVED_PROPS
 )
 from mypy.nodes import (
     FuncBase, FuncItem, FuncDef, OverloadedFuncDef, TypeInfo, ARG_STAR, ARG_STAR2, ARG_POS,
     Expression, StrExpr, Var, Decorator, SYMBOL_FUNCBASE_TYPES
 )
 from mypy.maptype import map_instance_to_supertype
 from mypy.expandtype import expand_type_by_instance, expand_type
+from mypy.copytype import copy_type
 
 from mypy.typevars import fill_typevars
 

mypy/types.py

@@ -1,6 +1,5 @@
 """Classes for representing mypy types."""
 
-import copy
 import sys
 from abc import abstractmethod
 
@@ -2893,16 +2892,6 @@ def is_named_instance(t: Type, fullnames: Union[str, Tuple[str, ...]]) -> bool:
     return isinstance(t, Instance) and t.type.fullname in fullnames
 
 
-TP = TypeVar('TP', bound=Type)
-
-
-def copy_type(t: TP) -> TP:
-    """
-    Build a copy of the type; used to mutate the copy with truthiness information
-    """
-    return copy.copy(t)
-
-
 class InstantiateAliasVisitor(TypeTranslator):
     def __init__(self, vars: List[str], subs: List[Type]) -> None:
         self.replacements = {v: s for (v, s) in zip(vars, subs)}