Add GetHieAsts rule, Replace SpanInfo, add support for DocumentHighlight and scope-aware completions for local variables (haskell/ghcide#784)

* Add GetHieAsts rule

* hlint

* fix build for 8.4

* Reimplement Hover/GotoDefn in terms of HIE Files.
Implement Document Hightlight LSP request
Add GetDocMap, GetHieFile rules.

* Fix gotodef for record fields

* Completion for locals

* Don't need to hack cursor position because of fuzzy ranges

* hlint

* fix bench and warning on 8.10

* disable 8.4 CI jobs

* Don't collect module level bindings

* tweaks

* Show kinds

* docs

* Defs for ModuleNames

* Fix some tests

* hlint

* Mark remaining tests as broken

* Add completion tests

* add highlight tests

* Fix HieAst for 8.6

* CPP away the unexpected success

* More CPP hacks for 8.10 tests

Author: wz1000

.hlint.yaml

@@ -95,7 +95,7 @@
 - flags:
   - default: false
   - {name: [-Wno-missing-signatures, -Wno-orphans, -Wno-overlapping-patterns, -Wno-incomplete-patterns, -Wno-missing-fields, -Wno-unused-matches]}
-  - {name: [-Wno-dodgy-imports], within: [Main, Development.IDE.GHC.Compat]}
+  - {name: [-Wno-dodgy-imports,-Wno-incomplete-uni-patterns], within: [Main, Development.IDE.GHC.Compat]}
 # - modules:
 #   - {name: [Data.Set, Data.HashSet], as: Set} # if you import Data.Set qualified, it must be as 'Set'
 #   - {name: Control.Arrow, within: []} # Certain modules are banned entirely

ghcide/.azure/linux-stack.yml

@@ -11,8 +11,6 @@ jobs:
         STACK_YAML: "stack88.yaml"
       stack_86:
         STACK_YAML: "stack.yaml"
-      stack_84:
-        STACK_YAML: "stack84.yaml"
       stack_ghc_lib_88:
         STACK_YAML: "stack-ghc-lib.yaml"
   variables:

ghcide/.azure/windows-stack.yml

@@ -11,8 +11,6 @@ jobs:
         STACK_YAML: "stack88.yaml"
       stack_86:
         STACK_YAML: "stack.yaml"
-      stack_84:
-        STACK_YAML: "stack84.yaml"
       stack_ghc_lib_88:
         STACK_YAML: "stack-ghc-lib.yaml"
   variables:

ghcide/ghcide.cabal

@@ -48,6 +48,7 @@ library
         extra,
         fuzzy,
         filepath,
+        fingertree,
         haddock-library >= 1.8,
         hashable,
         haskell-lsp-types == 0.22.*,
@@ -140,6 +141,8 @@ library
         Development.IDE.LSP.Protocol
         Development.IDE.LSP.Server
         Development.IDE.Spans.Common
+        Development.IDE.Spans.AtPoint
+        Development.IDE.Spans.LocalBindings
         Development.IDE.Types.Diagnostics
         Development.IDE.Types.Exports
         Development.IDE.Types.Location
@@ -173,10 +176,7 @@ library
         Development.IDE.GHC.WithDynFlags
         Development.IDE.Import.FindImports
         Development.IDE.LSP.Notifications
-        Development.IDE.Spans.AtPoint
-        Development.IDE.Spans.Calculate
         Development.IDE.Spans.Documentation
-        Development.IDE.Spans.Type
         Development.IDE.Plugin.CodeAction.PositionIndexed
         Development.IDE.Plugin.CodeAction.Rules
         Development.IDE.Plugin.CodeAction.RuleTypes

ghcide/src/Development/IDE/Core/Compile.hs

@@ -18,7 +18,8 @@ module Development.IDE.Core.Compile
   , addRelativeImport
   , mkTcModuleResult
   , generateByteCode
-  , generateAndWriteHieFile
+  , generateHieAsts
+  , writeHieFile
   , writeHiFile
   , getModSummaryFromImports
   , loadHieFile
@@ -56,7 +57,7 @@ import ErrUtils
 #endif
 
 import           Finder
-import           Development.IDE.GHC.Compat hiding (parseModule, typecheckModule)
+import           Development.IDE.GHC.Compat hiding (parseModule, typecheckModule, writeHieFile)
 import qualified Development.IDE.GHC.Compat     as GHC
 import qualified Development.IDE.GHC.Compat     as Compat
 import           GhcMonad
@@ -65,7 +66,7 @@ import qualified HeaderInfo                     as Hdr
 import           HscMain                        (hscInteractive, hscSimplify)
 import           MkIface
 import           StringBuffer                   as SB
-import           TcRnMonad (tct_id, TcTyThing(AGlobal, ATcId), initTc, initIfaceLoad, tcg_th_coreplugins)
+import           TcRnMonad (tct_id, TcTyThing(AGlobal, ATcId), initTc, initIfaceLoad, tcg_th_coreplugins, tcg_binds)
 import           TcIface                        (typecheckIface)
 import           TidyPgm
 
@@ -320,7 +321,7 @@ mkTcModuleResult tcm upgradedError = do
     (iface, _) <- liftIO $ mkIfaceTc session Nothing sf details tcGblEnv
 #endif
     let mod_info = HomeModInfo iface details Nothing
-    return $ TcModuleResult tcm mod_info upgradedError
+    return $ TcModuleResult tcm mod_info upgradedError Nothing
   where
     (tcGblEnv, details) = tm_internals_ tcm
 
@@ -331,19 +332,25 @@ atomicFileWrite targetPath write = do
   (tempFilePath, cleanUp) <- newTempFileWithin dir
   (write tempFilePath >> renameFile tempFilePath targetPath) `onException` cleanUp
 
-generateAndWriteHieFile :: HscEnv -> TypecheckedModule -> BS.ByteString -> IO [FileDiagnostic]
-generateAndWriteHieFile hscEnv tcm source =
-  handleGenerationErrors dflags "extended interface generation" $ do
+generateHieAsts :: HscEnv -> TypecheckedModule -> IO ([FileDiagnostic], Maybe (HieASTs Type))
+generateHieAsts hscEnv tcm =
+  handleGenerationErrors' dflags "extended interface generation" $ do
     case tm_renamed_source tcm of
-      Just rnsrc -> do
-        hf <- runHsc hscEnv $
-          GHC.mkHieFile mod_summary (fst $ tm_internals_ tcm) rnsrc source
-        atomicFileWrite targetPath $ flip GHC.writeHieFile hf
+      Just rnsrc -> runHsc hscEnv $
+        Just <$> GHC.enrichHie (tcg_binds $ fst $ tm_internals_ tcm) rnsrc
       _ ->
-        return ()
+        return Nothing
+  where
+    dflags       = hsc_dflags hscEnv
+
+writeHieFile :: HscEnv -> ModSummary -> [GHC.AvailInfo] -> HieASTs Type -> BS.ByteString -> IO [FileDiagnostic]
+writeHieFile hscEnv mod_summary exports ast source =
+  handleGenerationErrors dflags "extended interface write/compression" $ do
+    hf <- runHsc hscEnv $
+      GHC.mkHieFile' mod_summary exports ast source
+    atomicFileWrite targetPath $ flip GHC.writeHieFile hf
   where
     dflags       = hsc_dflags hscEnv
-    mod_summary  = pm_mod_summary $ tm_parsed_module tcm
     mod_location = ms_location mod_summary
     targetPath   = Compat.ml_hie_file mod_location
 
@@ -365,6 +372,14 @@ handleGenerationErrors dflags source action =
     . (("Error during " ++ T.unpack source) ++) . show @SomeException
     ]
 
+handleGenerationErrors' :: DynFlags -> T.Text -> IO (Maybe a) -> IO ([FileDiagnostic], Maybe a)
+handleGenerationErrors' dflags source action =
+  fmap ([],) action `catches`
+    [ Handler $ return . (,Nothing) . diagFromGhcException source dflags
+    , Handler $ return . (,Nothing) . diagFromString source DsError (noSpan "<internal>")
+    . (("Error during " ++ T.unpack source) ++) . show @SomeException
+    ]
+
 
 -- | Setup the environment that GHC needs according to our
 -- best understanding (!)

ghcide/src/Development/IDE/Core/PositionMapping.hs

@@ -74,7 +74,6 @@ toCurrentPosition (PositionMapping pm) = positionResultToMaybe . toDelta pm
 -- a specific version
 newtype PositionMapping = PositionMapping PositionDelta
 
-
 toCurrentRange :: PositionMapping -> Range -> Maybe Range
 toCurrentRange mapping (Range a b) =
     Range <$> toCurrentPosition mapping a <*> toCurrentPosition mapping b
@@ -121,7 +120,7 @@ toCurrent (Range start@(Position startLine startColumn) end@(Position endLine en
     | line > endLine || line == endLine && column >= endColumn =
       -- Position is after the change so increase line and column number
       -- as necessary.
-      PositionExact $ Position newLine newColumn
+      PositionExact $ newLine `seq` newColumn `seq` Position newLine newColumn
     | otherwise = PositionRange start end
     -- Position is in the region that was changed.
     where
@@ -131,10 +130,10 @@ toCurrent (Range start@(Position startLine startColumn) end@(Position endLine en
         newEndColumn
           | linesNew == 0 = startColumn + T.length t
           | otherwise = T.length $ T.takeWhileEnd (/= '\n') t
-        !newColumn
+        newColumn
           | line == endLine = column + newEndColumn - endColumn
           | otherwise = column
-        !newLine = line + lineDiff
+        newLine = line + lineDiff
 
 fromCurrent :: Range -> T.Text -> Position -> PositionResult Position
 fromCurrent (Range start@(Position startLine startColumn) end@(Position endLine endColumn)) t (Position line column)
@@ -144,7 +143,7 @@ fromCurrent (Range start@(Position startLine startColumn) end@(Position endLine
     | line > newEndLine || line == newEndLine && column >= newEndColumn =
       -- Position is after the change so increase line and column number
       -- as necessary.
-      PositionExact $ Position newLine newColumn
+      PositionExact $ newLine `seq` newColumn `seq` Position newLine newColumn
     | otherwise = PositionRange start end
     -- Position is in the region that was changed.
     where
@@ -155,7 +154,7 @@ fromCurrent (Range start@(Position startLine startColumn) end@(Position endLine
         newEndColumn
           | linesNew == 0 = startColumn + T.length t
           | otherwise = T.length $ T.takeWhileEnd (/= '\n') t
-        !newColumn
+        newColumn
           | line == newEndLine = column - (newEndColumn - endColumn)
           | otherwise = column
-        !newLine = line - lineDiff
+        newLine = line - lineDiff

ghcide/src/Development/IDE/Core/RuleTypes.hs

@@ -15,22 +15,24 @@ import           Control.DeepSeq
 import Data.Aeson.Types (Value)
 import Data.Binary
 import           Development.IDE.Import.DependencyInformation
-import Development.IDE.GHC.Compat
+import Development.IDE.GHC.Compat hiding (HieFileResult)
 import Development.IDE.GHC.Util
 import Development.IDE.Core.Shake (KnownTargets)
 import           Data.Hashable
 import           Data.Typeable
 import qualified Data.Set as S
+import qualified Data.Map as M
 import           Development.Shake
 import           GHC.Generics                             (Generic)
 
 import Module (InstalledUnitId)
 import HscTypes (hm_iface, CgGuts, Linkable, HomeModInfo, ModDetails)
 
-import           Development.IDE.Spans.Type
+import           Development.IDE.Spans.Common
+import           Development.IDE.Spans.LocalBindings
 import           Development.IDE.Import.FindImports (ArtifactsLocation)
 import Data.ByteString (ByteString)
-
+import Language.Haskell.LSP.Types (NormalizedFilePath)
 
 -- NOTATION
 --   Foo+ means Foo for the dependencies
@@ -66,6 +68,7 @@ data TcModuleResult = TcModuleResult
     -- HomeModInfo instead
     , tmrModInfo    :: HomeModInfo
     , tmrDeferedError :: !Bool -- ^ Did we defer any type errors for this module?
+    , tmrHieAsts :: !(Maybe (HieASTs Type)) -- ^ The HieASTs if we computed them
     }
 instance Show TcModuleResult where
     show = show . pm_mod_summary . tm_parsed_module . tmrModule
@@ -98,11 +101,38 @@ instance NFData HiFileResult where
 instance Show HiFileResult where
     show = show . hirModSummary
 
+-- | Save the uncompressed AST here, we compress it just before writing to disk
+data HieAstResult
+  = HAR
+  { hieModule :: Module
+  , hieAst :: !(HieASTs Type)
+  , refMap :: !RefMap
+  , importMap :: !(M.Map ModuleName NormalizedFilePath) -- ^ Where are the modules imported by this file located?
+  }
+ 
+instance NFData HieAstResult where
+    rnf (HAR m hf rm im) = rnf m `seq` rwhnf hf `seq` rnf rm `seq` rnf im
+ 
+instance Show HieAstResult where
+    show = show . hieModule
+
 -- | The type checked version of this file, requires TypeCheck+
 type instance RuleResult TypeCheck = TcModuleResult
 
--- | Information about what spans occur where, requires TypeCheck
-type instance RuleResult GetSpanInfo = SpansInfo
+-- | The uncompressed HieAST
+type instance RuleResult GetHieAst = HieAstResult
+
+-- | A IntervalMap telling us what is in scope at each point
+type instance RuleResult GetBindings = Bindings
+
+data DocAndKindMap = DKMap {getDocMap :: !DocMap, getKindMap :: !KindMap}
+instance NFData DocAndKindMap where
+    rnf (DKMap a b) = rnf a `seq` rnf b
+
+instance Show DocAndKindMap where
+    show = const "docmap"
+
+type instance RuleResult GetDocMap = DocAndKindMap
 
 -- | Convert to Core, requires TypeCheck*
 type instance RuleResult GenerateCore = (SafeHaskellMode, CgGuts, ModDetails)
@@ -196,11 +226,23 @@ instance Hashable TypeCheck
 instance NFData   TypeCheck
 instance Binary   TypeCheck
 
-data GetSpanInfo = GetSpanInfo
+data GetDocMap = GetDocMap
+    deriving (Eq, Show, Typeable, Generic)
+instance Hashable GetDocMap
+instance NFData   GetDocMap
+instance Binary   GetDocMap
+
+data GetHieAst = GetHieAst
+    deriving (Eq, Show, Typeable, Generic)
+instance Hashable GetHieAst
+instance NFData   GetHieAst
+instance Binary   GetHieAst
+
+data GetBindings = GetBindings
     deriving (Eq, Show, Typeable, Generic)
-instance Hashable GetSpanInfo
-instance NFData   GetSpanInfo
-instance Binary   GetSpanInfo
+instance Hashable GetBindings
+instance NFData   GetBindings
+instance Binary   GetBindings
 
 data GenerateCore = GenerateCore
     deriving (Eq, Show, Typeable, Generic)
@@ -262,4 +304,4 @@ instance Hashable GetClientSettings
 instance NFData   GetClientSettings
 instance Binary   GetClientSettings
 
-type instance RuleResult GetClientSettings = Hashed (Maybe Value)
+type instance RuleResult GetClientSettings = Hashed (Maybe Value)