[cocom] Add repository level analysis via lizard

Add repository level analysis by adding a new category of analyzer(RepositoryAnalyzer) which executes lizard at repository level for cocom backend.
Add and alter tests for corresponding changes.

Signed-off-by: inishchith <[email protected]>

Author: inishchith

graal/backends/core/analyzers/lizard.py

@@ -18,12 +18,11 @@
 #
 # Authors:
 #     Valerio Cosentino <[email protected]>
+#     Nishchith Shetty <[email protected]>
 #
 
-import warnings
-
 import lizard
-
+from graal.backends.core.analyzers.cloc import Cloc
 from .analyzer import Analyzer
 
 
@@ -45,10 +44,10 @@ class Lizard(Analyzer):
         Golang
         Lua
     """
-    version = '0.2.1'
+    version = '0.3.0'
 
-    def analyze(self, **kwargs):
-        """Add code complexity information using Lizard.
+    def __analyze_file(self, file_path, details):
+        """Add code complexity information for a file using Lizard.
 
         Current information includes cyclomatic complexity (ccn),
         avg lines of code and tokens, number of functions and tokens.
@@ -61,12 +60,7 @@ def analyze(self, **kwargs):
         :returns  result: dict of the results of the analysis
         """
         result = {}
-        file_path = kwargs['file_path']
-        details = kwargs['details']
-
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', DeprecationWarning)
-            analysis = lizard.analyze_file(file_path)
+        analysis = lizard.analyze_file(file_path)
 
         result['ccn'] = analysis.CCN
         result['avg_ccn'] = analysis.average_cyclomatic_complexity
@@ -94,3 +88,65 @@ def analyze(self, **kwargs):
 
         result['funs'] = funs_data
         return result
+
+    def __analyze_repository(self, repository_path, files_affected, details):
+        """Add code complexity information for a given repository
+        using Lizard and CLOC.
+
+        Current information includes cyclomatic complexity (ccn),
+        lines of code, number of functions, tokens, blanks and comments.
+
+        :param repository_path: repository path
+        :param details: if True, it returns fine-grained results
+
+        :returns  result: list of the results of the analysis
+        """
+        analysis_result = []
+
+        repository_analysis = lizard.analyze(
+            paths=[repository_path],
+            threads=1,
+            exts=lizard.get_extensions([]),
+        )
+        cloc = Cloc()
+
+        for analysis in repository_analysis:
+            cloc_analysis = cloc.analyze(file_path=analysis.filename)
+            file_path = analysis.filename.replace(repository_path + "/", '')
+            in_commit = True if file_path in files_affected else False
+
+            result = {
+                'loc': analysis.nloc,
+                'ccn': analysis.CCN,
+                'tokens': analysis.token_count,
+                'num_funs': len(analysis.function_list),
+                'file_path': file_path,
+                'in_commit': in_commit,
+                'blanks': cloc_analysis['blanks'],
+                'comments': cloc_analysis['comments']
+            }
+            analysis_result.append(result)
+
+        # TODO: implement details option
+
+        return analysis_result
+
+    def analyze(self, **kwargs):
+        """Add code complexity information using Lizard.
+
+        :param file_path: file path
+        :param repository_path: repository path
+        :param details: if True, it returns detailed information about an analysis
+
+        :returns  result: the results of the analysis
+        """
+
+        details = kwargs['details']
+
+        if kwargs.get('repository_level', False):
+            files_affected = kwargs['files_affected']
+            result = self.__analyze_repository(kwargs["repository_path"], files_affected, details)
+        else:
+            result = self.__analyze_file(kwargs['file_path'], details)
+
+        return result

graal/backends/core/cocom.py

@@ -23,14 +23,19 @@
 import logging
 
 from graal.graal import (Graal,
+                         GraalError,
                          GraalRepository,
                          GraalCommand,
                          DEFAULT_WORKTREE_PATH)
 from graal.backends.core.analyzers.cloc import Cloc
 from graal.backends.core.analyzers.lizard import Lizard
 from perceval.utils import DEFAULT_DATETIME, DEFAULT_LAST_DATETIME
 
-CATEGORY_COCOM = 'code_complexity'
+LIZARD_FILE = 'lizard_file'
+LIZARD_REPOSITORY = 'lizard_repository'
+
+CATEGORY_COCOM_LIZARD_FILE = 'code_complexity_' + LIZARD_FILE
+CATEGORY_COCOM_LIZARD_REPOSITORY = 'code_complexity_' + LIZARD_REPOSITORY
 
 logger = logging.getLogger(__name__)
 
@@ -70,37 +75,55 @@ class CoCom(Graal):
     :raises RepositoryError: raised when there was an error cloning or
         updating the repository.
     """
-    version = '0.2.4'
+    version = '0.2.5'
 
-    CATEGORIES = [CATEGORY_COCOM]
+    CATEGORIES = [CATEGORY_COCOM_LIZARD_FILE,
+                  CATEGORY_COCOM_LIZARD_REPOSITORY]
 
     def __init__(self, uri, git_path, worktreepath=DEFAULT_WORKTREE_PATH,
                  entrypoint=None, in_paths=None, out_paths=None, details=False,
                  tag=None, archive=None):
         super().__init__(uri, git_path, worktreepath,
                          entrypoint=entrypoint, in_paths=in_paths, out_paths=out_paths, details=details,
                          tag=tag, archive=archive)
-        self.file_analyzer = FileAnalyzer(details)
 
-    def fetch(self, category=CATEGORY_COCOM, paths=None,
+        self.analyzer = None
+        self.analyzer_kind = None
+
+    def fetch(self, category=CATEGORY_COCOM_LIZARD_FILE, paths=None,
               from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
               branches=None, latest_items=False):
         """Fetch commits and add code complexity information."""
 
         items = super().fetch(category,
                               from_date=from_date, to_date=to_date,
                               branches=branches, latest_items=latest_items)
+        if category == CATEGORY_COCOM_LIZARD_FILE:
+            self.analyzer_kind = LIZARD_FILE
+            self.analyzer = FileAnalyzer(self.details)
+        elif category == CATEGORY_COCOM_LIZARD_REPOSITORY:
+            self.analyzer_kind = LIZARD_REPOSITORY
+            self.analyzer = RepositoryAnalyzer(self.details)
+        else:
+            raise GraalError(cause="Unknown category %s" % category)
 
         return items
 
     @staticmethod
     def metadata_category(item):
         """Extracts the category from a Code item.
 
-        This backend only generates one type of item which is
-        'code_complexity'.
+        This backend generates the following types of item:
+        - 'code_complexity_lizard_file'.
+        - 'code_complexity_lizard_repository'
         """
-        return CATEGORY_COCOM
+
+        if item['analyzer'] == LIZARD_FILE:
+            return CATEGORY_COCOM_LIZARD_FILE
+        elif item['analyzer'] == LIZARD_REPOSITORY:
+            return CATEGORY_COCOM_LIZARD_REPOSITORY
+        else:
+            raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
 
     def _filter_commit(self, commit):
         """Filter a commit according to its data (e.g., author, sha, etc.)
@@ -121,51 +144,54 @@ def _filter_commit(self, commit):
 
     def _analyze(self, commit):
         """Analyse a commit and the corresponding
-        checkout version of the repository
+        checkout version of the repository.
 
         :param commit: a Perceval commit item
         """
         analysis = []
 
-        for committed_file in commit['files']:
-
-            file_path = committed_file['file']
-            if self.in_paths:
-                found = [p for p in self.in_paths if file_path.endswith(p)]
-                if not found:
-                    continue
-
-            local_path = self.worktreepath + '/' + file_path
-            if not GraalRepository.exists(local_path):
-                file_info = {
-                    'blanks': None,
-                    'comments': None,
-                    'loc': None,
-                    'ccn': None,
-                    'avg_ccn': None,
-                    'avg_loc': None,
-                    'avg_tokens': None,
-                    'num_funs': None,
-                    'tokens': None,
-                    'file_path': file_path,
-                }
-                if self.details:
-                    file_info['funs'] = []
-
-                if committed_file.get("newfile", None):
-                    file_path = committed_file["newfile"]
-                    local_path = self.worktreepath + '/' + file_path
-                    analysis.append(file_info)
-                elif committed_file.get("action", None) == "D":
-                    analysis.append(file_info)
-                    continue
-                else:
-                    continue
-
-            file_info = self.file_analyzer.analyze(local_path)
-            file_info.update({'file_path': file_path})
-            analysis.append(file_info)
-
+        if self.analyzer_kind == LIZARD_FILE:
+            for committed_file in commit['files']:
+
+                file_path = committed_file['file']
+                if self.in_paths:
+                    found = [p for p in self.in_paths if file_path.endswith(p)]
+                    if not found:
+                        continue
+
+                local_path = self.worktreepath + '/' + file_path
+                if not GraalRepository.exists(local_path):
+                    file_info = {
+                        'blanks': None,
+                        'comments': None,
+                        'loc': None,
+                        'ccn': None,
+                        'avg_ccn': None,
+                        'avg_loc': None,
+                        'avg_tokens': None,
+                        'num_funs': None,
+                        'tokens': None,
+                        'file_path': file_path,
+                    }
+                    if self.details:
+                        file_info['funs'] = []
+
+                    if committed_file.get("newfile", None):
+                        file_path = committed_file["newfile"]
+                        local_path = self.worktreepath + '/' + file_path
+                        analysis.append(file_info)
+                    elif committed_file.get("action", None) == "D":
+                        analysis.append(file_info)
+                        continue
+                    else:
+                        continue
+
+                file_info = self.analyzer.analyze(local_path)
+                file_info.update({'file_path': file_path})
+                analysis.append(file_info)
+        else:
+            files_affected = [file_info['file'] for file_info in commit['files']]
+            analysis = self.analyzer.analyze(self.worktreepath, files_affected)
         return analysis
 
     def _post(self, commit):
@@ -176,6 +202,8 @@ def _post(self, commit):
         commit.pop('files', None)
         commit.pop('parents', None)
         commit.pop('refs', None)
+        commit['analyzer'] = self.analyzer_kind
+
         return commit
 
 
@@ -192,7 +220,7 @@ def __init__(self, details=False):
         self.lizard = Lizard()
 
     def analyze(self, file_path):
-        """Analyze the content of a file using CLOC and Lizard
+        """Analyze the content of a file using CLOC and Lizard.
 
         :param file_path: file path
 
@@ -227,6 +255,43 @@ def analyze(self, file_path):
         return lizard_analysis
 
 
+class RepositoryAnalyzer:
+    """Class to analyse the content of a repository"""
+
+    def __init__(self, details=False):
+        self.details = details
+        self.lizard = Lizard()
+
+    def analyze(self, repository_path, files_affected):
+        """Analyze the content of a repository using CLOC and Lizard.
+
+        :param repository_path: repository path
+
+        :returns a list containing the results of the analysis
+        [ {
+          'loc': ..,
+          'ccn': ..,
+          'tokens': ..,
+          'num_funs': ..,
+          'file_path': ..,
+          'in_commit': ..,
+          'blanks': ..,
+          'comments': ..,
+          },
+          ...
+        ]
+        """
+        kwargs = {
+            'repository_path': repository_path,
+            'repository_level': True,
+            'files_affected': files_affected,
+            'details': self.details
+        }
+        lizard_analysis = self.lizard.analyze(**kwargs)
+
+        return lizard_analysis
+
+
 class CoComCommand(GraalCommand):
     """Class to run CoCom backend from the command line."""