[cocom] Add repository level analysis option for CoCom backend

graal/backends/core/cocom.py

@@ -70,7 +70,7 @@ class CoCom(Graal):
     :raises RepositoryError: raised when there was an error cloning or
         updating the repository.
     """
-    version = '0.2.3'
+    version = '0.2.5'
 
     CATEGORIES = [CATEGORY_COCOM]
 
@@ -80,9 +80,11 @@ def __init__(self, uri, git_path, worktreepath=DEFAULT_WORKTREE_PATH,
         super().__init__(uri, git_path, worktreepath,
                          entrypoint=entrypoint, in_paths=in_paths, out_paths=out_paths, details=details,
                          tag=tag, archive=archive)
+
+        self.history = dict()
         self.file_analyzer = FileAnalyzer(details)
 
-    def fetch(self, category=CATEGORY_COCOM, paths=None,
+    def fetch(self, category=CATEGORY_COCOM, paths=None, repository_level=False,
               from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
               branches=None, latest_items=False):
         """Fetch commits and add code complexity information."""
@@ -91,6 +93,7 @@ def fetch(self, category=CATEGORY_COCOM, paths=None,
                               from_date=from_date, to_date=to_date,
                               branches=branches, latest_items=latest_items)
 
+        self.repository_level = repository_level
         return items
 
     @staticmethod
@@ -137,13 +140,58 @@ def _analyze(self, commit):
 
             local_path = self.worktreepath + '/' + file_path
             if not GraalRepository.exists(local_path):
-                continue
+                file_info = {
+                    'blanks': None,
+                    'comments': None,
+                    'loc': None,
+                    'ccn': None,
+                    'avg_ccn': None,
+                    'avg_loc': None,
+                    'avg_tokens': None,
+                    'num_funs': None,
+                    'tokens': None,
+                    'file_path': file_path,
+                }
+                if self.details:
+                    file_info['funs'] = []
+
+                if committed_file.get("newfile", None):
+                    file_path = committed_file["newfile"]
+                    local_path = self.worktreepath + '/' + file_path
+                    self.history.pop(file_path, None)
+                    analysis.append(file_info)
+                elif committed_file.get("action", None) == "D":
+                    self.history.pop(file_path, None)
+                    analysis.append(file_info)
+                    continue
+                else:
+                    continue
 
             file_info = self.file_analyzer.analyze(local_path)
             file_info.update({'file_path': file_path})
             analysis.append(file_info)
-
-        return analysis
+            self.history[file_path] = file_info
+
+        if self.repository_level:
+            repository_level_analysis = {
+                'blanks': 0,
+                'comments': 0,
+                'loc': 0,
+                'ccn': 0,
+                'num_funs': 0,
+                'tokens': 0,
+            }
+            for file_path in self.history:
+                repository_level_analysis['blanks'] += self.history[file_path].get('blanks', 0)
+                repository_level_analysis['comments'] += self.history[file_path].get('comments', 0)
+                repository_level_analysis['loc'] += self.history[file_path].get('loc', 0)
+                repository_level_analysis['ccn'] += self.history[file_path].get('ccn', 0)
+                repository_level_analysis['num_funs'] += self.history[file_path].get('num_funs', 0)
+                repository_level_analysis['tokens'] += self.history[file_path].get('tokens', 0)
+
+            return repository_level_analysis
+        else:
+            return analysis
 
     def _post(self, commit):
         """Remove attributes of the Graal item obtained
@@ -182,9 +230,9 @@ def analyze(self, file_path):
           'avg_ccn': ..,
           'avg_loc': ..,
           'avg_tokens': ..,
-          'funs': ..,
+          'num_funs': ..,
           'tokens': ..,
-          'funs_data': [..]
+          'funs': [..]
         }
         """
         kwargs = {'file_path': file_path}

graal/graal.py

@@ -497,6 +497,9 @@ def setup_cmd_parser(categories, exec_path=False):
         group.add_argument('--details', dest='details',
                            action='store_true', default=False,
                            help="include details")
+        group.add_argument('--repository-level', dest='repository_level',
+                           action='store_true', default=False,
+                           help="Perform analysis at repository level")
 
         # Required arguments
         parser.parser.add_argument('uri',

tests/test_cocom.py

@@ -119,6 +119,29 @@ def test_fetch(self):
             self.assertFalse('parents' in commit['data'])
             self.assertFalse('refs' in commit['data'])
 
+    def test_fetch_analysis(self):
+        """Test whether commits have properly set values"""
+
+        cc = CoCom('http://example.com', self.git_path, self.worktree_path)
+        commits = [commit for commit in cc.fetch()]
+
+        self.assertEqual(len(commits), 6)
+        self.assertFalse(os.path.exists(cc.worktreepath))
+
+        deleted_file_commit = commits[5]
+
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['file_path'],
+                         'perceval/backends/graal.py')
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['blanks'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['comments'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['loc'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['ccn'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['avg_ccn'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['avg_loc'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['avg_tokens'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['num_funs'], None)
+        self.assertEqual(deleted_file_commit['data']['analysis'][0]['tokens'], None)
+
 
 class TestFileAnalyzer(TestCaseAnalyzer):
     """FileAnalyzer tests"""

tests/test_codep.py

@@ -90,7 +90,7 @@ def test_fetch(self):
         cd = CoDep('http://example.com', self.git_path, self.worktree_path, entrypoint="perceval")
         commits = [commit for commit in cd.fetch()]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(cd.worktreepath))
 
         commit = commits[0]

tests/test_colang.py

@@ -91,7 +91,7 @@ def test_fetch_linguist(self):
         cl = CoLang('http://example.com', self.git_path, tag="test")
         commits = [commit for commit in cl.fetch()]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(cl.worktreepath))
 
         commit = commits[0]
@@ -106,7 +106,7 @@ def test_fetch_cloc(self):
         cl = CoLang('http://example.com', self.git_path, tag="test")
         commits = [commit for commit in cl.fetch(category=CATEGORY_COLANG_CLOC)]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(cl.worktreepath))
 
         commit = commits[0]

tests/test_coqua.py

@@ -98,7 +98,7 @@ def test_fetch_pylint(self):
                    self.worktree_path, entrypoint="perceval")
         commits = [commit for commit in cq.fetch()]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(cq.worktreepath))
 
         commit = commits[0]
@@ -121,7 +121,7 @@ def test_fetch_flake8(self):
         commits = [commit for commit in cq.fetch(
             category=CATEGORY_COQUA_FLAKE8)]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(cq.worktreepath))
 
         commit = commits[0]

tests/test_covuln.py

@@ -90,7 +90,7 @@ def test_fetch(self):
         cd = CoVuln('http://example.com', self.git_path, self.worktree_path, entrypoint="perceval")
         commits = [commit for commit in cd.fetch()]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(cd.worktreepath))
 
         commit = commits[0]

tests/test_graal.py

@@ -247,7 +247,7 @@ def test_fetch_no_analysis(self):
         graal = Graal('http://example.com', self.git_path, self.worktree_path)
         commits = [commit for commit in graal.fetch()]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(graal.worktreepath))
 
         for commit in commits:
@@ -261,7 +261,7 @@ def test_fetch_analysis(self):
         mocked = MockedGraal('http://example.com', self.git_path, self.worktree_path)
         commits = [commit for commit in mocked.fetch()]
 
-        self.assertEqual(len(commits), 3)
+        self.assertEqual(len(commits), 6)
         self.assertFalse(os.path.exists(mocked.worktreepath))
 
         commit = commits[0]
@@ -762,7 +762,7 @@ def test_items(self):
         items = graal.graal.fetch(CommandBackend, args, CATEGORY_MOCKED)
         items = [item for item in items]
 
-        self.assertEqual(len(items), 3)
+        self.assertEqual(len(items), 6)
         for i in items:
             self.assertEqual(i['category'], CATEGORY_MOCKED)
 
@@ -778,7 +778,7 @@ def test_items_no_category(self):
         items = graal.graal.fetch(CommandBackend, args, None)
         items = [item for item in items]
 
-        self.assertEqual(len(items), 3)
+        self.assertEqual(len(items), 6)
         for i in items:
             self.assertEqual(i['category'], CATEGORY_MOCKED)