Merge pull request #7 from Tecnativa/master-depth

Master depth

.travis.yml

@@ -22,6 +22,7 @@ install:
 
 # command to run tests
 script:
+  - git --version
   - flake8 . --exclude=__init__.py
   - coverage run --source git_aggregator setup.py test
 

README.rst

@@ -64,6 +64,49 @@ You can specify that you want to fetch all references from all remotes you have
 
     fetch_all: true
 
+Shallow repositories
+--------------------
+
+To save big amounts of bandwidth and disk space, you can use shallow clones.
+These download only a restricted amount of commits depending on some criteria.
+Available options are `depth`_, `shallow-since`_ and `shallow-exclude`_.
+
+.. warning::
+
+    Available options depend on server and client Git version, be sure to use
+    options available for your environment.
+
+.. _depth: https://git-scm.com/docs/git-fetch#git-fetch---depthltdepthgt
+.. _shallow-since: https://git-scm.com/docs/git-fetch#git-fetch---shallow-sinceltdategt
+.. _shallow-exclude: https://git-scm.com/docs/git-fetch#git-fetch---shallow-excludeltrevisiongt
+
+You can use those in the ``defaults`` sections to apply them everywhere, or
+specifying them in the corresponding ``merges`` section, for which you must use
+the ``dict`` alternate construction. If you need to disable a default in
+``merges``, set it to ``false``:
+
+.. code-block:: yaml
+
+    ./odoo:
+        defaults:
+            depth: 20
+        remotes:
+            odoo: https://github.com/odoo/odoo.git
+            ocb: https://github.com/OCA/OCB.git
+            acsone: https://github.com/acsone/odoo.git
+        merges:
+            -
+                remote: ocb
+                ref: "9.0"
+                depth: 1000
+            -
+                remote: odoo
+                ref: refs/pull/14859/head
+        target: acsone 9.0
+
+Remember that you need to fetch at least the common ancestor of all merges for
+it to succeed.
+
 Triggers
 --------
 

git_aggregator/config.py

@@ -25,6 +25,7 @@ def get_repos(config):
             directory = os.path.abspath(directory)
         repo_dict = {
             'cwd': directory,
+            'defaults': repo_data.get('defaults', dict()),
         }
         remote_names = set()
         if 'remotes' in repo_data:
@@ -50,21 +51,32 @@ def get_repos(config):
             merges = []
             merge_data = repo_data.get('merges') or []
             for merge in merge_data:
-                parts = merge.split(' ')
-                if len(parts) != 2:
-                    raise ConfigException(
-                        '%s: Merge must be formatted as '
-                        '"remote_name ref".' % directory)
-
-                remote_name, ref = merge.split(' ')
-                if remote_name not in remote_names:
+                try:
+                    # Assume parts is a str
+                    parts = merge.split(' ')
+                    if len(parts) != 2:
+                        raise ConfigException(
+                            '%s: Merge must be formatted as '
+                            '"remote_name ref".' % directory)
+                    merge = {
+                        "remote": parts[0],
+                        "ref": parts[1],
+                    }
+                except AttributeError:
+                    # Parts is a dict
+                    try:
+                        merge["remote"] = str(merge["remote"])
+                        merge["ref"] = str(merge["ref"])
+                    except KeyError:
+                        raise ConfigException(
+                            '%s: Merge lacks mandatory '
+                            '`remote` or `ref` keys.' % directory)
+                # Check remote is available
+                if merge["remote"] not in remote_names:
                     raise ConfigException(
                         '%s: Merge remote %s not defined in remotes.' %
-                        (directory, remote_name))
-                merges.append({
-                    'remote': remote_name,
-                    'ref': ref,
-                })
+                        (directory, merge["remote"]))
+                merges.append(merge)
             repo_dict['merges'] = merges
             if not merges:
                 raise ConfigException(

git_aggregator/repo.py

@@ -12,6 +12,7 @@
 from .exception import GitAggregatorException
 from ._compat import console_to_str
 
+FETCH_DEFAULTS = ("depth", "shallow-since", "shallow-exclude")
 logger = logging.getLogger(__name__)
 
 
@@ -34,7 +35,7 @@ class Repo(object):
     _git_version = None
 
     def __init__(self, cwd, remotes, merges, target,
-                 shell_command_after=None, fetch_all=False):
+                 shell_command_after=None, fetch_all=False, defaults=None):
         """Initialize a git repository aggregator
 
         :param cwd: path to the directory where to initialize the repository
@@ -43,12 +44,14 @@ def __init__(self, cwd, remotes, merges, target,
         :param: merges list of merge to apply to build the aggregated
         repository. A merge is a dict {'remote': '', 'ref': ''}
         :param target:
+        :param shell_command_after: an optional list of shell command to
+        execute after the aggregation
         :param fetch_all:
             Can be an iterable (recommended: ``frozenset``) that yields names
             of remotes where all refs should be fetched, or ``True`` to do it
             for every configured remote.
-        :param shell_command_after: an optional list of shell command to
-        execute after the aggregation
+        :param defaults:
+            Collection of default parameters to be passed to git.
         """
         self.cwd = cwd
         self.remotes = remotes
@@ -59,6 +62,7 @@ def __init__(self, cwd, remotes, merges, target,
         self.merges = merges
         self.target = target
         self.shell_command_after = shell_command_after or []
+        self.defaults = defaults or dict()
 
     @property
     def git_version(self):
@@ -174,9 +178,9 @@ def aggregate(self):
                 # reset to the first merge
                 origin = merges[0]
                 merges = merges[1:]
-                self._reset_to(**origin)
+                self._reset_to(origin["remote"], origin["ref"])
             for merge in merges:
-                self._merge(**merge)
+                self._merge(merge)
             self._execute_shell_command_after()
         logger.info('End aggregation of %s', self.cwd)
 
@@ -188,7 +192,7 @@ def fetch(self):
         basecmd = ("git", "fetch")
         logger.info("Fetching required remotes")
         for merge in self.merges:
-            cmd = basecmd + (merge["remote"],)
+            cmd = basecmd + self._fetch_options(merge) + (merge["remote"],)
             if merge["remote"] not in self.fetch_all:
                 cmd += (merge["ref"],)
             self.log_call(cmd, cwd=self.cwd)
@@ -201,6 +205,15 @@ def push(self):
             os.chdir(self.cwd)
             self.log_call(['git', 'push', '-f', remote, branch])
 
+    def _fetch_options(self, merge):
+        """Get the fetch options from the given merge dict."""
+        cmd = tuple()
+        for option in FETCH_DEFAULTS:
+            value = merge.get(option, self.defaults.get(option))
+            if value:
+                cmd += ("--%s" % option, str(value))
+        return cmd
+
     def _reset_to(self, remote, ref):
         logger.info('Reset branch to %s %s', remote, ref)
         rtype, sha = self.query_remote_ref(remote, ref)
@@ -223,16 +236,17 @@ def _execute_shell_command_after(self):
         for cmd in self.shell_command_after:
             self.log_call(cmd, shell=True)
 
-    def _merge(self, remote, ref):
-        logger.info("Pull %s, %s", remote, ref)
-        cmd = ['git', 'pull', remote, ref]
+    def _merge(self, merge):
+        logger.info("Pull %s, %s", merge["remote"], merge["ref"])
+        cmd = ("git", "pull")
         if self.git_version >= (1, 7, 10):
             # --edit and --no-edit appear with Git 1.7.10
             # see Documentation/RelNotes/1.7.10.txt of Git
             # (https://git.kernel.org/cgit/git/git.git/tree)
-            cmd.insert(2, '--no-edit')
+            cmd += ('--no-edit',)
         if logger.getEffectiveLevel() != logging.DEBUG:
-            cmd.insert(2, '--quiet')
+            cmd += ('--quiet',)
+        cmd += self._fetch_options(merge) + (merge["remote"], merge["ref"])
         self.log_call(cmd)
 
     def _get_remotes(self):

tests/test_config.py

@@ -41,6 +41,7 @@ def test_load(self):
             repos[0],
             {'cwd': '/product_attribute',
              'fetch_all': False,
+             'defaults': {},
              'merges': [{'ref': '8.0', 'remote': 'oca'},
                         {'ref': 'refs/pull/105/head', 'remote': 'oca'},
                         {'ref': 'refs/pull/106/head', 'remote': 'oca'}],
@@ -57,6 +58,52 @@ def test_load(self):
               'url':
               'git+ssh://git@github.com/acsone/product-attribute.git'}])
 
+    def test_load_defaults(self):
+        config_yaml = dedent("""
+            /web:
+                defaults:
+                    depth: 1
+                remotes:
+                    oca: https://github.com/OCA/web.git
+                    acsone: git+ssh://git@github.com/acsone/web.git
+                merges:
+                    -
+                        remote: oca
+                        ref: 8.0
+                        depth: 1000
+                    - oca refs/pull/105/head
+                    -
+                        remote: oca
+                        ref: refs/pull/106/head
+                target: acsone aggregated_branch_name
+        """)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertEquals(len(repos), 1)
+        # remotes are configured as dict therefore the order is not preserved
+        # when parsed
+        remotes = repos[0]['remotes']
+        repos[0]['remotes'] = []
+        self.assertDictEqual(
+            repos[0],
+            {'cwd': '/web',
+             'fetch_all': False,
+             'defaults': {'depth': 1},
+             'merges': [{'ref': '8.0', 'remote': 'oca', 'depth': 1000},
+                        {'ref': 'refs/pull/105/head', 'remote': 'oca'},
+                        {'ref': 'refs/pull/106/head', 'remote': 'oca'}],
+             'remotes': [],
+             'shell_command_after': [],
+             'target': {'branch': 'aggregated_branch_name',
+                        'remote': 'acsone'}})
+        assertfn = self.assertItemsEqual if PY2 else self.assertCountEqual
+        assertfn(
+            remotes,
+            [{'name': 'oca',
+              'url': 'https://github.com/OCA/web.git'},
+             {'name': 'acsone',
+              'url':
+              'git+ssh://git@github.com/acsone/web.git'}])
+
     def test_load_shell_command_after(self):
         """Shell command after are alway parser as a list
         """
@@ -181,6 +228,21 @@ def test_load_merges_exception(self):
             ex.exception.args[0],
             '/product_attribute: Merge remote oba not defined in remotes.')
 
+        config_yaml = dedent("""
+            /web:
+                remotes:
+                    oca: https://github.com/OCA/web.git
+                merges:
+                    -
+                        depth: 1
+                target: oca aggregated_branch
+        """)
+        with self.assertRaises(ConfigException) as ex:
+            config.get_repos(self._parse_config(config_yaml))
+        self.assertEquals(
+            ex.exception.args[0],
+            '/web: Merge lacks mandatory `remote` or `ref` keys.')
+
     def test_load_target_exception(self):
         config_yaml = """
 /product_attribute:

tests/test_repo.py

@@ -213,3 +213,70 @@ def test_update_aggregate_2(self):
             self.remote1, 'tracked_new', "last", msg="new file on remote1")
         repo.aggregate()
         self.assertFalse(os.path.isfile(os.path.join(self.cwd, 'tracked_new')))
+
+    def test_depth_1(self):
+        """Ensure a simple shallow clone with 1 commit works."""
+        remotes = [{
+            'name': 'shallow',
+            'url': self.url_remote1
+        }]
+        merges = [{
+            'remote': 'shallow',
+            "ref": "master",
+        }]
+        target = {
+            'remote': 'shallow',
+            'branch': 'master'
+        }
+        defaults = {
+            "depth": 1,
+        }
+        repo = Repo(self.cwd, remotes, merges, target, defaults=defaults)
+        repo.aggregate()
+        self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked')))
+
+        with working_directory_keeper:
+            os.chdir(self.cwd)
+            log_shallow = subprocess.check_output(
+                ("git", "rev-list", "shallow/master"))
+        # Shallow fetch: just 1 commmit
+        self.assertEqual(len(log_shallow.splitlines()), 1)
+
+    def test_depth(self):
+        """Ensure `depth` is used correctly."""
+        remotes = [{
+            'name': 'r1',
+            'url': self.url_remote1
+        }, {
+            'name': 'r2',
+            'url': self.url_remote2
+        }]
+        merges = [{
+            'remote': 'r1',
+            "ref": "master",
+        }, {
+            "remote": "r2",
+            'ref': "b2",
+        }]
+        target = {
+            'remote': 'r1',
+            'branch': 'agg'
+        }
+        defaults = {
+            "depth": 2,
+        }
+        repo = Repo(self.cwd, remotes, merges, target, defaults=defaults)
+        repo.aggregate()
+        self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked')))
+        self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked2')))
+
+        with working_directory_keeper:
+            os.chdir(self.cwd)
+            log_r1 = subprocess.check_output(
+                ("git", "rev-list", "r1/master"))
+            log_r2 = subprocess.check_output(
+                ("git", "rev-list", "r2/b2"))
+        # Shallow fetch: just 1 commmit
+        self.assertEqual(len(log_r1.splitlines()), 2)
+        # Full fetch: all 3 commits
+        self.assertEqual(len(log_r2.splitlines()), 2)