Merge pull request #180 from msimacek/collection-branching

Collection branching

admin.py

@@ -31,7 +31,7 @@
 from koschei.backend import koji_util
 from koschei.db import get_engine, create_all
 from koschei.models import (Package, PackageGroup, AdminNotice, Collection,
-                            CollectionGroup)
+                            CollectionGroup, CollectionGroupRelation)
 from koschei.config import load_config, get_config
 
 
@@ -61,6 +61,7 @@ def main():
         parser = subparser.add_parser(cmd_name, help=cmd.__doc__)
         cmd.setup_parser(parser)
         parser.set_defaults(cmd=cmd)
+        parser.description = cmd.__doc__
     args = main_parser.parse_args()
     cmd = args.cmd
     kwargs = vars(args)
@@ -450,6 +451,70 @@ def execute(self, session, name, force):
         session.db.delete(collection)
 
 
+class BranchCollection(CreateOrEditCollectionCommand, Command):
+    """
+    Performs branching for given collection. It performs these steps:
+    1. creates a branched collection with the same attributes as the master one,
+       except for display name and bugzilla version passed as arguments
+    2. copies most recent (1 month) builds from master collection to the branched one
+    3. sets new name for the master collection and moves it to give new koji target
+    """
+
+    def setup_parser(self, parser):
+        parser.add_argument('master_collection',
+                            help="Master collection from which new collection should be branched")
+        parser.add_argument('new_name',
+                            help="New name for the master collection after the branched copy is created")
+        parser.add_argument('-d', '--display-name',
+                            required=True,
+                            help="Human readable name for branched collection")
+        parser.add_argument('-t', '--target',
+                            required=True,
+                            help="New Koji target for the master collection")
+        parser.add_argument('--bugzilla-version',
+                            help="Product version used in bugzilla template for"
+                                 "the branched collection")
+
+    def execute(self, session, master_collection, new_name, display_name,
+                target, bugzilla_version):
+        master = (
+            session.db.query(Collection)
+            .filter_by(name=master_collection)
+        ).first()
+        if not master:
+            sys.exit("Collection not found")
+        branched = (
+            session.db.query(Collection)
+            .filter_by(name=new_name)
+        ).first()
+        if branched:
+            sys.exit("Branched collection exists already")
+        branched = Collection(
+            display_name=display_name,
+            bugzilla_version=bugzilla_version,
+        )
+        for key in ('secondary_mode', 'priority_coefficient', 'bugzilla_product',
+                    'poll_untracked', 'build_group', 'target', 'name', 'order'):
+            setattr(branched, key, getattr(master, key))
+        master.name = new_name
+        master.target = target
+        master.order += 1
+        self.set_koji_tags(session, branched)
+        self.set_koji_tags(session, master)
+        session.db.add(branched)
+        session.db.flush()
+        for group_rel in (
+                session.db.query(CollectionGroupRelation)
+                .filter_by(collection_id=master.id)
+        ):
+            session.db.add(CollectionGroupRelation(
+                collection_id=branched.id,
+                group_id=group_rel.group_id,
+            ))
+
+        data.copy_collection(session, master, branched)
+
+
 class CreateOrEditCollectionGroupCommand(object):
     create = True
 

alembic/versions/94f1b9dde3e1_drop_applied_change_prev_build_id.py

@@ -0,0 +1,24 @@
+"""
+Drop applied_change.prev_build_id
+
+Create Date: 2017-08-28 14:30:54.584354
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '94f1b9dde3e1'
+down_revision = '7ea8ffafe48b'
+
+from alembic import op
+
+
+def upgrade():
+    op.execute("""
+        ALTER TABLE applied_change DROP COLUMN prev_build_id;
+        ALTER TABLE unapplied_change DROP COLUMN prev_build_id;
+    """)
+
+
+def downgrade():
+    # not possible to restore the data
+    raise NotImplementedError()

koschei/backend/services/resolver.py

@@ -412,7 +412,7 @@ def generate_dependency_changes(self, sack, collection, packages, brs):
                     )
                     changes = create_dependency_changes(
                         prev_deps, curr_deps, package_id=package.id,
-                        prev_build_id=prev_build.id)
+                    )
             results.append(ResolutionOutput(
                 package=package,
                 prev_resolved=package.resolved,
@@ -568,8 +568,7 @@ def process_build(self, sack, entry, curr_deps):
             prev_deps = self.dependency_cache.get_by_ids(self.db, prev.dependency_keys)
             if prev_deps and curr_deps:
                 changes = create_dependency_changes(prev_deps, curr_deps,
-                                                    build_id=entry.id,
-                                                    prev_build_id=prev.id)
+                                                    build_id=entry.id)
                 if changes:
                     self.db.execute(AppliedChange.__table__.insert(), changes)
         self.db.query(Build)\

koschei/data.py

@@ -21,9 +21,11 @@
 from sqlalchemy import insert
 
 from koschei.db import get_or_create
-from koschei.models import (Package, BasePackage, PackageGroup,
-                            PackageGroupRelation, GroupACL, User,
-                            Collection, CollectionGroupRelation)
+from koschei.models import (
+    Package, BasePackage, PackageGroup, PackageGroupRelation, GroupACL, User,
+    Collection, CollectionGroupRelation, Build, AppliedChange, KojiTask,
+    ResolutionChange, ResolutionProblem,
+)
 
 
 class PackagesDontExist(Exception):
@@ -129,3 +131,88 @@ def set_collection_group_content(session, group, collection_names):
         .delete()
     if rels:
         session.db.execute(insert(CollectionGroupRelation, rels))
+
+
+def copy_collection(session, source, copy):
+
+    def get_cols(entity, exclude=(), qualify=False):
+        return ', '.join(
+            (entity.__tablename__ + '.' + c.name if qualify else c.name)
+            for c in entity.__table__.columns
+            if c.name != 'id' and c.name not in exclude
+        )
+
+    def deepcopy_table(entity, whereclause=''):
+        foreign_keys = entity.__table__.foreign_keys
+        assert len(foreign_keys) == 1
+        foreign_key = next(iter(foreign_keys))
+        parent = foreign_key.column.table
+        fk_cols = [fk.parent.name for fk in foreign_keys if fk.column.table is parent]
+        assert len(fk_cols) == 1
+        fk_col = fk_cols[0]
+        session.log.info("Copying {} table".format(entity.__tablename__))
+        session.db.execute("""
+            CREATE TEMPORARY TABLE {table}_copy AS
+                SELECT {table}.id AS original_id,
+                        nextval('{table}_id_seq') AS id,
+                        {parent}.id AS {fk_col}
+                    FROM {table} JOIN {parent}_copy AS {parent}
+                            ON {fk_col} = {parent}.original_id
+                    {whereclause}
+                    ORDER BY {table}.id;
+            INSERT INTO {table}(id, {fk_col}, {cols})
+                SELECT {table}_copy.id AS id,
+                       {table}_copy.{fk_col} AS {fk_col},
+                       {cols_q}
+                    FROM {table} JOIN {table}_copy
+                        ON {table}.id = {table}_copy.original_id
+                    ORDER BY {table}_copy.original_id;
+        """.format(
+            table=entity.__tablename__,
+            parent=parent.name,
+            fk_col=fk_col,
+            cols=get_cols(entity, exclude=[fk_col]),
+            cols_q=get_cols(entity, exclude=[fk_col], qualify=True),
+            whereclause=whereclause,
+        ))
+
+    session.log.info("Copying package table")
+    session.db.execute("""
+        CREATE TEMPORARY TABLE package_copy AS
+            SELECT id AS original_id,
+                    nextval('package_id_seq') AS id
+                FROM package
+                WHERE collection_id = {source.id};
+        INSERT INTO package(id, collection_id, {package_cols})
+            SELECT package_copy.id AS id,
+                   {copy.id} AS collection_id,
+                   {package_cols_q}
+                FROM package JOIN package_copy
+                    ON package.id = package_copy.original_id;
+
+        -- NOTE: package.last_[complete_]build is updated by trigger
+    """.format(
+        copy=copy, source=source,
+        package_cols=get_cols(Package, exclude=['collection_id']),
+        build_cols=get_cols(Build, exclude=['package_id']),
+        applied_change_cols=get_cols(AppliedChange, exclude=['build_id']),
+        package_cols_q=get_cols(Package, exclude=['collection_id'], qualify=True),
+        build_cols_q=get_cols(Build, exclude=['package_id'], qualify=True),
+    ))
+
+    deepcopy_table(
+        Build,
+        whereclause="""
+            WHERE started > now() - '1 month'::interval
+                  OR build.id IN (
+                        SELECT last_complete_build_id
+                        FROM package
+                        WHERE collection_id = {copy.id}
+                  )
+        """.format(copy=copy),
+    )
+
+    deepcopy_table(KojiTask)
+    deepcopy_table(ResolutionChange)
+    deepcopy_table(ResolutionProblem)
+    deepcopy_table(AppliedChange)

koschei/frontend/views.py

@@ -431,6 +431,7 @@ def unified_package_view(template, query_fn=None, **template_args):
 def collection_list():
     groups = db.query(CollectionGroup)\
         .options(joinedload(CollectionGroup.collections))\
+        .order_by(CollectionGroup.name)\
         .all()
     categorized_ids = {c.id for g in groups for c in g.collections}
     uncategorized = [c for c in g.collections if c.id not in categorized_ids]

koschei/models.py

@@ -530,9 +530,6 @@ class AppliedChange(Base):
     build_id = Column(ForeignKey('build.id', ondelete='CASCADE'), index=True,
                       nullable=False)
     build = None  # backref
-    # needs to be nullable because we delete old builds
-    prev_build_id = Column(ForeignKey('build.id', ondelete='SET NULL'),
-                           index=True)
     _prev_evr = composite(
         RpmEVR,
         prev_epoch, prev_version, prev_release,
@@ -571,8 +568,6 @@ class UnappliedChange(Base):
 
     package_id = Column(ForeignKey('package.id', ondelete='CASCADE'),
                         index=True, nullable=False)
-    prev_build_id = Column(ForeignKey('build.id', ondelete='CASCADE'),
-                           index=True, nullable=False)
     prev_evr = composite(
         RpmEVR,
         prev_epoch, prev_version, prev_release,
@@ -827,7 +822,7 @@ class ResourceConsumptionStats(MaterializedView):
 CollectionGroup.collections = relationship(
     Collection,
     secondary=CollectionGroupRelation.__table__,
-    order_by=(Collection.order, Collection.name.desc()),
+    order_by=(Collection.order.desc(), Collection.name.desc()),
     passive_deletes=True,
 )
 CoprRebuildRequest.collection = relationship(Collection)

test/common.py

@@ -231,7 +231,8 @@ def prepare_packages(self, *pkg_names):
         self.db.commit()
         return pkgs
 
-    def prepare_build(self, pkg_name, state=None, repo_id=None, resolved=True, arches=()):
+    def prepare_build(self, pkg_name, state=None, repo_id=None, resolved=True,
+                      arches=(), started=None):
         states = {
             True: Build.COMPLETE,
             False: Build.FAILED,
@@ -245,7 +246,7 @@ def prepare_build(self, pkg_name, state=None, repo_id=None, resolved=True, arche
                       repo_id=repo_id or (1 if state != Build.RUNNING else None),
                       version='1', release='1.fc25',
                       task_id=self.task_id_counter,
-                      started=datetime.fromtimestamp(self.task_id_counter),
+                      started=started or datetime.fromtimestamp(self.task_id_counter),
                       deps_resolved=resolved)
         self.task_id_counter += 1
         self.db.add(build)

test/data_test.py

@@ -20,8 +20,13 @@
 
 import six
 
+from datetime import datetime
+
 from test.common import DBTest
-from koschei.models import PackageGroup, PackageGroupRelation
+from koschei.models import (
+    PackageGroup, PackageGroupRelation, Collection, Package, AppliedChange,
+    KojiTask, ResolutionChange, ResolutionProblem,
+)
 from koschei import data
 
 
@@ -78,3 +83,73 @@ def test_track_packages(self):
     def test_track_packages_nonexistent(self):
         with self.assertRaises(data.PackagesDontExist):
             data.track_packages(self.session, self.collection, ['bar'])
+
+    def test_copy_collection(self):
+        now = datetime.now()
+        source = self.collection
+        _, _, maven1 = self.prepare_packages('rnv', 'eclipse', 'maven')
+        self.prepare_build('rnv')
+        self.prepare_build('eclipse')
+        # the next build is old and shouldn't be copied
+        self.prepare_build('maven', state=True, started='2016-01-01')
+        old_build1 = self.prepare_build('maven', state=True, started=now)
+        new_build1 = self.prepare_build('maven', started=now)
+        copy = Collection(
+            name='copy', display_name='copy', target='a', build_tag='b',
+            dest_tag='c',
+        )
+        self.db.add(copy)
+        change1 = AppliedChange(
+            dep_name='foo', build_id=old_build1.id,
+            prev_version='1', prev_release='1',
+            curr_version='1', curr_release='2',
+        )
+        self.db.add(change1)
+        task1 = KojiTask(
+            build_id=new_build1.id,
+            task_id=new_build1.task_id,
+            state=1,
+            arch='x86_64',
+            started=new_build1.started,
+        )
+        self.db.add(task1)
+        rchange1 = ResolutionChange(
+            package_id=maven1.id,
+            resolved=False,
+            timestamp=now,
+        )
+        self.db.add(rchange1)
+        self.db.flush()
+        problem1 = ResolutionProblem(
+            resolution_id=rchange1.id,
+            problem="It's broken",
+        )
+        self.db.add(problem1)
+        self.db.commit()
+
+        data.copy_collection(self.session, source, copy)
+        self.db.commit()
+
+        maven2 = self.db.query(Package).filter_by(collection=copy, name='maven').first()
+        self.assertIsNotNone(maven2)
+        self.assertNotEqual(maven1.id, maven2.id)
+        self.assertEqual(source.id, maven1.collection_id)
+        self.assertEqual(copy.id, maven2.collection_id)
+        self.assertEqual('maven', maven2.name)
+
+        self.assertEqual(2, len(maven2.all_builds))
+        new_build2, old_build2 = maven2.all_builds
+        self.assertNotEqual(new_build1.id, new_build2.id)
+        self.assertNotEqual(old_build1.id, old_build2.id)
+        self.assertEqual(new_build1.id, maven1.last_build_id)
+        self.assertEqual(old_build1.id, maven1.last_complete_build_id)
+        self.assertEqual(new_build2.id, maven2.last_build_id)
+        self.assertEqual(old_build2.id, maven2.last_complete_build_id)
+        self.assertEqual(1, new_build2.build_arch_tasks[0].state)
+
+        self.assertEqual(1, len(old_build2.dependency_changes))
+        change2 = old_build2.dependency_changes[0]
+        self.assertEqual('2', change2.curr_release)
+
+        rchange2 = self.db.query(ResolutionChange).filter_by(package_id=maven2.id).one()
+        self.assertEqual("It's broken", rchange2.problems[0].problem)