Skip to content

Commit

Permalink
Use artefact builds for promotion
Browse files Browse the repository at this point in the history
  • Loading branch information
@nadzyah
nadzyah committed Jun 9, 2023
1 parent 8f2eaef commit 57f7623
Show file tree
Hide file tree
Showing 6 changed files with 238 additions and 106 deletions.
158 changes: 86 additions & 72 deletions backend/test_observer/controllers/artefacts/artefacts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,14 @@
from test_observer.data_access.repository import (
get_stage_by_name,
get_artefacts_by_family_name,
get_latest_builds_for_artefact,
)
from test_observer.data_access.models import Artefact
from test_observer.data_access.models_enums import FamilyName
from test_observer.data_access.setup import get_db
from test_observer.external_apis.snapcraft import get_channel_map_from_snapcraft
from test_observer.external_apis.snapcraft import (
get_channel_map_from_snapcraft,
)
from test_observer.external_apis.archive import ArchiveManager

router = APIRouter()
Expand Down Expand Up @@ -115,53 +118,56 @@ def run_snap_promoter(session: Session, artefact: Artefact) -> None:
Check snap artefacts state and move/archive them if necessary
:session: DB connection session
:artefact: an Artefact object
:artefact_build: an ArtefactBuild object
"""
arch = artefact.source["architecture"]
channel_map = get_channel_map_from_snapcraft(
arch=arch,
snapstore=artefact.source["store"],
snap_name=artefact.name,
)
track = artefact.source.get("track", "latest")

for channel_info in channel_map:
if not (
channel_info.channel.track == track
and channel_info.channel.architecture == arch
):
continue

risk = channel_info.channel.risk
try:
version = channel_info.version
revision = channel_info.revision
except KeyError as exc:
logger.warning(
"No key '%s' is found. Continue processing...",
str(exc),
)
continue

# If the snap with this name in this channel is a
# different revision, then this is old. So, we archive it
if risk == artefact.stage.name and revision != artefact.source["revision"]:
continue

next_risk = CHANNEL_PROMOTION_MAP[artefact.stage.name]
if (
risk == next_risk != artefact.stage.name.lower()
and version == artefact.version
and revision == artefact.source["revision"]
):
logger.info("Move artefact '%s' to the '%s' stage", artefact, next_risk)
stage = get_stage_by_name(
session, stage_name=next_risk, family=artefact.stage.family
)
if stage:
artefact.stage = stage
session.commit()
break
latest_builds = get_latest_builds_for_artefact(session, artefact)
for build in latest_builds:
arch = build.architecture
channel_map = get_channel_map_from_snapcraft(
arch=arch,
snapstore=artefact.source["store"],
snap_name=artefact.name,
)
track = artefact.source.get("track", "latest")

for channel_info in channel_map:
if not (
channel_info.channel.track == track
and channel_info.channel.architecture == arch
):
continue

risk = channel_info.channel.risk
try:
version = channel_info.version
revision = channel_info.revision
except KeyError as exc:
logger.warning(
"No key '%s' is found. Continue processing...",
str(exc),
)
continue

# If the snap with this name in this channel is a
# different revision, then this is old. So, we archive it
if risk == artefact.stage.name and revision != build.revision:
continue

next_risk = CHANNEL_PROMOTION_MAP[artefact.stage.name]
if (
risk == next_risk != artefact.stage.name.lower()
and version == artefact.version
and revision == build.revision
):
logger.info("Move artefact '%s' to the '%s' stage", artefact, next_risk)
stage = get_stage_by_name(
session, stage_name=next_risk, family=artefact.stage.family
)
if stage:
artefact.stage = stage
session.commit()
# The artefact was promoted, so we're done
return


def run_deb_promoter(session: Session, artefact: Artefact) -> None:
Expand All @@ -171,29 +177,37 @@ def run_deb_promoter(session: Session, artefact: Artefact) -> None:
:session: DB connection session
:artefact: an Artefact object
"""
arch = artefact.source["architecture"]
for repo in REPOSITORY_PROMOTION_MAP:
with ArchiveManager(
arch=arch,
series=artefact.source["series"],
pocket=repo,
apt_repo=artefact.source["repo"],
) as archivemanager:
deb_version = archivemanager.get_deb_version(artefact.name)
if deb_version is None:
logger.error(
"Cannot find deb_version with deb %s in package data",
artefact.name,
)
continue
next_repo = REPOSITORY_PROMOTION_MAP.get(artefact.stage.name)
logger.debug(
"Artefact version: %s, deb version: %s", artefact.version, deb_version
)
if repo == next_repo != artefact.stage.name and deb_version == artefact.version:
logger.info("Move artefact '%s' to the '%s' stage", artefact, next_repo)
artefact.stage = get_stage_by_name(
session, stage_name=next_repo, family=artefact.stage.family
latest_builds = get_latest_builds_for_artefact(session, artefact)
for build in latest_builds:
arch = build.architecture
for repo in REPOSITORY_PROMOTION_MAP:
with ArchiveManager(
arch=arch,
series=artefact.source["series"],
pocket=repo,
apt_repo=artefact.source["repo"],
) as archivemanager:
deb_version = archivemanager.get_deb_version(artefact.name)
if deb_version is None:
logger.error(
"Cannot find deb_version with deb %s in package data",
artefact.name,
)
continue
next_repo = REPOSITORY_PROMOTION_MAP.get(artefact.stage.name)
logger.debug(
"Artefact version: %s, deb version: %s", artefact.version, deb_version
)
session.commit()
break
if (
repo == next_repo != artefact.stage.name
and deb_version == artefact.version
):
logger.info("Move artefact '%s' to the '%s' stage", artefact, next_repo)
stage = get_stage_by_name(
session, stage_name=next_repo, family=artefact.stage.family
)
if stage:
artefact.stage = stage
session.commit()
# The artefact was promoted, so we're done
return
46 changes: 44 additions & 2 deletions backend/test_observer/data_access/repository.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@
# Omar Selo <omar.selo@canonical.com>
"""Services for working with objects from DB"""


from sqlalchemy import func, and_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import joinedload, Session
from sqlalchemy.orm import joinedload, aliased, Session

from .models_enums import FamilyName
from .models import DataModel, Family, Stage, Artefact
from .models import DataModel, Family, Stage, Artefact, ArtefactBuild


def get_stage_by_name(
Expand Down Expand Up @@ -65,6 +67,46 @@ def get_artefacts_by_family_name(
return artefacts


def get_latest_builds_for_artefact(
session: Session, artefact: Artefact
) -> list[ArtefactBuild]:
"""
Get the latest artefact build for each architecture for a given artefact.
:session: DB session
:artefact: The artefact for which to get the latest builds
:return: list of latest ArtefactBuilds for each architecture
"""
ab_alias = aliased(ArtefactBuild)

# Get the latest created_at for each architecture
subquery = (
session.query(
ab_alias.architecture, func.max(ab_alias.created_at).label("max_created_at")
)
.filter(ab_alias.artefact_id == artefact.id)
.group_by(ab_alias.architecture)
.subquery()
)

# Join with the subquery on architecture and created_at to get the latest builds
latest_builds = (
session.query(ArtefactBuild)
.join(
subquery,
and_(
ArtefactBuild.architecture == subquery.c.architecture,
ArtefactBuild.created_at == subquery.c.max_created_at,
),
)
.filter(ArtefactBuild.artefact_id == artefact.id)
.order_by(ArtefactBuild.architecture)
.all()
)

return latest_builds


def get_or_create(db: Session, model: type[DataModel], **kwargs) -> DataModel:
"""
Creates an object if it doesn't exist, otherwise returns the existing one
Expand Down
12 changes: 2 additions & 10 deletions backend/test_observer/external_apis/archive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,11 @@


import os
from io import StringIO
import re
import gzip
import random
import string
import tempfile
from urllib.error import HTTPError
import requests
from types import TracebackType
import logging
Expand Down Expand Up @@ -71,7 +69,7 @@ def __exit__(
os.remove(self.gz_filepath)
os.remove(self.decompressed_filepath)

def get_deb_version(self, debname: str) -> str:
def get_deb_version(self, debname: str) -> str | None:
"""
Convert Packages file from archive to json and get version from it
This function corresponds the method used by jenkins from the
Expand Down Expand Up @@ -120,13 +118,7 @@ def _download_data(self) -> None:
"""Download Packages.gz file from archive"""
response = requests.get(self.url, stream=True)
if not response.ok:
raise HTTPError(
self.url,
code=response.status_code,
msg=f"Cannot retrieve file from {self.url}",
hdrs={},
fp=StringIO(),
)
response.raise_for_status()

with open(self.gz_filepath, "wb") as file:
file.write(response.content)
Expand Down
49 changes: 29 additions & 20 deletions backend/tests/controllers/artefacts/test_artefacts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@
from requests_mock import Mocker
from sqlalchemy.orm import Session

from ...helpers import create_artefact
from test_observer.data_access.repository import get_latest_builds_for_artefact

from ...helpers import create_artefact, create_artefact_builds


def test_run_to_move_artefact_snap(
Expand All @@ -35,13 +37,22 @@ def test_run_to_move_artefact_snap(
snapcraft, the artefact is moved to the next stage
"""
# Arrange
artefact = create_artefact(
db_session,
"edge",
name="core20",
version="1.1.1",
source={"revision": 1883, "store": "ubuntu"},
)
create_artefact_builds(db_session, artefact)
latest_build = get_latest_builds_for_artefact(db_session, artefact)[1]
requests_mock.get(
"https://api.snapcraft.io/v2/snaps/info/core20",
json={
"channel-map": [
{
"channel": {
"architecture": "amd64",
"architecture": latest_build.architecture,
"name": "beta",
"released-at": "2023-05-17T12:39:07.471800+00:00",
"risk": "beta",
Expand All @@ -54,22 +65,14 @@ def test_run_to_move_artefact_snap(
"size": 130830336,
"url": "https://api.snapcraft.io/api/v1/snaps/download/...",
},
"revision": 1883,
"revision": latest_build.revision,
"type": "app",
"version": "1.1.1",
},
]
},
)

artefact = create_artefact(
db_session,
"edge",
name="core20",
version="1.1.1",
source={"revision": 1883, "architecture": "amd64", "store": "ubuntu"},
)

# Act
test_client.put("/v0/artefacts/promote")

Expand All @@ -92,20 +95,26 @@ def test_run_to_move_artefact_deb(
"proposed",
name="linux-generic",
version="5.19.0.43.39",
source={"series": "kinetic", "repo": "main", "architecture": "amd64"},
source={"series": "kinetic", "repo": "main"},
)
create_artefact_builds(db_session, artefact)

with open("tests/test_data/Packages-proposed.gz", "rb") as f:
proposed_content = f.read()
requests_mock.get(
"http://us.archive.ubuntu.com/ubuntu/dists/kinetic-proposed/main/binary-amd64/Packages.gz",
content=proposed_content,
)
with open("tests/test_data/Packages-updates.gz", "rb") as f:
updates_content = f.read()
requests_mock.get(
"http://us.archive.ubuntu.com/ubuntu/dists/kinetic-updates/main/binary-amd64/Packages.gz",
content=updates_content,
)

for build in get_latest_builds_for_artefact(db_session, artefact):
requests_mock.get(
"http://us.archive.ubuntu.com/ubuntu/dists/kinetic-proposed/main/"
f"binary-{build.architecture}/Packages.gz",
content=proposed_content,
)
requests_mock.get(
"http://us.archive.ubuntu.com/ubuntu/dists/kinetic-updates/main/"
f"binary-{build.architecture}/Packages.gz",
content=updates_content,
)

# Act
test_client.put("/v0/artefacts/promote")
Expand Down
Loading

0 comments on commit 57f7623

Please sign in to comment.