Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bundle analysis: measurements optional "after" data and fetch asset gzip size from DB #663

Merged
merged 6 commits into from
Jul 9, 2024
Merged

bundle analysis: measurements optional "after" data and fetch asset gzip size from DB #663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6466c65
bundle analysis: measurements optional after date
JerrySentry Jul 5, 2024
53d5419
Merge branch 'main' into jul_04_cache_1
JerrySentry Jul 5, 2024
d843e89
Merge branch 'main' into jul_04_cache_1
JerrySentry Jul 8, 2024
4d195cb
Merge branch 'main' into jul_04_cache_1
JerrySentry Jul 8, 2024
73ce4e3
add fetch gzip info for assets
JerrySentry Jul 8, 2024
da95d7e
Merge branch 'main' into jul_04_cache_1
JerrySentry Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions graphql_api/actions/measurements.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,23 @@ def measurements_by_ids(
measurable_name: str,
measurable_ids: Iterable[str],
interval: Interval,
after: datetime,
before: datetime,
after: Optional[datetime] = None,
branch: Optional[str] = None,
) -> Mapping[int, Iterable[dict]]:
queryset = MeasurementSummary.agg_by(interval).filter(
name=measurable_name,
owner_id=repository.author_id,
repo_id=repository.pk,
measurable_id__in=measurable_ids,
timestamp_bin__gte=aligned_start_date(interval, after),
timestamp_bin__lte=before,
)

if after is not None:
queryset = queryset.filter(
timestamp_bin__gte=aligned_start_date(interval, after)
)

if branch:
queryset = queryset.filter(branch=branch)

Expand Down
215 changes: 215 additions & 0 deletions graphql_api/tests/test_bundle_analysis_measurements.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2207,3 +2207,218 @@ def test_bundle_report_branch(self, get_storage_service):
],
},
}

@patch("graphql_api.dataloader.bundle_analysis.get_appropriate_storage_service")
def test_bundle_report_no_after(self, get_storage_service):
measurements_data = [
# 2024-06-10
["bundle_analysis_report_size", "super", "2024-06-10T19:07:23", 123],
# 2024-06-06
["bundle_analysis_report_size", "super", "2024-06-06T19:07:23", 456],
]

for item in measurements_data:
MeasurementFactory(
name=item[0],
owner_id=self.org.pk,
repo_id=self.repo.pk,
branch="feat",
measurable_id=item[1],
commit_sha=self.commit.pk,
timestamp=item[2],
value=item[3],
)

storage = MemoryStorageService({})
get_storage_service.return_value = storage

with open("./services/tests/samples/bundle_with_uuid.sqlite", "rb") as f:
storage_path = StoragePaths.bundle_report.path(
repo_key=ArchiveService.get_archive_hash(self.repo),
report_key=self.head_commit_report.external_id,
)
storage.write_file(get_bucket_name(), storage_path, f)

query = """
query FetchMeasurements(
$org: String!,
$repo: String!,
$commit: String!
$filters: BundleAnalysisMeasurementsSetFilters
$orderingDirection: OrderingDirection!
$interval: MeasurementInterval!
$before: DateTime!
$after: DateTime
$branch: String!
) {
owner(username: $org) {
repository(name: $repo) {
... on Repository {
commit(id: $commit) {
bundleAnalysisReport {
__typename
... on BundleAnalysisReport {
bundle(name: "super") {
name
measurements(
filters: $filters
orderingDirection: $orderingDirection
after: $after
interval: $interval
before: $before
branch: $branch
){
assetType
name
size {
loadTime {
threeG
highSpeed
}
size {
gzip
uncompress
}
}
change {
loadTime {
threeG
highSpeed
}
size {
gzip
uncompress
}
}
measurements {
avg
min
max
timestamp
}
}
}
}
}
}
}
}
}
}
"""

variables = {
"org": self.org.username,
"repo": self.repo.name,
"commit": self.commit.commitid,
"orderingDirection": "ASC",
"interval": "INTERVAL_1_DAY",
"after": None,
"before": "2024-06-10",
"branch": "feat",
"filters": {},
}
data = self.gql_request(query, variables=variables)
commit = data["owner"]["repository"]["commit"]

assert commit["bundleAnalysisReport"] == {
"__typename": "BundleAnalysisReport",
"bundle": {
"name": "super",
"measurements": [
{
"assetType": "ASSET_SIZE",
"name": "asset-*.js",
"size": None,
"change": None,
"measurements": [],
},
{
"assetType": "ASSET_SIZE",
"name": "asset-*.js",
"size": None,
"change": None,
"measurements": [],
},
{
"assetType": "ASSET_SIZE",
"name": "asset-*.js",
"size": None,
"change": None,
"measurements": [],
},
{
"assetType": "FONT_SIZE",
"name": None,
"size": None,
"change": None,
"measurements": [],
},
{
"assetType": "IMAGE_SIZE",
"name": None,
"size": None,
"change": None,
"measurements": [],
},
{
"assetType": "JAVASCRIPT_SIZE",
"name": None,
"size": None,
"change": None,
"measurements": [],
},
{
"assetType": "REPORT_SIZE",
"name": None,
"size": {
"loadTime": {"threeG": 1, "highSpeed": 0},
"size": {"gzip": 0, "uncompress": 123},
},
"change": {
"loadTime": {"threeG": -3, "highSpeed": 0},
"size": {"gzip": 0, "uncompress": -333},
},
"measurements": [
{
"avg": 456.0,
"min": 456.0,
"max": 456.0,
"timestamp": "2024-06-06T00:00:00+00:00",
},
{
"avg": None,
"min": None,
"max": None,
"timestamp": "2024-06-07T00:00:00+00:00",
},
{
"avg": None,
"min": None,
"max": None,
"timestamp": "2024-06-08T00:00:00+00:00",
},
{
"avg": None,
"min": None,
"max": None,
"timestamp": "2024-06-09T00:00:00+00:00",
},
{
"avg": 123.0,
"min": 123.0,
"max": 123.0,
"timestamp": "2024-06-10T00:00:00+00:00",
},
],
},
{
"assetType": "STYLESHEET_SIZE",
"name": None,
"size": None,
"change": None,
"measurements": [],
},
],
},
}
4 changes: 2 additions & 2 deletions graphql_api/types/bundle_analysis/base.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ type BundleAsset {
measurements(
interval: MeasurementInterval!
before: DateTime!
after: DateTime!
after: DateTime
branch: String
): BundleAnalysisMeasurements
}
Expand All @@ -65,7 +65,7 @@ type BundleReport {
measurements(
interval: MeasurementInterval!
before: DateTime!
after: DateTime!
after: DateTime
branch: String
orderingDirection: OrderingDirection
filters: BundleAnalysisMeasurementsSetFilters
Expand Down
10 changes: 5 additions & 5 deletions graphql_api/types/bundle_analysis/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def resolve_extension(bundle_asset: AssetReport, info: GraphQLResolveInfo) -> st
def resolve_bundle_asset_bundle_data(
bundle_asset: AssetReport, info: GraphQLResolveInfo
) -> BundleData:
return BundleData(bundle_asset.size_total)
return BundleData(bundle_asset.size_total, bundle_asset.gzip_size_total)


@bundle_asset_bindable.field("modules")
Expand All @@ -101,14 +101,14 @@ def resolve_asset_report_measurements(
info: GraphQLResolveInfo,
interval: Interval,
before: datetime,
after: datetime,
after: Optional[datetime] = None,
branch: Optional[str] = None,
) -> Optional[BundleAnalysisMeasurementData]:
bundle_analysis_measurements = BundleAnalysisMeasurementsService(
repository=info.context["commit"].repository,
interval=interval,
after=after,
before=before,
after=after,
branch=branch,
)
return bundle_analysis_measurements.compute_asset(bundle_asset)
Expand Down Expand Up @@ -157,7 +157,7 @@ def resolve_bundle_report_measurements(
info: GraphQLResolveInfo,
interval: Interval,
before: datetime,
after: datetime,
after: Optional[datetime] = None,
branch: Optional[str] = None,
filters: Mapping = {},
ordering_direction: Optional[OrderingDirection] = OrderingDirection.ASC,
Expand All @@ -172,8 +172,8 @@ def resolve_bundle_report_measurements(
bundle_analysis_measurements = BundleAnalysisMeasurementsService(
repository=info.context["commit"].repository,
interval=interval,
after=after,
before=before,
after=after,
branch=branch,
)

Expand Down
2 changes: 1 addition & 1 deletion requirements.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ factory-boy
fakeredis
freezegun
https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
https://github.com/codecov/shared/archive/b3927cc7765e207030b25d5370bab9e56bf3e21a.tar.gz#egg=shared
https://github.com/codecov/shared/archive/b679996ed9df69753d8dd24cd8e1fc3817cb6b59.tar.gz#egg=shared
google-cloud-pubsub
gunicorn>=22.0.0
https://github.com/photocrowd/django-cursor-pagination/archive/f560902696b0c8509e4d95c10ba0d62700181d84.tar.gz
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ sentry-sdk[celery]==1.44.1
# shared
setproctitle==1.1.10
# via -r requirements.in
shared @ https://github.com/codecov/shared/archive/b3927cc7765e207030b25d5370bab9e56bf3e21a.tar.gz
shared @ https://github.com/codecov/shared/archive/b679996ed9df69753d8dd24cd8e1fc3817cb6b59.tar.gz
# via -r requirements.in
simplejson==3.17.2
# via -r requirements.in
Expand Down
18 changes: 14 additions & 4 deletions services/bundle_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,14 +157,20 @@ class BundleSize:

@dataclass
class BundleData:
def __init__(self, size_in_bytes: int):
def __init__(self, size_in_bytes: int, gzip_size_in_bytes: Optional[int] = None):
self.size_in_bytes = size_in_bytes
self.size_in_bits = size_in_bytes * 8
self.gzip_size_in_bytes = gzip_size_in_bytes

@cached_property
def size(self) -> BundleSize:
gzip_size = (
self.gzip_size_in_bytes
if self.gzip_size_in_bytes is not None
else int(float(self.size_in_bytes) * BundleSize.GZIP)
)
return BundleSize(
gzip=int(float(self.size_in_bytes) * BundleSize.GZIP),
gzip=gzip_size,
uncompress=int(float(self.size_in_bytes) * BundleSize.UNCOMPRESS),
)

Expand Down Expand Up @@ -216,6 +222,10 @@ def extension(self) -> str:
def size_total(self) -> int:
return self.asset.size

@cached_property
def gzip_size_total(self) -> int:
return self.asset.gzip_size

@cached_property
def modules(self) -> List[ModuleReport]:
return [ModuleReport(module) for module in self.asset.modules()]
Expand Down Expand Up @@ -361,8 +371,8 @@ def __init__(
self,
repository: Repository,
interval: Interval,
after: datetime,
before: datetime,
after: Optional[datetime] = None,
branch: Optional[str] = None,
) -> None:
self.repository = repository
Expand All @@ -386,7 +396,7 @@ def _compute_measurements(

# Carry over previous available value for start date if its value is null
for measurable_id, measurements in all_measurements.items():
if measurements[0]["timestamp_bin"] > self.after:
if self.after is not None and measurements[0]["timestamp_bin"] > self.after:
carryover_measurement = measurements_last_uploaded_before_start_date(
repo_id=self.repository.repoid,
measurable_name=measurable_name,
Expand Down
Loading