diff --git a/.flake8 b/.flake8 index 2693eaa29b..67ba24e7b1 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -exclude = ./docs/*,*/migrations/* +exclude = ./docs/*,*/migrations/*,./pulpcore/app/protobuf/* ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 max-line-length = 100 diff --git a/pulpcore/app/protobuf/__init__.py b/pulpcore/app/protobuf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pulpcore/app/protobuf/telemetry_pb2.py b/pulpcore/app/protobuf/telemetry_pb2.py new file mode 100644 index 0000000000..50b61cab2d --- /dev/null +++ b/pulpcore/app/protobuf/telemetry_pb2.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: telemetry.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0ftelemetry.proto\"\xcb\x02\n\tTelemetry\x12\x11\n\tsystem_id\x18\x01 \x02(\t\x12\x39\n\x13online_content_apps\x18\x02 \x01(\x0b\x32\x1c.Telemetry.OnlineContentApps\x12\x30\n\x0eonline_workers\x18\x03 \x01(\x0b\x32\x18.Telemetry.OnlineWorkers\x12(\n\ncomponents\x18\x04 \x03(\x0b\x32\x14.Telemetry.Component\x1a\x35\n\x11OnlineContentApps\x12\x11\n\tprocesses\x18\x01 \x01(\r\x12\r\n\x05hosts\x18\x02 \x01(\r\x1a\x31\n\rOnlineWorkers\x12\x11\n\tprocesses\x18\x01 \x01(\r\x12\r\n\x05hosts\x18\x02 \x01(\r\x1a*\n\tComponent\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x0f\n\x07version\x18\x02 \x02(\t') + + + +_TELEMETRY = DESCRIPTOR.message_types_by_name['Telemetry'] +_TELEMETRY_ONLINECONTENTAPPS = _TELEMETRY.nested_types_by_name['OnlineContentApps'] +_TELEMETRY_ONLINEWORKERS = _TELEMETRY.nested_types_by_name['OnlineWorkers'] +_TELEMETRY_COMPONENT = _TELEMETRY.nested_types_by_name['Component'] +Telemetry = _reflection.GeneratedProtocolMessageType('Telemetry', (_message.Message,), { + + 'OnlineContentApps' : _reflection.GeneratedProtocolMessageType('OnlineContentApps', (_message.Message,), { + 'DESCRIPTOR' : _TELEMETRY_ONLINECONTENTAPPS, + '__module__' : 'telemetry_pb2' + # @@protoc_insertion_point(class_scope:Telemetry.OnlineContentApps) + }) + , + + 'OnlineWorkers' : _reflection.GeneratedProtocolMessageType('OnlineWorkers', (_message.Message,), { + 'DESCRIPTOR' : _TELEMETRY_ONLINEWORKERS, + '__module__' : 'telemetry_pb2' + # @@protoc_insertion_point(class_scope:Telemetry.OnlineWorkers) + }) + , + + 'Component' : _reflection.GeneratedProtocolMessageType('Component', (_message.Message,), { + 'DESCRIPTOR' : _TELEMETRY_COMPONENT, + '__module__' : 'telemetry_pb2' + # @@protoc_insertion_point(class_scope:Telemetry.Component) + }) + , + 'DESCRIPTOR' : _TELEMETRY, + '__module__' : 'telemetry_pb2' + # @@protoc_insertion_point(class_scope:Telemetry) + }) +_sym_db.RegisterMessage(Telemetry) +_sym_db.RegisterMessage(Telemetry.OnlineContentApps) +_sym_db.RegisterMessage(Telemetry.OnlineWorkers) +_sym_db.RegisterMessage(Telemetry.Component) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _TELEMETRY._serialized_start=20 + _TELEMETRY._serialized_end=351 + _TELEMETRY_ONLINECONTENTAPPS._serialized_start=203 + _TELEMETRY_ONLINECONTENTAPPS._serialized_end=256 + _TELEMETRY_ONLINEWORKERS._serialized_start=258 + _TELEMETRY_ONLINEWORKERS._serialized_end=307 + _TELEMETRY_COMPONENT._serialized_start=309 + _TELEMETRY_COMPONENT._serialized_end=351 +# @@protoc_insertion_point(module_scope) diff --git a/pulpcore/app/tasks/telemetry.py b/pulpcore/app/tasks/telemetry.py index 75a48d3f05..df929e29ee 100644 --- a/pulpcore/app/tasks/telemetry.py +++ b/pulpcore/app/tasks/telemetry.py @@ -1,16 +1,19 @@ import asyncio +import json import logging import aiohttp import async_timeout from asgiref.sync import sync_to_async +from google.protobuf.json_format import MessageToJson from pulpcore.app.apps import pulp_plugin_configs from pulpcore.app.util import get_telemetry_posting_url from pulpcore.app.models import SystemID from pulpcore.app.models.status import ContentAppStatus from pulpcore.app.models.task import Worker +from pulpcore.app.protobuf.telemetry_pb2 import Telemetry logger = logging.getLogger(__name__) @@ -24,70 +27,53 @@ async def _num_hosts(qs): return len(hosts) -async def _versions_data(): - versions = [] - +async def _versions_data(telemetry): for app in pulp_plugin_configs(): - versions.append({"component": app.label, "version": app.version}) - - return {"versions": versions} - + new_component = telemetry.components.add() + new_component.name = app.label + new_component.version = app.version -async def _online_content_apps_data(): - online_content_apps = ContentAppStatus.objects.online() - online_content_apps_processes = await sync_to_async(online_content_apps.count)() - online_content_apps_hosts = await _num_hosts(online_content_apps) - return { - "online_content_apps": { - "processes": online_content_apps_processes, - "hosts": online_content_apps_hosts, - }, - } +async def _online_content_apps_data(telemetry): + online_content_apps_qs = ContentAppStatus.objects.online() + telemetry.online_content_apps.processes = await sync_to_async(online_content_apps_qs.count)() + telemetry.online_content_apps.hosts = await _num_hosts(online_content_apps_qs) -async def _online_workers_data(): - online_workers = Worker.objects.online_workers() - online_workers_processes = await sync_to_async(online_workers.count)() - online_workers_hosts = await _num_hosts(online_workers) +async def _online_workers_data(telemetry): + online_workers_qs = Worker.objects.online_workers() + telemetry.online_workers.processes = await sync_to_async(online_workers_qs.count)() + telemetry.online_workers.hosts = await _num_hosts(online_workers_qs) - return { - "online_workers": { - "processes": online_workers_processes, - "hosts": online_workers_hosts, - }, - } - -async def _system_id(): - system_id_entry = await sync_to_async(SystemID.objects.get)() - return {"system_id": str(system_id_entry.pk)} +async def _system_id(telemetry): + system_id_obj = await sync_to_async(SystemID.objects.get)() + telemetry.system_id = str(system_id_obj.pk) async def post_telemetry(): url = get_telemetry_posting_url() - data = {} + + telemetry = Telemetry() awaitables = ( - _system_id(), - _versions_data(), - _online_content_apps_data(), - _online_workers_data(), + _system_id(telemetry), + _versions_data(telemetry), + _online_content_apps_data(telemetry), + _online_workers_data(telemetry), ) - data_iterable_to_merge = await asyncio.gather(*awaitables) - for data_to_merge in data_iterable_to_merge: - data.update(data_to_merge) + await asyncio.gather(*awaitables) try: async with aiohttp.ClientSession() as session: - async with async_timeout.timeout(30): - async with session.post(url, json=data) as resp: + async with async_timeout.timeout(300): + async with session.post(url, data=telemetry.SerializeToString()) as resp: if resp.status == 200: logger.info( ("Submitted telemetry to %s. " "Information submitted includes %s"), url, - data, + json.loads(MessageToJson(telemetry)), ) else: logger.warning( diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 6dbd45a1fc..5243bf4eab 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -19,8 +19,8 @@ # a little cache so viewset_for_model doesn't have iterate over every app every time _model_viewset_cache = {} -PRODUCTION_URL = "https://analytics-pulpproject-org.pulpproject.workers.dev/" -DEV_URL = "https://dev-analytics-pulpproject-org.pulpproject.workers.dev/" +PRODUCTION_URL = "https://analytics.pulpproject.org/" +DEV_URL = "https://dev.analytics.pulpproject.org/" def get_url(model): diff --git a/pyproject.toml b/pyproject.toml index 7471678911..84c7151812 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ exclude = ''' | dist | migrations | docs + | protobuf )/ ''' diff --git a/requirements.txt b/requirements.txt index 9a7b6457b8..d5a15d9fb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ dynaconf~=3.1.9 gunicorn~=20.1.0 jinja2~=3.1 naya~=1.1.1 +protobuf<3.21 pygtrie~=2.5.0 psycopg2~=2.9.3 PyYAML>=5.1.1,<6.1.0