Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bug-1906113: remove eliot references #2989

Merged
merged 3 commits into from
Aug 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@

# General information about the project.
project = "Tecken: Symbols at Mozilla"
copyright = "2016-2022 Mozilla Foundation"
author = "Tecken team"
copyright = "2016-2024 Mozilla Foundation"
author = "Observability Team"

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
Expand Down
2 changes: 1 addition & 1 deletion docs/exts/document_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""Generates the documentation for Eliot metrics."""
"""Generates the documentation for metrics."""

import importlib
import sys
Expand Down
9 changes: 6 additions & 3 deletions docs/metrics.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
Metrics
=======

Metrics in Tecken
=================
StatsD Metrics in Tecken
========================

.. autometrics:: tecken.libmarkus.TECKEN_METRICS
Tecken uses `StatsD <https://github.com/statsd/statsd>`__ with
`DogStatsD extensions <https://docs.datadoghq.com/developers/dogstatsd/?tab=hostagent>`__.

.. autometrics:: tecken.libmarkus.STATSD_METRICS
4 changes: 2 additions & 2 deletions tecken/apps.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from django.apps import AppConfig

from tecken.libdockerflow import get_release_name
from tecken.libmarkus import METRICS, set_up_markus
from tecken.libmarkus import METRICS, set_up_metrics


logger = logging.getLogger("django")
Expand Down Expand Up @@ -104,7 +104,7 @@ def ready(self):

@staticmethod
def _configure_markus():
set_up_markus(
set_up_metrics(
backends=settings.MARKUS_BACKENDS,
hostname=settings.HOSTNAME,
debug=settings.LOCAL_DEV_ENV or settings.TEST_ENV,
Expand Down
68 changes: 11 additions & 57 deletions tecken/libmarkus.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,39 +2,34 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

import json
from pathlib import Path
import time
from functools import partialmethod

import markus
from markus import INCR, GAUGE, HISTOGRAM, TIMING
from markus.backends import BackendBase
from markus.filters import AddTagFilter, RegisteredMetricsFilter
import yaml


_IS_MARKUS_SETUP = False
_IS_MARKUS_SET_UP = False

METRICS = markus.get_metrics("tecken")


# Complete index of all Eliot metrics. This is used in documentation and to filter
# outgoing metrics.
# Complete index of all metrics. This is used in documentation and to filter outgoing
# metrics.
def _load_registered_metrics():
# Load the eliot_metrics.yaml file in this directory
path = Path(__file__).parent / "tecken_metrics.yaml"
# Load the metrics yaml file in this directory
path = Path(__file__).parent / "statsd_metrics.yaml"
with open(path) as fp:
data = yaml.safe_load(fp)
return data


TECKEN_METRICS = _load_registered_metrics()
STATSD_METRICS = _load_registered_metrics()


def set_up_markus(backends, hostname, debug=False):
global _IS_MARKUS_SETUP, METRICS
if _IS_MARKUS_SETUP:
def set_up_metrics(backends, hostname, debug=False):
global _IS_MARKUS_SET_UP, METRICS
if _IS_MARKUS_SET_UP:
return

markus.configure(backends)
Expand All @@ -43,53 +38,12 @@ def set_up_markus(backends, hostname, debug=False):
# In local dev and test environments, we want the RegisteredMetricsFilter to
# raise exceptions when metrics are used incorrectly.
metrics_filter = RegisteredMetricsFilter(
registered_metrics=TECKEN_METRICS, raise_error=True
registered_metrics=STATSD_METRICS, raise_error=True
)
METRICS.filters.append(metrics_filter)

if hostname:
# Define host tag here instead of in the backend so it shows up in tests
METRICS.filters.append(AddTagFilter(f"host:{hostname}"))

_IS_MARKUS_SETUP = True


class LogAllMetricsKeys(BackendBase): # pragma: no cover
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't need this since we only allow documented metrics to be used via the RegisteredMetricsFilter.

"""A markus backend that uses the filesystem to jot down ALL keys
that ever get used.
This then becomes handy when you want to make sure that you're using
all metrics in places like Datadog.
If you're not, it's maybe time to delete the metrics key.

Don't enable this in production. Use it during local development
and/or local test running.
"""

def notice_use(self, state, type, *args, **kwargs):
filename = "all-metrics-keys.json"
try:
with open(filename) as f:
all_keys = json.load(f)
except (FileNotFoundError, json.decoder.JSONDecodeError):
# This file gets written in an not thread-safe way
# so sometimes the file is all messed up. Pretend
# it didn't exist if the exception was a JSONDecodeError.
all_keys = {
"_documentation": (
"This file was created so you can see all metrics "
"keys that get used. It won't delete keys that are no "
"longer used. Feel free to delete this file and run again."
)
}
all_keys[state] = {
"type": type,
"timestamp": time.time(),
"count": all_keys.get(state, {}).get("count", 0) + 1,
}
with open(filename, "w") as f:
json.dump(all_keys, f, sort_keys=True, indent=3)

incr = partialmethod(notice_use, type=INCR)
gauge = partialmethod(notice_use, type=GAUGE)
histogram = partialmethod(notice_use, type=HISTOGRAM)
timing = partialmethod(notice_use, type=TIMING)
_IS_MARKUS_SET_UP = True
File renamed without changes.
10 changes: 10 additions & 0 deletions tecken/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,25 @@
import pytest
import requests_mock

from django.conf import settings
from django.contrib.auth.models import Group
from django.core.cache import caches

from tecken.base.symbolstorage import SymbolStorage
from tecken.ext.gcs.storage import GCSStorage
from tecken.ext.s3.storage import S3Storage
from tecken.libmarkus import set_up_metrics
from tecken.libstorage import StorageBackend


def pytest_sessionstart(session):
set_up_metrics(
backends=[{"class": "markus.backends.logging.LoggingMetrics"}],
hostname=settings.HOSTNAME,
debug=True,
)


@pytest.fixture(autouse=True)
def clear_cache():
caches["default"].clear()
Expand Down