Semantic convention generator

.github/workflows/protobuf-dockerimage.yml

@@ -1,5 +1,5 @@
 name: Protobuf Docker Image
-on: 
+on:
   push:
     tags: [ '**' ]
     branches: [ master ]
@@ -9,7 +9,7 @@ on:
       - .github/workflows/protobuf-dockerimage.yml
       - protobuf/Dockerfile
       - protobuf/protoc-wrapper
-          
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/semcongen-dockerimage.yml

@@ -0,0 +1,33 @@
+name: Semantic Convention Generator Docker Image
+on:
+  push:
+    tags: [ '**' ]
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+    paths:
+      - .github/workflows/semconvgen-dockerimage.yml
+      - 'semantic-conventions/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Build the Docker image
+      run: docker build semantic-conventions/. -t semconvgen
+    - name: Push the Docker image
+      if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
+      run: |
+        echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+        function tag_and_push {
+          docker tag semconvgen "otel/semconvgen:${1}" && docker push "otel/semconvgen:${1}"
+        }
+        if [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then
+          tag_and_push "latest"
+        elif [[ "${GITHUB_REF}" =~ refs/tags/v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            TAG="${GITHUB_REF#"refs/tags/v"}"
+            tag_and_push "${TAG}"
+        else
+          tag_and_push "${GITHUB_REF#"refs/tags/"}"
+        fi

README.md

@@ -1,3 +1,4 @@
 # Build Tools
 
 * [Protobuf](./protobuf/README.md)
+* [Semantic Convention Generator](./semantic-conventions/README.md)

semantic-conventions/.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

semantic-conventions/Dockerfile

@@ -0,0 +1,14 @@
+ARG ALPINE_VERSION=3.12
+ARG PYTHON_VERSION=3.8.5
+
+FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
+LABEL maintainer="The OpenTelemetry Authors"
+ADD . /semconvgen/
+WORKDIR semconvgen
+RUN rm -f README.md
+RUN apk --update add --virtual build-dependencies build-base \
+  && pip install -r requirements.txt \
+  && apk del build-dependencies
+
+RUN python setup.py install
+ENTRYPOINT ["gen-semconv"]

semantic-conventions/README.md

@@ -0,0 +1,98 @@
+# Semantic Convention generator + Docker
+
+A docker image to process Semantic Convention YAML models.
+
+
+# Usage
+
+The image can be used to generate Markdown tables or code.
+
+```bash
+docker run --rm -v<yaml-path>:<some-path> -v<output-path>:<some-path> otel/semconvgen [OPTION]
+```
+
+For help try:
+
+```bash
+docker run --rm otel/semconvgen -h
+```
+
+## Markdown Tables
+
+Tables can be generated using the command:
+
+```bash
+docker run --rm otel/semconvgen --yaml-root {yaml_folder} markdown --markdown-root {markdown_folder}
+```
+
+Where `{yaml_folder}` is the absolute path to the directory containing the yaml files and
+`{markdown_folder}` the absolute path to the directory containing the markdown definitions
+(`specification` for [opentelemetry-specification](https://github.com/open-telemetry/opentelemetry-specification/tree/master/)).
+
+The tool will automatically replace the tables with the up to date definition of the semantic conventions.
+To do so, the tool looks for special tags in the markdown.
+
+```
+<!-- semconv {semantic_convention_id} -->
+<!-- endsemconv -->
+```
+
+Everything between these two tags will be replaced with the table definition.
+The `{semantic_convention_id}` MUST be the `id` field in the yaml files of the semantic convention
+for which we want to generate the table.
+After `{semantic_convention_id}`, optional parameters enclosed in parentheses can be added to customize the output:
+
+- `tag={tag}`: prints only the attributes that have `{tag}` as a tag;
+- `full`: prints attributes and constraints inherited from the parent semantic conventions or from included ones;
+- `ref`: prints attributes that are referenced from another semantic convention;
+- `remove_constraint`: does not print additional constraints of the semantic convention.
+
+### Examples
+
+These examples assume that a semantic convention with the id `http.server` extends another semantic convention with the id `http`.
+
+`<!-- semconv http.server -->` will print only the attributes and constraints of the `http.server` semantic
+convention.
+
+`<!-- semconv http.server(full) -->` will print the attributes and constraints of the `http` semantic
+convention and also the attributes and constraints of the `http.server` semantic convention.
+
+`<!-- semconv http.server() -->` is equivalent to `<!-- semconv http.server -->`.
+
+`<!-- semconv http.server(tag=network) -->` will print the constraints and attributes of the `http.server` semantic
+convention that have the tag `network`.
+
+`<!-- semconv http.server(tag=network, full) -->` will print the constraints and attributes of both `http` and `http.server`
+semantic conventions that have the tag `network`.
+
+## Code Generator
+
+The image supports [Jinja](https://jinja.palletsprojects.com/en/2.11.x/) templates to generate code from the models.
+
+For example, the following template is used by the [opentelemetry-java-instrumentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/master/library-api/src/main/java/io/opentelemetry/instrumentation/api/typedspan) to generate Java classes.
+[Template Link.](https://gist.github.com/thisthat/7e34742f4a7f1b5df57118f859a19c3b)
+
+The image can generate code with the following command:
+
+```bash
+docker run --rm otel/semconvgen --yaml-root {yaml_folder} code --template {jinja-file} --output {output-file}
+```
+
+By default, all models are fed into the specified template at once, i.e. only a single file is generated.
+This is helpful to generate constants for the semantic attributes, [example from opentelemetry-java](https://github.com/open-telemetry/opentelemetry-java/blob/master/api/src/main/java/io/opentelemetry/trace/attributes/SemanticAttributes.java).
+
+If the parameter `--file-per-group {pattern}` is set, a single yaml model is fed into the template
+and the value of `pattern` is resolved from the model and attached as prefix to the output argument.
+This way, multiple files are generated. The value of `pattern` can be one of the following:
+- `semconv_id`: The id of the semantic convention.
+- `prefix`: The prefix with which all attributes starts with.
+- `extends`: The id of the parent semantic convention.
+
+Finally, additional value can be passed to the template in form of `key=value` pairs separated by
+comma using the `--parameters [{key=value},]+` flag.
+
+For example, to generate the typed spans used by the [opentelemetry-java-instrumentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/master/library-api/src/main/java/io/opentelemetry/instrumentation/api/typedspan), the following command can be used:
+
+```bash
+docker run --rm otel/semconvgen --yaml-root ${yamls} code --template typed_span_class.java.j2 --file-per-group semconv_id -o ${output}/Span.java
+```

semantic-conventions/requirements.txt

@@ -0,0 +1,9 @@
+black==19.10b0
+dataclasses==0.6
+mypy==0.770
+mypy-extensions==0.4.3
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.0
+typed-ast==1.4.1
+typing-extensions==3.7.4.2
+Jinja2==2.11.2

semantic-conventions/setup.cfg

@@ -0,0 +1,36 @@
+# Copyright 2019, Dynatrace
+
+[metadata]
+name = semconvgen
+description = Dynatrace Semantic Convention utility
+author = Dynatrace
+author_email = giovanni.liva@dynatrace.com
+url = https://www.dynatrace.com/
+platforms = any
+classifiers =
+    Development Status :: 1 - Alpha
+    Intended Audience :: Developers
+    Programming Language :: Python :: 3.6
+
+[options]
+python_requires = >=3.6
+package_dir=
+    =src
+packages=find_namespace:
+install_requires =
+    black==19.10b0
+    dataclasses==0.6
+    mypy==0.770
+    mypy-extensions==0.4.3
+    ruamel.yaml==0.16.10
+    ruamel.yaml.clib==0.2.0
+    typed-ast==1.4.1
+    typing-extensions==3.7.4.2
+    Jinja2==2.11.2
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+console_scripts =
+    gen-semconv = opentelemetry.semconv.main:main

semantic-conventions/setup.py

@@ -0,0 +1,19 @@
+import os
+
+import setuptools
+
+BASE_DIR = os.path.dirname(__file__)
+VERSION_FILENAME = os.path.join(BASE_DIR, "src", "opentelemetry", "semconv", "version.py")
+PACKAGE_INFO = {}
+with open(VERSION_FILENAME) as f:
+    exec(f.read(), PACKAGE_INFO)
+
+
+VERSION_SUFFIX = os.environ.get("SEMCONGEN_VERSION_SUFFIX")
+PUBLIC_VERSION = PACKAGE_INFO["__version__"]
+
+setuptools.setup(
+    version=PUBLIC_VERSION
+    if not VERSION_SUFFIX
+    else PUBLIC_VERSION + "+" + VERSION_SUFFIX
+)