[Draft] Feature/open telemetry

Dockerfile

@@ -42,6 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 COPY --from=build /usr/local/lib/python3.7 /usr/local/lib/python3.7
 COPY --from=build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
+COPY --from=build /usr/local/bin/opentelemetry-instrument /usr/local/bin/opentelemetry-instrument
 
 # Stage 3.2 - Copy source code
 WORKDIR /app

bin/docker_start.sh

@@ -51,14 +51,14 @@ fi
 
 # Start server
 >&2 echo "Starting server"
-uwsgi \
+cd src/
+opentelemetry-instrument uwsgi \
     --http :$uwsgi_port \
     --http-keepalive \
     --manage-script-name \
     --mount $mountpoint=objects.wsgi:application \
     --static-map /static=/app/static \
     --static-map /media=/app/media  \
-    --chdir src \
     --enable-threads \
     --processes $uwsgi_processes \
     --threads $uwsgi_threads \

docs/installation/config.rst

@@ -43,6 +43,18 @@ Database settings
 
 * ``DB_PORT``: Port number of the database. Defaults to ``5432``.
 
+Elastic APM settings
+--------------------
+
+An integration with `Elastic APM <https://www.elastic.co/observability/application-performance-monitoring>`_
+can be configured by setting the following environment variables
+
+* ``ELASTIC_APM_SERVICE_NAME``: the name of the service in APM, i.e. "Objects API - staging"
+
+* ``ELASTIC_APM_SERVER_URL``: the URL of the APM instance to connect with
+
+* ``ELASTIC_APM_SECRET_TOKEN``: the token that is required to communicate with the APM instance
+
 Other settings
 --------------
 

requirements/base.in

@@ -31,7 +31,9 @@ drf-spectacular  # api documentation
 # WSGI servers & monitoring - production oriented
 uwsgi
 sentry-sdk  # error monitoring
-elastic-apm  # Elastic APM integration
+elastic-apm[opentelemetry]  # Elastic APM integration
+opentelemetry-sdk
+opentelemetry-instrumentation-django
 
 # Common ground libraries
 vng_api_common[markdown_docs]>=1.6.4

requirements/base.txt

@@ -32,6 +32,8 @@ cryptography==3.4.8
     #   josepy
     #   mozilla-django-oidc
     #   pyopenssl
+deprecated==1.2.13
+    # via opentelemetry-api
 django-admin-index==1.5.0
     # via -r requirements/base.in
 django-axes==5.14.0
@@ -125,7 +127,7 @@ drf-spectacular==0.16.0
     # via -r requirements/base.in
 drf-yasg==1.20.0
     # via vng-api-common
-elastic-apm==6.1.1
+elastic-apm[opentelemetry]==6.9.1
     # via -r requirements/base.in
 face==20.1.1
     # via glom
@@ -171,6 +173,34 @@ mozilla-django-oidc-db==0.7.2
     # via -r requirements/base.in
 mozilla-django-oidc==1.2.4
     # via mozilla-django-oidc-db
+opentelemetry-api==1.10.0
+    # via
+    #   elastic-apm
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-instrumentation-django==0.29b0
+    # via -r requirements/base.in
+opentelemetry-instrumentation-wsgi==0.29b0
+    # via opentelemetry-instrumentation-django
+opentelemetry-instrumentation==0.29b0
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+opentelemetry-sdk==1.10.0
+    # via
+    #   -r requirements/base.in
+    #   elastic-apm
+opentelemetry-semantic-conventions==0.29b0
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-util-http==0.29b0
+    # via
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
 oyaml==1.0
     # via vng-api-common
 packaging==20.9
@@ -254,7 +284,9 @@ sqlparse==0.4.2
 text-unidecode==1.3
     # via faker
 typing-extensions==3.10.0.2
-    # via importlib-metadata
+    # via
+    #   importlib-metadata
+    #   opentelemetry-sdk
 unidecode==1.2.0
     # via vng-api-common
 uritemplate==3.0.1
@@ -271,6 +303,10 @@ uwsgi==2.0.19.1
     # via -r requirements/base.in
 vng-api-common[markdown_docs]==1.6.4
     # via -r requirements/base.in
+wrapt==1.14.0
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
 zgw-consumers==0.15.2
     # via -r requirements/base.in
 zipp==3.5.0

requirements/ci.txt

@@ -49,6 +49,10 @@ cryptography==3.4.8
     #   pyopenssl
 cssselect==1.1.0
     # via pyquery
+deprecated==1.2.13
+    # via
+    #   -r requirements/base.txt
+    #   opentelemetry-api
 django-admin-index==1.5.0
     # via -r requirements/base.txt
 django-axes==5.14.0
@@ -177,7 +181,7 @@ drf-yasg==1.20.0
     # via
     #   -r requirements/base.txt
     #   vng-api-common
-elastic-apm==6.1.1
+elastic-apm[opentelemetry]==6.9.1
     # via -r requirements/base.txt
 face==20.1.1
     # via
@@ -255,6 +259,40 @@ mozilla-django-oidc==1.2.4
     # via
     #   -r requirements/base.txt
     #   mozilla-django-oidc-db
+opentelemetry-api==1.10.0
+    # via
+    #   -r requirements/base.txt
+    #   elastic-apm
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-instrumentation-django==0.29b0
+    # via -r requirements/base.txt
+opentelemetry-instrumentation-wsgi==0.29b0
+    # via
+    #   -r requirements/base.txt
+    #   opentelemetry-instrumentation-django
+opentelemetry-instrumentation==0.29b0
+    # via
+    #   -r requirements/base.txt
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+opentelemetry-sdk==1.10.0
+    # via
+    #   -r requirements/base.txt
+    #   elastic-apm
+opentelemetry-semantic-conventions==0.29b0
+    # via
+    #   -r requirements/base.txt
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-util-http==0.29b0
+    # via
+    #   -r requirements/base.txt
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
 oyaml==1.0
     # via
     #   -r requirements/base.txt
@@ -385,6 +423,7 @@ typing-extensions==3.10.0.2
     # via
     #   -r requirements/base.txt
     #   importlib-metadata
+    #   opentelemetry-sdk
 unidecode==1.2.0
     # via
     #   -r requirements/base.txt
@@ -411,6 +450,11 @@ webob==1.8.7
     # via webtest
 webtest==2.0.35
     # via django-webtest
+wrapt==1.14.0
+    # via
+    #   -r requirements/base.txt
+    #   deprecated
+    #   opentelemetry-instrumentation
 zgw-consumers==0.15.2
     # via -r requirements/base.txt
 zipp==3.5.0

requirements/dev.txt

@@ -71,6 +71,10 @@ cssselect==1.1.0
     # via
     #   -r requirements/ci.txt
     #   pyquery
+deprecated==1.2.13
+    # via
+    #   -r requirements/ci.txt
+    #   opentelemetry-api
 django-admin-index==1.5.0
     # via -r requirements/ci.txt
 django-axes==5.14.0
@@ -210,7 +214,7 @@ drf-yasg==1.20.0
     # via
     #   -r requirements/ci.txt
     #   vng-api-common
-elastic-apm==6.1.1
+elastic-apm[opentelemetry]==6.9.1
     # via -r requirements/ci.txt
 face==20.1.1
     # via
@@ -303,6 +307,40 @@ mozilla-django-oidc==1.2.4
     #   mozilla-django-oidc-db
 mypy-extensions==0.4.3
     # via black
+opentelemetry-api==1.10.0
+    # via
+    #   -r requirements/ci.txt
+    #   elastic-apm
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-instrumentation-django==0.29b0
+    # via -r requirements/ci.txt
+opentelemetry-instrumentation-wsgi==0.29b0
+    # via
+    #   -r requirements/ci.txt
+    #   opentelemetry-instrumentation-django
+opentelemetry-instrumentation==0.29b0
+    # via
+    #   -r requirements/ci.txt
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+opentelemetry-sdk==1.10.0
+    # via
+    #   -r requirements/ci.txt
+    #   elastic-apm
+opentelemetry-semantic-conventions==0.29b0
+    # via
+    #   -r requirements/ci.txt
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
+    #   opentelemetry-sdk
+opentelemetry-util-http==0.29b0
+    # via
+    #   -r requirements/ci.txt
+    #   opentelemetry-instrumentation-django
+    #   opentelemetry-instrumentation-wsgi
 oyaml==1.0
     # via
     #   -r requirements/ci.txt
@@ -487,6 +525,7 @@ typing-extensions==3.10.0.2
     #   -r requirements/ci.txt
     #   black
     #   importlib-metadata
+    #   opentelemetry-sdk
 unidecode==1.2.0
     # via
     #   -r requirements/ci.txt
@@ -519,6 +558,11 @@ webtest==2.0.35
     # via
     #   -r requirements/ci.txt
     #   django-webtest
+wrapt==1.14.0
+    # via
+    #   -r requirements/ci.txt
+    #   deprecated
+    #   opentelemetry-instrumentation
 zgw-consumers==0.15.2
     # via -r requirements/ci.txt
 zipp==3.5.0

src/manage.py

@@ -1,11 +1,16 @@
 #!/usr/bin/env python
 import sys
 
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+
 from objects.setup import setup_env
 
 if __name__ == "__main__":
     setup_env()
 
+    # This call is what makes the Django application be instrumented
+    DjangoInstrumentor().instrument()
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError:

src/objects/conf/base.py

@@ -388,13 +388,23 @@
         **SENTRY_CONFIG, integrations=SENTRY_SDK_INTEGRATIONS, send_default_pii=True
     )
 
+#
 # Elastic APM
-
+#
+ELASTIC_APM_SERVER_URL = os.getenv("ELASTIC_APM_SERVER_URL", None)
 ELASTIC_APM = {
-    "SERVICE_NAME": "objects",
+    "SERVICE_NAME": os.getenv("ELASTIC_APM_SERVICE_NAME", "Objects API"),
     "SECRET_TOKEN": os.getenv("ELASTIC_APM_SECRET_TOKEN", "default"),
-    "SERVER_URL": os.getenv("ELASTIC_APM_SERVER_URL", "http://example.com"),
+    "SERVER_URL": ELASTIC_APM_SERVER_URL,
 }
+if not ELASTIC_APM_SERVER_URL:
+    ELASTIC_APM["ENABLED"] = False
+    ELASTIC_APM["SERVER_URL"] = "http://localhost:8200"
+else:
+    MIDDLEWARE = ["elasticapm.contrib.django.middleware.TracingMiddleware"] + MIDDLEWARE
+    INSTALLED_APPS = INSTALLED_APPS + [
+        "elasticapm.contrib.django",
+    ]
 
 SITE_ID = os.getenv("SITE_ID", 1)
 

src/objects/utils/apps.py

@@ -1,9 +1,55 @@
 from django.apps import AppConfig
+import json
+# from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchSpanProcessor,
+    ConsoleSpanExporter,
+    SpanExportResult,
+    SpanExporter,
+)
+import sys
+from os import linesep
+from elasticapm.contrib.opentelemetry import trace
+
+# class CustomSpanExporter(SpanExporter):
+#     def __init__(
+#         self,
+#         service_name=None,
+#         out=sys.stdout,
+#         formatter=lambda span: span.to_json()
+#         + linesep,
+#     ):
+#         self.out = out
+#         self.formatter = formatter
+#         self.service_name = service_name
+
+#     def export(self, spans) -> SpanExportResult:
+#         for span in spans:
+#             data = json.loads(span.to_json())
+#             print(data)
+#             self.out.write(self.formatter(span))
+#         self.out.flush()
+#         return SpanExportResult.SUCCESS
 
 
 class UtilsConfig(AppConfig):
     name = "objects.utils"
 
     def ready(self):
+        from opentelemetry import trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import (
+            BatchSpanProcessor,
+            ConsoleSpanExporter,
+        )
+
         from . import checks  # noqa
         from . import oas_extensions  # noqa
+
+        provider = TracerProvider()
+        trace.set_tracer_provider(provider)
+
+        provider.add_span_processor(
+            BatchSpanProcessor(ConsoleSpanExporter(service_name="Objects API"))
+        )