diff --git a/monitoring/settings.py b/monitoring/settings.py index 29dd4b6..15dd118 100644 --- a/monitoring/settings.py +++ b/monitoring/settings.py @@ -76,6 +76,7 @@ 'rest_framework', 'monitoring.publishing', 'monitoring.availability', + 'monitoring.synchronisation', ] REST_FRAMEWORK = { diff --git a/monitoring/synchronisation/__init__.py b/monitoring/synchronisation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/synchronisation/admin.py b/monitoring/synchronisation/admin.py new file mode 100644 index 0000000..13be29d --- /dev/null +++ b/monitoring/synchronisation/admin.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.contrib import admin + +# Register your models here. diff --git a/monitoring/synchronisation/apps.py b/monitoring/synchronisation/apps.py new file mode 100644 index 0000000..55b3c3b --- /dev/null +++ b/monitoring/synchronisation/apps.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.apps import AppConfig + + +class SychronisationConfig(AppConfig): + name = 'sychronisation' diff --git a/monitoring/synchronisation/migrations/__init__.py b/monitoring/synchronisation/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/synchronisation/models.py b/monitoring/synchronisation/models.py new file mode 100644 index 0000000..c42dba8 --- /dev/null +++ b/monitoring/synchronisation/models.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models + + +class GridSiteSync(models.Model): + fetched = models.DateTimeField(auto_now=True) + site = models.CharField(max_length=255) + year = models.IntegerField() + month = models.IntegerField() + site_count = models.IntegerField() + repository_count = models.IntegerField() + difference = models.IntegerField() + + class Meta: + unique_together = ['site', 'year', 'month'] + + +class VSuperSummaries(models.Model): + Site = models.CharField(max_length=255) + Year = models.IntegerField() + Month = models.IntegerField() + NumberOfJobs = models.IntegerField() + + class Meta: + managed = False + db_table = 'VSuperSummaries' + unique_together = ('Site', 'Year', 'Month') + + +class VSyncRecords(models.Model): + Site = models.CharField(max_length=255) + Year = models.IntegerField() + Month = models.IntegerField() + NumberOfJobs = models.IntegerField() + + class Meta: + managed = False + db_table = 'VSyncRecords' + unique_together = ('Site', 'Year', 'Month') diff --git a/monitoring/synchronisation/serializers.py b/monitoring/synchronisation/serializers.py new file mode 100644 index 0000000..8e49835 --- /dev/null +++ b/monitoring/synchronisation/serializers.py @@ -0,0 +1,15 @@ +from rest_framework import serializers + +from monitoring.synchronisation.models import GridSiteSync + + +class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer): + # Override default format with None so that Python datetime is used as + # ouput format. Encoding will be determined by the renderer and can be + # formatted by a template filter. + updated = serializers.DateTimeField(format=None) + + class Meta: + model = GridSiteSync + fields = ('url', 'site', 'year', 'month', + 'site_count', 'repository_count', 'difference') diff --git a/monitoring/synchronisation/tests.py b/monitoring/synchronisation/tests.py new file mode 100644 index 0000000..5982e6b --- /dev/null +++ b/monitoring/synchronisation/tests.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.test import TestCase + +# Create your tests here. diff --git a/monitoring/synchronisation/urls.py b/monitoring/synchronisation/urls.py new file mode 100644 index 0000000..0128703 --- /dev/null +++ b/monitoring/synchronisation/urls.py @@ -0,0 +1,13 @@ +from django.conf.urls import include, url + +from rest_framework import routers + +from monitoring.synchronisation import views + +router = routers.SimpleRouter() +router.register(r'grid', views.GridSiteSyncViewSet) + + +urlpatterns = [ + url(r'^', include(router.urls)), +] diff --git a/monitoring/synchronisation/views.py b/monitoring/synchronisation/views.py new file mode 100644 index 0000000..5a2fecc --- /dev/null +++ b/monitoring/synchronisation/views.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from datetime import datetime, timedelta + +from django.db.models import Max + +from rest_framework import viewsets +from rest_framework.renderers import TemplateHTMLRenderer + +from monitoring.synchronisation.models import GridSiteSync, VSuperSummaries, VSyncRecords +from monitoring.synchronisation.serializers import GridSiteSyncSerializer + + +class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet): + queryset = GridSiteSync.objects.all() + serializer_class = GridSiteSyncSerializer + template_name = 'gridsites.html' + + def retrieve(self, request, pk=None): + last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max'] + # If there's no data then last_fetched is None. + if last_fetched is not None: + print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20)) + if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)): + print('Out of date') + fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;") + for f in fetchset: + GridSiteSync.objects.update_or_create(defaults={'updated': f.LatestPublish}, name=f.Site) + else: + print('No need to update') + + response = super(GridSiteSyncViewSet, self).retrieve(request) + date = response.data['updated'].replace(tzinfo=None) + + # Wrap data in a dict so that it can display in template. + if type(request.accepted_renderer) is TemplateHTMLRenderer: + # Single result put in list to work with same HTML template. + response.data = {'sites': [response.data], 'last_fetched': last_fetched} + + diff = datetime.today() - date + if diff <= timedelta(days=7): + response.data['returncode'] = 0 + response.data['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) + elif diff > timedelta(days=7): + response.data['returncode'] = 1 + response.data['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d")) + else: + response.data['returncode'] = 3 + response.data['stdout'] = "UNKNOWN" + + return response diff --git a/monitoring/urls.py b/monitoring/urls.py index 26e4055..72c2af2 100644 --- a/monitoring/urls.py +++ b/monitoring/urls.py @@ -20,6 +20,7 @@ url(r'^admin/', admin.site.urls), url(r'^availability/', include('monitoring.availability.urls')), url(r'^publishing/', include('monitoring.publishing.urls')), + url(r'^synchronisation/', include('monitoring.synchronisation.urls')), url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), ]