diff --git a/cove_ofds/process.py b/cove_ofds/process.py index 8e3ff19..ebd9dba 100644 --- a/cove_ofds/process.py +++ b/cove_ofds/process.py @@ -10,6 +10,7 @@ from libcoveweb2.models import SuppliedDataFile from libcoveweb2.process import ProcessDataTask +from libcoveweb2.utils import group_data_list_by class WasJSONUploaded(ProcessDataTask): @@ -317,5 +318,9 @@ def get_context(self): with open(self.checks_and_stats_filename) as fp: context.update(json.load(fp)) context["additional_checks_count"] = len(context["additional_checks"]) + context["additional_checks"] = group_data_list_by( + context["additional_checks"], lambda i: i["type"] + ) + print(context["additional_checks"]) # done! return context diff --git a/cove_ofds/templates/cove_ofds/additional_checks_table.html b/cove_ofds/templates/cove_ofds/additional_checks_table.html index ab3c709..ed406cc 100644 --- a/cove_ofds/templates/cove_ofds/additional_checks_table.html +++ b/cove_ofds/templates/cove_ofds/additional_checks_table.html @@ -1,409 +1,687 @@ {% load i18n %} - - - - - - - - - - - {% for additional_check in additional_checks %} - {% if additional_check.type == 'span_start_node_not_found' %} - - - - - - - {% elif additional_check.type == 'span_end_node_not_found' %} - - - - - - - {% elif additional_check.type == 'node_location_type_incorrect' %} - - - - - - - {% elif additional_check.type == 'node_location_coordinates_incorrect' %} - - - - - - - {% elif additional_check.type == 'span_route_type_incorrect' %} - - - - - - - {% elif additional_check.type == 'span_route_coordinates_incorrect' %} - - - - - - - {% elif additional_check.type == 'node_phase_reference_id_not_found' %} - - - - - - - {% elif additional_check.type == 'node_phase_reference_name_does_not_match' %} - - - - - - - {% elif additional_check.type == 'node_phase_reference_name_set_but_not_in_original' %} - - - - - - - {% elif additional_check.type == 'span_phase_reference_id_not_found' %} - - - - - - - {% elif additional_check.type == 'span_phase_reference_name_does_not_match' %} - - - - - - - {% elif additional_check.type == 'span_phase_reference_name_set_but_not_in_original' %} - - - - - - - {% elif additional_check.type == 'contract_related_phase_reference_id_not_found' %} - - - - - - - {% elif additional_check.type == 'contract_related_phase_reference_name_does_not_match' %} - - - - - - - {% elif additional_check.type == 'contract_related_phase_reference_name_set_but_not_in_original' %} - - - - - - - {% elif additional_check.type == 'node_organisation_reference_id_not_found' %} + +{% if 'span_start_node_not_found' in additional_checks or 'span_end_node_not_found' in additional_checks %} +

{% trans 'Node references' %}

+

{% trans 'Your data contains spans with node references that cannot be resolved. `Span.start` and `Span.end` must match the `.id` of exactly one node in the `/nodes` array.' %}

+
{% trans 'Check' %}{% trans 'Information' %}{% trans 'Where problem occurs' %}{% trans 'Network ID' %}
- {% trans 'span_start_node_not_found' %} - - Node Id we can't find: {{ additional_check.missing_node_id }} - - Span Id: {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
- {% trans 'span_end_node_not_found' %} - - Node Id we can't find: {{ additional_check.missing_node_id }} - - Span Id: {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
- {% trans 'node_location_type_incorrect' %} - - - Node Id: {{ additional_check.node_id }} - - {{ additional_check.network_id }} -
- {% trans 'node_location_coordinates_incorrect' %} - - - Node Id: {{ additional_check.node_id }} - - {{ additional_check.network_id }} -
- {% trans 'span_route_type_incorrect' %} - - - Span Id: {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
- {% trans 'span_route_coordinates_incorrect' %} - - - Span Id: {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
- {% trans 'node_phase_reference_id_not_found' %} - - - Node Id: {{ additional_check.node_id }} - - {{ additional_check.network_id }} -
- {% trans 'node_phase_reference_name_does_not_match' %} - - - Node Id: {{ additional_check.node_id }} - - {{ additional_check.network_id }} -
- {% trans 'node_phase_reference_name_set_but_not_in_original' %} - - - Node Id: {{ additional_check.node_id }} - - {{ additional_check.network_id }} -
- {% trans 'span_phase_reference_id_not_found' %} - - - Span Id: {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
- {% trans 'span_phase_reference_name_does_not_match' %} - - - Span Id: {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
- {% trans 'span_phase_reference_name_set_but_not_in_original' %} - - - Span Id: {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
- {% trans 'contract_related_phase_reference_id_not_found' %} - - - Contract Id: {{ additional_check.contract_id }} - - {{ additional_check.network_id }} -
- {% trans 'contract_related_phase_reference_name_does_not_match' %} - - - Contract Id: {{ additional_check.contract_id }} - - {{ additional_check.network_id }} -
- {% trans 'contract_related_phase_reference_name_set_but_not_in_original' %} - - - Contract Id: {{ additional_check.contract_id }} - - {{ additional_check.network_id }} -
+ - - - - + + + - {% elif additional_check.type == 'node_organisation_reference_name_does_not_match' %} + + + {% for additional_check in additional_checks.span_start_node_not_found %} + + + + + + {% endfor %} + {% for additional_check in additional_checks.span_end_node_not_found %} + + + + + + {% endfor %} + +
- {% trans 'node_organisation_reference_id_not_found' %} - - - Node Id: {{ additional_check.node_id }} Field: {{ additional_check.field }} - - {{ additional_check.network_id }} - {% trans 'Node Id we can\'t find' %}{% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.missing_node_id }} + + {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+ {{ additional_check.missing_node_id }} + + {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'node_location_type_incorrect' in additional_checks %} +

{% trans 'Node location type' %}

+

{% trans 'Your data contains nodes with incorrect location types. `/nodes/location/type` must be `Point`.' %}

+ + - - - - + + - {% elif additional_check.type == 'node_organisation_reference_name_set_but_not_in_original' %} + + + {% for additional_check in additional_checks.node_location_type_incorrect %} + + + + + {% endfor %} + +
- {% trans 'node_organisation_reference_name_does_not_match' %} - - - Node Id: {{ additional_check.node_id }} Field: {{ additional_check.field }} - - {{ additional_check.network_id }} - {% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'node_location_coordinates_incorrect' in additional_checks %} +

{% trans 'Node location coordinates' %}

+

{% trans 'Your data contains nodes with incorrectly formatted location coordinates. `/nodes/location/coordinates` must be a single position, i.e. an array of numbers.' %}

+ + - - - - + + - {% elif additional_check.type == 'span_organisation_reference_id_not_found' %} + + + {% for additional_check in additional_checks.node_location_coordinates_incorrect %} + + + + + {% endfor %} + +
- {% trans 'node_organisation_reference_name_set_but_not_in_original' %} - - - Node Id: {{ additional_check.node_id }} Field: {{ additional_check.field }} - - {{ additional_check.network_id }} - {% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'span_route_type_incorrect' in additional_checks %} +

{% trans 'Span route geometry' %}

+

{% trans 'Your data contains spans with incorrect route types. `/spans/route/type` must be `LineString`.' %}

+ + - - - - + + - {% elif additional_check.type == 'span_organisation_reference_name_does_not_match' %} + + + {% for additional_check in additional_checks.span_route_type_incorrect %} + + + + + {% endfor %} + +
- {% trans 'span_organisation_reference_id_not_found' %} - - - Span Id: {{ additional_check.span_id }} Field: {{ additional_check.field }} - - {{ additional_check.network_id }} - {% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'span_route_coordinates_incorrect' in additional_checks %} +

{% trans 'Span route coordinates' %}

+

{% trans 'Your data contains spans with incorrectly formatted route coordinates. `/spans/route/coordinates` must be an array of positions, i.e. an array of arrays of numbers.' %}

+ + - - - - + + - {% elif additional_check.type == 'span_organisation_reference_name_set_but_not_in_original' %} + + + {% for additional_check in additional_checks.span_route_coordinates_incorrect %} + + + + + {% endfor %} + +
- {% trans 'span_organisation_reference_name_does_not_match' %} - - - Span Id: {{ additional_check.span_id }} Field: {{ additional_check.field }} - - {{ additional_check.network_id }} - {% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'node_phase_reference_id_not_found' in additional_checks or 'span_phase_reference_id_not_found' in additional_checks or 'contract_related_phase_reference_id_not_found' in additional_checks %} +

{% trans 'Phase references' %}

+

{% trans 'Your data contains phase references that cannot be resolved. The `.id` of each phase reference must match the `.id` of exactly one phase in the `/phases` array.' %}

+ {% if 'node_phase_reference_id_not_found' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.node_phase_reference_id_not_found %} + + + + + {% endfor %} + +
{% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'span_phase_reference_id_not_found' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.span_phase_reference_id_not_found %} + + + + + {% endfor %} + +
{% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'contract_related_phase_reference_id_not_found' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.contract_related_phase_reference_id_not_found %} + + + + + {% endfor %} + +
{% trans 'Contract Id' %}{% trans 'Network ID' %}
+ {{ additional_check.contract_id }} + + {{ additional_check.network_id }} +
+ {% endif %} +{% endif %} + + + + +{% if 'node_phase_reference_name_does_not_match' in additional_checks or 'span_phase_reference_name_does_not_match' in additional_checks or 'contract_related_phase_reference_name_does_not_match' in additional_checks %} +

{% trans 'Phase names' %}

+

{% trans 'Your data contains phase references with inconsistent names. The `.name` of each phase reference must match the `.name` of the phase it references.' %}

+ {% if 'node_phase_reference_name_does_not_match' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.node_phase_reference_name_does_not_match %} + + + + + {% endfor %} + +
{% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'span_phase_reference_name_does_not_match' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.span_phase_reference_name_does_not_match %} + + + + + {% endfor %} + +
{% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'contract_related_phase_reference_name_does_not_match' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.contract_related_phase_reference_name_does_not_match %} + + + + + {% endfor %} + +
{% trans 'Contract Id' %}{% trans 'Network ID' %}
+ {{ additional_check.contract_id }} + + {{ additional_check.network_id }} +
+ {% endif %} +{% endif %} + +{% if 'node_phase_reference_name_set_but_not_in_original' in additional_checks %} +

{% trans 'TODO node_phase_reference_name_set_but_not_in_original' %}

+

{% trans 'TODO' %}

+ + - - - - + + - {% elif additional_check.type == 'phase_organisation_reference_id_not_found' %} + + + {% for additional_check in additional_checks.node_phase_reference_name_set_but_not_in_original %} + + + + + {% endfor %} + +
- {% trans 'span_organisation_reference_name_set_but_not_in_original' %} - - - Span Id: {{ additional_check.span_id }} Field: {{ additional_check.field }} - - {{ additional_check.network_id }} - {% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + + + +{% if 'span_phase_reference_name_set_but_not_in_original' in additional_checks %} +

{% trans 'TODO span_phase_reference_name_set_but_not_in_original' %}

+

{% trans 'TODO' %}

+ + - - - - + + - {% elif additional_check.type == 'phase_organisation_reference_name_does_not_match' %} + + + {% for additional_check in additional_checks.span_phase_reference_name_set_but_not_in_original %} + + + + + {% endfor %} + +
- {% trans 'phase_organisation_reference_id_not_found' %} - - - Phase Id: {{ additional_check.phase_id }} - - {{ additional_check.network_id }} - {% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'contract_related_phase_reference_name_set_but_not_in_original' in additional_checks %} +

{% trans 'TODO span_phase_reference_name_set_but_not_in_original' %}

+

{% trans 'TODO' %}

+ + - - - - + + - {% elif additional_check.type == 'phase_organisation_reference_name_set_but_not_in_original' %} + + + {% for additional_check in additional_checks.contract_related_phase_reference_name_set_but_not_in_original %} + + + + + {% endfor %} + +
- {% trans 'phase_organisation_reference_name_does_not_match' %} - - - Phase Id: {{ additional_check.phase_id }} - - {{ additional_check.network_id }} - {% trans 'Contract Id' %}{% trans 'Network ID' %}
+ {{ additional_check.contract_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'node_organisation_reference_id_not_found' in additional_checks or 'node_organisation_reference_id_not_found' in additional_checks or 'phase_organisation_reference_id_not_found' in additional_checks %} +

{% trans 'Organisation references' %}

+

{% trans 'Your data contains organisation references that cannot be resolved. The `.id` of each organisation reference must match the `.id` of exactly one organisation in the `/organisations` array.' %}

+ {% if 'node_organisation_reference_id_not_found' in additional_checks %} + + + + + + + + + + {% for additional_check in additional_checks.node_organisation_reference_id_not_found %} + + + + + + {% endfor %} + +
{% trans 'Field' %}{% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.field }} + + {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'span_organisation_reference_id_not_found' in additional_checks %} + + + + + + + + + + {% for additional_check in additional_checks.span_organisation_reference_id_not_found %} + + + + + + {% endfor %} + +
{% trans 'Field' %}{% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.field }} + + {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'phase_organisation_reference_id_not_found' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.phase_organisation_reference_id_not_found %} + + + + + {% endfor %} + +
{% trans 'Phase Id' %}{% trans 'Network ID' %}
+ {{ additional_check.phase_id }} + + {{ additional_check.network_id }} +
+ {% endif %} +{% endif %} + + + +{% if 'node_organisation_reference_name_does_not_match' in additional_checks %} +

{% trans 'Organisation names' %}

+

{% trans 'Your data contains organisation references with inconsistent names. The `.name` of each organisation reference must match the `.name` of the organisation it references.' %}

+ {% if 'node_organisation_reference_name_does_not_match' in additional_checks %} + + + + + + + + + + {% for additional_check in additional_checks.node_organisation_reference_name_does_not_match %} + + + + + + {% endfor %} + +
{% trans 'Field' %}{% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.field }} + + {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'span_organisation_reference_name_does_not_match' in additional_checks %} + + + + + + + + + + {% for additional_check in additional_checks.span_organisation_reference_name_does_not_match %} + + + + + + {% endfor %} + +
{% trans 'Field' %}{% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.field }} + + {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+ {% endif %} + {% if 'phase_organisation_reference_name_does_not_match' in additional_checks %} + + + + + + + + + {% for additional_check in additional_checks.phase_organisation_reference_name_does_not_match %} + + + + + {% endfor %} + +
{% trans 'Phase ID' %}{% trans 'Network ID' %}
+ {{ additional_check.phase_id }} + + {{ additional_check.network_id }} +
+ {% endif %} +{% endif %} + + +{% if 'node_organisation_reference_name_set_but_not_in_original' in additional_checks %} +

{% trans 'TODO node_organisation_reference_name_set_but_not_in_original' %}

+

{% trans 'TODO ' %}

+ + - - - - + + + - {% elif additional_check.type == 'node_international_connections_country_not_set' %} + + + {% for additional_check in additional_checks.node_organisation_reference_name_set_but_not_in_original %} + + + + + + {% endfor %} + +
- {% trans 'phase_organisation_reference_name_set_but_not_in_original' %} - - - Phase Id: {{ additional_check.phase_id }} - - {{ additional_check.network_id }} - {% trans 'Field' %}{% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.field }} + + {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'span_organisation_reference_name_set_but_not_in_original' in additional_checks %} +

{% trans 'TODO span_organisation_reference_name_set_but_not_in_original' %}

+

{% trans 'TODO ' %}

+ + - - - - + + + - {% elif additional_check.type == 'has_links_with_external_node_data' %} + + + {% for additional_check in additional_checks.span_organisation_reference_name_set_but_not_in_original %} + + + + + + {% endfor %} + +
- {% trans 'node_international_connections_country_not_set' %} - - - Node Id: {{ additional_check.node_id }} - - {{ additional_check.network_id }} - {% trans 'Field' %}{% trans 'Span Id' %}{% trans 'Network ID' %}
+ {{ additional_check.field }} + + {{ additional_check.span_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + + + +{% if 'phase_organisation_reference_name_set_but_not_in_original' in additional_checks %} +

{% trans 'TODO phase_organisation_reference_name_set_but_not_in_original' %}

+

{% trans 'TODO ' %}

+ + - - - - + + - {% elif additional_check.type == 'has_links_with_external_span_data' %} + + + {% for additional_check in additional_checks.phase_organisation_reference_name_set_but_not_in_original %} + + + + + {% endfor %} + +
- {% trans 'has_links_with_external_node_data' %} - - - - {{ additional_check.network_id }} - {% trans 'Phase Id' %}{% trans 'Network ID' %}
+ {{ additional_check.phase_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'node_international_connections_country_not_set' in additional_checks %} +

{% trans 'International connection countries' %}

+

{% trans 'Your data contains nodes with international connections that do not specify a country. `.country` must be set for each international connection in `/nodes/internationalConnections`.' %}

+ + - - - - + + - {% elif additional_check.type == 'node_not_used_in_any_spans' %} + + + {% for additional_check in additional_checks.node_international_connections_country_not_set %} + + + + + {% endfor %} + +
- {% trans 'has_links_with_external_span_data' %} - - - - {{ additional_check.network_id }} - {% trans 'Node Id' %}{% trans 'Network ID' %}
+ {{ additional_check.node_id }} + + {{ additional_check.network_id }} +
+{% endif %} + + +{% if 'has_links_with_external_node_data' in additional_checks %} +

{% trans 'Links to nodes data' %}

+

{% trans 'Your data contains links to API endpoints or bulk files for nodes. The additional data available from the links has not been checked. You can check the data by embedding it in network package and submitting it to CoVE.' %}

+{% endif %} + + +{% if 'has_links_with_external_span_data' in additional_checks %} +

{% trans 'Links to spans data' %}

+

{% trans 'Your data contains links to API endpoints or bulk files for nodes. The additional data available from the links has not been checked. You can check the data by embedding it in network package and submitting it to CoVE.' %}

+{% endif %} + + +{% if 'node_not_used_in_any_spans' in additional_checks %} +

{% trans 'Orphaned nodes' %}

+

{% trans 'Your data contains nodes that are not referenced by any spans.' %}

+ + - - - - + + - {% endif %} - {% endfor %} - -
- {% trans 'node_not_used_in_any_spans' %} - - Node Id: {{ additional_check.node_id }} - - - {{ additional_check.network_id }} - {% trans 'Node Id' %}{% trans 'Network ID' %}
+ + + {% for additional_check in additional_checks.node_not_used_in_any_spans %} + + + {{ additional_check.node_id }} + + + {{ additional_check.network_id }} + + + {% endfor %} + + +{% endif %} + diff --git a/libcoveweb2/utils.py b/libcoveweb2/utils.py new file mode 100644 index 0000000..f6c956e --- /dev/null +++ b/libcoveweb2/utils.py @@ -0,0 +1,8 @@ +def group_data_list_by(data_list, get_key_function): + out = {} + for item in data_list: + key = get_key_function(item) + if not key in out: + out[key] = [] + out[key].append(item) + return out