From fa06664f2557dabbbd429efc60931f475fc03970 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Nov 2022 10:36:31 +0000 Subject: [PATCH] csvs: Allow uploading multiple CSVs https://github.com/Open-Telecoms-Data/cove-ofds/issues/15 Also change to using settings from Django settings, which allows possibility that COVE app can override settings in libcoveweb --- cove_ofds/process.py | 49 +++++++ cove_ofds/templates/cove_ofds/index.html | 3 + cove_ofds/views.py | 2 + cove_project/settings.py | 9 ++ libcoveweb2/forms.py | 121 ++++++++++++++++-- libcoveweb2/settings.py | 3 + .../templates/libcoveweb2/new_csvs.html | 19 +++ libcoveweb2/urls.py | 1 + libcoveweb2/views.py | 63 +++++++-- 9 files changed, 250 insertions(+), 20 deletions(-) create mode 100644 libcoveweb2/templates/libcoveweb2/new_csvs.html diff --git a/cove_ofds/process.py b/cove_ofds/process.py index 1f476e1..7d893fb 100644 --- a/cove_ofds/process.py +++ b/cove_ofds/process.py @@ -97,6 +97,55 @@ def get_context(self): return context +class ConvertCSVsIntoJSON(ProcessDataTask): + """If User uploaded CSVs, convert to our primary format, JSON.""" + + def process(self, process_data: dict) -> dict: + if self.supplied_data.format != "csvs": + return process_data + + # check already done + # TODO + + output_dir = os.path.join(self.supplied_data.data_dir(), "unflatten") + + os.makedirs(output_dir, exist_ok=True) + + unflatten_kwargs = { + "output_name": os.path.join(output_dir, "unflattened.json"), + "root_list_path": "networks", + "input_format": "csv", + } + + flattentool.unflatten(self.supplied_data.upload_dir(), **unflatten_kwargs) + + process_data["json_data_filename"] = os.path.join( + self.supplied_data.data_dir(), "unflatten", "unflattened.json" + ) + + return process_data + + def get_context(self): + context = {} + # original format + if self.supplied_data.format == "csvs": + context["original_format"] = "csvs" + # Download data + filename = os.path.join( + self.supplied_data.data_dir(), "unflatten", "unflattened.json" + ) + if os.path.exists(filename): + context["can_download_json"] = True + context["download_json_url"] = os.path.join( + self.supplied_data.data_url(), "unflatten", "unflattened.json" + ) + context["download_json_size"] = os.stat(filename).st_size + else: + context["can_download_json"] = False + # Return + return context + + class ConvertGeoJSONIntoJSON(ProcessDataTask): """If User uploaded GeoJSON, convert to our primary format, JSON.""" diff --git a/cove_ofds/templates/cove_ofds/index.html b/cove_ofds/templates/cove_ofds/index.html index 28ef289..35dbdc8 100644 --- a/cove_ofds/templates/cove_ofds/index.html +++ b/cove_ofds/templates/cove_ofds/index.html @@ -10,6 +10,9 @@
Upload Spreadsheet
+
+ Upload CSVs +
Upload GeoJSON
diff --git a/cove_ofds/views.py b/cove_ofds/views.py index 340590b..88ba64f 100644 --- a/cove_ofds/views.py +++ b/cove_ofds/views.py @@ -9,6 +9,7 @@ from cove_ofds.forms import NewGeoJSONUploadForm from cove_ofds.process import ( AdditionalFieldsChecksTask, + ConvertCSVsIntoJSON, ConvertGeoJSONIntoJSON, ConvertJSONIntoGeoJSON, ConvertJSONIntoSpreadsheets, @@ -85,6 +86,7 @@ def explore_ofds(request, pk): # Make sure uploads are in primary format WasJSONUploaded(db_data), ConvertSpreadsheetIntoJSON(db_data), + ConvertCSVsIntoJSON(db_data), ConvertGeoJSONIntoJSON(db_data), # Convert into output formats ConvertJSONIntoGeoJSON(db_data), diff --git a/cove_project/settings.py b/cove_project/settings.py index 8a96d98..146ed30 100644 --- a/cove_project/settings.py +++ b/cove_project/settings.py @@ -158,6 +158,15 @@ # https://github.com/OpenDataServices/cove/issues/1098 FILE_UPLOAD_PERMISSIONS = 0o644 +ALLOWED_JSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES +ALLOWED_JSON_EXTENSIONS = settings.ALLOWED_JSON_EXTENSIONS + +ALLOWED_SPREADSHEET_CONTENT_TYPES = settings.ALLOWED_SPREADSHEET_CONTENT_TYPES +ALLOWED_SPREADSHEET_EXTENSIONS = settings.ALLOWED_SPREADSHEET_EXTENSIONS + +ALLOWED_CSV_CONTENT_TYPES = settings.ALLOWED_CSV_CONTENT_TYPES +ALLOWED_CSV_EXTENSIONS = settings.ALLOWED_CSV_EXTENSIONS + ALLOWED_GEOJSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES + [ "application/geo+json" ] diff --git a/libcoveweb2/forms.py b/libcoveweb2/forms.py index 530f46d..e26dd29 100644 --- a/libcoveweb2/forms.py +++ b/libcoveweb2/forms.py @@ -1,18 +1,15 @@ from django import forms - -from libcoveweb2.settings import ( - ALLOWED_JSON_CONTENT_TYPES, - ALLOWED_JSON_EXTENSIONS, - ALLOWED_SPREADSHEET_CONTENT_TYPES, - ALLOWED_SPREADSHEET_EXTENSIONS, -) +from django.conf import settings class NewJSONUploadForm(forms.Form): file_upload = forms.FileField( widget=forms.FileInput( attrs={ - "accept": ",".join(ALLOWED_JSON_CONTENT_TYPES + ALLOWED_JSON_EXTENSIONS) + "accept": ",".join( + settings.ALLOWED_JSON_CONTENT_TYPES + + settings.ALLOWED_JSON_EXTENSIONS + ) } ) ) @@ -23,8 +20,114 @@ class NewSpreadsheetUploadForm(forms.Form): widget=forms.FileInput( attrs={ "accept": ",".join( - ALLOWED_SPREADSHEET_CONTENT_TYPES + ALLOWED_SPREADSHEET_EXTENSIONS + settings.ALLOWED_SPREADSHEET_CONTENT_TYPES + + settings.ALLOWED_SPREADSHEET_EXTENSIONS ) } ) ) + + +class NewCSVsUploadForm(forms.Form): + # I know it's hacky to copy and paste code like this but as this needs to be replaced by + # something that allows any number of uploads with no limits this will do for now + file_field_names = ["file_upload" + str(i) for i in range(0, 10)] + file_upload0 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ) + ) + file_upload1 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload2 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload3 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload4 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload5 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload6 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload7 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload8 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload9 = forms.FileField( + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) diff --git a/libcoveweb2/settings.py b/libcoveweb2/settings.py index bfc3e7a..17969cb 100644 --- a/libcoveweb2/settings.py +++ b/libcoveweb2/settings.py @@ -182,3 +182,6 @@ "application/vnd.oasis.opendocument.spreadsheet", ] ALLOWED_SPREADSHEET_EXTENSIONS = [".ods", ".xlsx"] + +ALLOWED_CSV_CONTENT_TYPES = ["text/csv"] +ALLOWED_CSV_EXTENSIONS = [".csv"] diff --git a/libcoveweb2/templates/libcoveweb2/new_csvs.html b/libcoveweb2/templates/libcoveweb2/new_csvs.html new file mode 100644 index 0000000..c294701 --- /dev/null +++ b/libcoveweb2/templates/libcoveweb2/new_csvs.html @@ -0,0 +1,19 @@ +{% extends request.current_app_base_template %} +{% load i18n %} +{% load bootstrap3 %} + +{% block content %} + +
+
+ {% csrf_token %} + {% bootstrap_form forms.upload_form %} + {% buttons %} + + {% endbuttons %} +
+
+ +{% endblock %} diff --git a/libcoveweb2/urls.py b/libcoveweb2/urls.py index d8a933a..e877bc9 100644 --- a/libcoveweb2/urls.py +++ b/libcoveweb2/urls.py @@ -6,6 +6,7 @@ urlpatterns = [ re_path(r"^new_json$", libcoveweb2.views.new_json, name="new_json"), + re_path(r"^new_csvs$", libcoveweb2.views.new_csvs, name="new_csvs"), re_path( r"^new_spreadsheet$", libcoveweb2.views.new_spreadsheet, name="new_spreadsheet" ), diff --git a/libcoveweb2/views.py b/libcoveweb2/views.py index deff4d6..1e08f67 100644 --- a/libcoveweb2/views.py +++ b/libcoveweb2/views.py @@ -1,16 +1,15 @@ +from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpResponseRedirect from django.shortcuts import render from django.utils.translation import gettext_lazy as _ -from libcoveweb2.forms import NewJSONUploadForm, NewSpreadsheetUploadForm -from libcoveweb2.models import SuppliedData, SuppliedDataFile -from libcoveweb2.settings import ( - ALLOWED_JSON_CONTENT_TYPES, - ALLOWED_JSON_EXTENSIONS, - ALLOWED_SPREADSHEET_CONTENT_TYPES, - ALLOWED_SPREADSHEET_EXTENSIONS, +from libcoveweb2.forms import ( + NewCSVsUploadForm, + NewJSONUploadForm, + NewSpreadsheetUploadForm, ) +from libcoveweb2.models import SuppliedData, SuppliedDataFile def new_json(request): @@ -23,11 +22,14 @@ def new_json(request): form = forms["upload_form"] if form.is_valid(): # Extra Validation - if not request.FILES["file_upload"].content_type in ALLOWED_JSON_CONTENT_TYPES: + if ( + not request.FILES["file_upload"].content_type + in settings.ALLOWED_JSON_CONTENT_TYPES + ): form.add_error("file_upload", "This does not appear to be a JSON file") if not [ e - for e in ALLOWED_JSON_EXTENSIONS + for e in settings.ALLOWED_JSON_EXTENSIONS if str(request.FILES["file_upload"].name).lower().endswith(e) ]: form.add_error("file_upload", "This does not appear to be a JSON file") @@ -45,6 +47,45 @@ def new_json(request): return render(request, "libcoveweb2/new_json.html", {"forms": forms}) +def new_csvs(request): + + forms = { + "upload_form": NewCSVsUploadForm(request.POST, request.FILES) + if request.POST + else NewCSVsUploadForm() + } + form = forms["upload_form"] + if form.is_valid(): + # Extra Validation + for field in form.file_field_names: + if request.FILES.get(field): + if ( + not request.FILES[field].content_type + in settings.ALLOWED_CSV_CONTENT_TYPES + ): + form.add_error(field, "This does not appear to be a CSV file") + if not [ + e + for e in settings.ALLOWED_CSV_EXTENSIONS + if str(request.FILES[field].name).lower().endswith(e) + ]: + form.add_error(field, "This does not appear to be a CSV file") + + # Process + if form.is_valid(): + supplied_data = SuppliedData() + supplied_data.format = "csvs" + supplied_data.save() + + for field in form.file_field_names: + if request.FILES.get(field): + supplied_data.save_file(request.FILES[field]) + + return HttpResponseRedirect(supplied_data.get_absolute_url()) + + return render(request, "libcoveweb2/new_csvs.html", {"forms": forms}) + + def new_spreadsheet(request): forms = { @@ -57,12 +98,12 @@ def new_spreadsheet(request): # Extra Validation if ( not request.FILES["file_upload"].content_type - in ALLOWED_SPREADSHEET_CONTENT_TYPES + in settings.ALLOWED_SPREADSHEET_CONTENT_TYPES ): form.add_error("file_upload", "This does not appear to be a spreadsheet") if not [ e - for e in ALLOWED_SPREADSHEET_EXTENSIONS + for e in settings.ALLOWED_SPREADSHEET_EXTENSIONS if str(request.FILES["file_upload"].name).lower().endswith(e) ]: form.add_error("file_upload", "This does not appear to be a spreadsheet")