diff --git a/cove_ofds/process.py b/cove_ofds/process.py
index 1f476e1..7d893fb 100644
--- a/cove_ofds/process.py
+++ b/cove_ofds/process.py
@@ -97,6 +97,55 @@ def get_context(self):
return context
+class ConvertCSVsIntoJSON(ProcessDataTask):
+ """If User uploaded CSVs, convert to our primary format, JSON."""
+
+ def process(self, process_data: dict) -> dict:
+ if self.supplied_data.format != "csvs":
+ return process_data
+
+ # check already done
+ # TODO
+
+ output_dir = os.path.join(self.supplied_data.data_dir(), "unflatten")
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ unflatten_kwargs = {
+ "output_name": os.path.join(output_dir, "unflattened.json"),
+ "root_list_path": "networks",
+ "input_format": "csv",
+ }
+
+ flattentool.unflatten(self.supplied_data.upload_dir(), **unflatten_kwargs)
+
+ process_data["json_data_filename"] = os.path.join(
+ self.supplied_data.data_dir(), "unflatten", "unflattened.json"
+ )
+
+ return process_data
+
+ def get_context(self):
+ context = {}
+ # original format
+ if self.supplied_data.format == "csvs":
+ context["original_format"] = "csvs"
+ # Download data
+ filename = os.path.join(
+ self.supplied_data.data_dir(), "unflatten", "unflattened.json"
+ )
+ if os.path.exists(filename):
+ context["can_download_json"] = True
+ context["download_json_url"] = os.path.join(
+ self.supplied_data.data_url(), "unflatten", "unflattened.json"
+ )
+ context["download_json_size"] = os.stat(filename).st_size
+ else:
+ context["can_download_json"] = False
+ # Return
+ return context
+
+
class ConvertGeoJSONIntoJSON(ProcessDataTask):
"""If User uploaded GeoJSON, convert to our primary format, JSON."""
diff --git a/cove_ofds/templates/cove_ofds/index.html b/cove_ofds/templates/cove_ofds/index.html
index 28ef289..35dbdc8 100644
--- a/cove_ofds/templates/cove_ofds/index.html
+++ b/cove_ofds/templates/cove_ofds/index.html
@@ -10,6 +10,9 @@
+
diff --git a/cove_ofds/views.py b/cove_ofds/views.py
index 340590b..88ba64f 100644
--- a/cove_ofds/views.py
+++ b/cove_ofds/views.py
@@ -9,6 +9,7 @@
from cove_ofds.forms import NewGeoJSONUploadForm
from cove_ofds.process import (
AdditionalFieldsChecksTask,
+ ConvertCSVsIntoJSON,
ConvertGeoJSONIntoJSON,
ConvertJSONIntoGeoJSON,
ConvertJSONIntoSpreadsheets,
@@ -85,6 +86,7 @@ def explore_ofds(request, pk):
# Make sure uploads are in primary format
WasJSONUploaded(db_data),
ConvertSpreadsheetIntoJSON(db_data),
+ ConvertCSVsIntoJSON(db_data),
ConvertGeoJSONIntoJSON(db_data),
# Convert into output formats
ConvertJSONIntoGeoJSON(db_data),
diff --git a/cove_project/settings.py b/cove_project/settings.py
index 8a96d98..146ed30 100644
--- a/cove_project/settings.py
+++ b/cove_project/settings.py
@@ -158,6 +158,15 @@
# https://github.com/OpenDataServices/cove/issues/1098
FILE_UPLOAD_PERMISSIONS = 0o644
+ALLOWED_JSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES
+ALLOWED_JSON_EXTENSIONS = settings.ALLOWED_JSON_EXTENSIONS
+
+ALLOWED_SPREADSHEET_CONTENT_TYPES = settings.ALLOWED_SPREADSHEET_CONTENT_TYPES
+ALLOWED_SPREADSHEET_EXTENSIONS = settings.ALLOWED_SPREADSHEET_EXTENSIONS
+
+ALLOWED_CSV_CONTENT_TYPES = settings.ALLOWED_CSV_CONTENT_TYPES
+ALLOWED_CSV_EXTENSIONS = settings.ALLOWED_CSV_EXTENSIONS
+
ALLOWED_GEOJSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES + [
"application/geo+json"
]
diff --git a/libcoveweb2/forms.py b/libcoveweb2/forms.py
index 530f46d..e26dd29 100644
--- a/libcoveweb2/forms.py
+++ b/libcoveweb2/forms.py
@@ -1,18 +1,15 @@
from django import forms
-
-from libcoveweb2.settings import (
- ALLOWED_JSON_CONTENT_TYPES,
- ALLOWED_JSON_EXTENSIONS,
- ALLOWED_SPREADSHEET_CONTENT_TYPES,
- ALLOWED_SPREADSHEET_EXTENSIONS,
-)
+from django.conf import settings
class NewJSONUploadForm(forms.Form):
file_upload = forms.FileField(
widget=forms.FileInput(
attrs={
- "accept": ",".join(ALLOWED_JSON_CONTENT_TYPES + ALLOWED_JSON_EXTENSIONS)
+ "accept": ",".join(
+ settings.ALLOWED_JSON_CONTENT_TYPES
+ + settings.ALLOWED_JSON_EXTENSIONS
+ )
}
)
)
@@ -23,8 +20,114 @@ class NewSpreadsheetUploadForm(forms.Form):
widget=forms.FileInput(
attrs={
"accept": ",".join(
- ALLOWED_SPREADSHEET_CONTENT_TYPES + ALLOWED_SPREADSHEET_EXTENSIONS
+ settings.ALLOWED_SPREADSHEET_CONTENT_TYPES
+ + settings.ALLOWED_SPREADSHEET_EXTENSIONS
)
}
)
)
+
+
+class NewCSVsUploadForm(forms.Form):
+ # I know it's hacky to copy and paste code like this but as this needs to be replaced by
+ # something that allows any number of uploads with no limits this will do for now
+ file_field_names = ["file_upload" + str(i) for i in range(0, 10)]
+ file_upload0 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ )
+ )
+ file_upload1 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload2 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload3 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload4 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload5 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload6 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload7 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload8 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
+ file_upload9 = forms.FileField(
+ widget=forms.FileInput(
+ attrs={
+ "accept": ",".join(
+ settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+ )
+ }
+ ),
+ required=False,
+ )
diff --git a/libcoveweb2/settings.py b/libcoveweb2/settings.py
index bfc3e7a..17969cb 100644
--- a/libcoveweb2/settings.py
+++ b/libcoveweb2/settings.py
@@ -182,3 +182,6 @@
"application/vnd.oasis.opendocument.spreadsheet",
]
ALLOWED_SPREADSHEET_EXTENSIONS = [".ods", ".xlsx"]
+
+ALLOWED_CSV_CONTENT_TYPES = ["text/csv"]
+ALLOWED_CSV_EXTENSIONS = [".csv"]
diff --git a/libcoveweb2/templates/libcoveweb2/new_csvs.html b/libcoveweb2/templates/libcoveweb2/new_csvs.html
new file mode 100644
index 0000000..c294701
--- /dev/null
+++ b/libcoveweb2/templates/libcoveweb2/new_csvs.html
@@ -0,0 +1,19 @@
+{% extends request.current_app_base_template %}
+{% load i18n %}
+{% load bootstrap3 %}
+
+{% block content %}
+
+
+
+
+
+{% endblock %}
diff --git a/libcoveweb2/urls.py b/libcoveweb2/urls.py
index d8a933a..e877bc9 100644
--- a/libcoveweb2/urls.py
+++ b/libcoveweb2/urls.py
@@ -6,6 +6,7 @@
urlpatterns = [
re_path(r"^new_json$", libcoveweb2.views.new_json, name="new_json"),
+ re_path(r"^new_csvs$", libcoveweb2.views.new_csvs, name="new_csvs"),
re_path(
r"^new_spreadsheet$", libcoveweb2.views.new_spreadsheet, name="new_spreadsheet"
),
diff --git a/libcoveweb2/views.py b/libcoveweb2/views.py
index deff4d6..1e08f67 100644
--- a/libcoveweb2/views.py
+++ b/libcoveweb2/views.py
@@ -1,16 +1,15 @@
+from django.conf import settings
from django.core.exceptions import ValidationError
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.utils.translation import gettext_lazy as _
-from libcoveweb2.forms import NewJSONUploadForm, NewSpreadsheetUploadForm
-from libcoveweb2.models import SuppliedData, SuppliedDataFile
-from libcoveweb2.settings import (
- ALLOWED_JSON_CONTENT_TYPES,
- ALLOWED_JSON_EXTENSIONS,
- ALLOWED_SPREADSHEET_CONTENT_TYPES,
- ALLOWED_SPREADSHEET_EXTENSIONS,
+from libcoveweb2.forms import (
+ NewCSVsUploadForm,
+ NewJSONUploadForm,
+ NewSpreadsheetUploadForm,
)
+from libcoveweb2.models import SuppliedData, SuppliedDataFile
def new_json(request):
@@ -23,11 +22,14 @@ def new_json(request):
form = forms["upload_form"]
if form.is_valid():
# Extra Validation
- if not request.FILES["file_upload"].content_type in ALLOWED_JSON_CONTENT_TYPES:
+ if (
+ not request.FILES["file_upload"].content_type
+ in settings.ALLOWED_JSON_CONTENT_TYPES
+ ):
form.add_error("file_upload", "This does not appear to be a JSON file")
if not [
e
- for e in ALLOWED_JSON_EXTENSIONS
+ for e in settings.ALLOWED_JSON_EXTENSIONS
if str(request.FILES["file_upload"].name).lower().endswith(e)
]:
form.add_error("file_upload", "This does not appear to be a JSON file")
@@ -45,6 +47,45 @@ def new_json(request):
return render(request, "libcoveweb2/new_json.html", {"forms": forms})
+def new_csvs(request):
+
+ forms = {
+ "upload_form": NewCSVsUploadForm(request.POST, request.FILES)
+ if request.POST
+ else NewCSVsUploadForm()
+ }
+ form = forms["upload_form"]
+ if form.is_valid():
+ # Extra Validation
+ for field in form.file_field_names:
+ if request.FILES.get(field):
+ if (
+ not request.FILES[field].content_type
+ in settings.ALLOWED_CSV_CONTENT_TYPES
+ ):
+ form.add_error(field, "This does not appear to be a CSV file")
+ if not [
+ e
+ for e in settings.ALLOWED_CSV_EXTENSIONS
+ if str(request.FILES[field].name).lower().endswith(e)
+ ]:
+ form.add_error(field, "This does not appear to be a CSV file")
+
+ # Process
+ if form.is_valid():
+ supplied_data = SuppliedData()
+ supplied_data.format = "csvs"
+ supplied_data.save()
+
+ for field in form.file_field_names:
+ if request.FILES.get(field):
+ supplied_data.save_file(request.FILES[field])
+
+ return HttpResponseRedirect(supplied_data.get_absolute_url())
+
+ return render(request, "libcoveweb2/new_csvs.html", {"forms": forms})
+
+
def new_spreadsheet(request):
forms = {
@@ -57,12 +98,12 @@ def new_spreadsheet(request):
# Extra Validation
if (
not request.FILES["file_upload"].content_type
- in ALLOWED_SPREADSHEET_CONTENT_TYPES
+ in settings.ALLOWED_SPREADSHEET_CONTENT_TYPES
):
form.add_error("file_upload", "This does not appear to be a spreadsheet")
if not [
e
- for e in ALLOWED_SPREADSHEET_EXTENSIONS
+ for e in settings.ALLOWED_SPREADSHEET_EXTENSIONS
if str(request.FILES["file_upload"].name).lower().endswith(e)
]:
form.add_error("file_upload", "This does not appear to be a spreadsheet")