Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Oct 13, 2023
1 parent 63db1c9 commit 498dae6
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 34 deletions.
1 change: 1 addition & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ Right now, we are in the process of building the prototype. We warmly welcome yo
Create pull requests (PRs) for changes that you think are needed. We would really appreciate your help!

Skills with the following would be beneficial:

- Python
- FastAPI
- Javascript
Expand Down
44 changes: 23 additions & 21 deletions src/backend/app/projects/project_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,20 @@
# You should have received a copy of the GNU General Public License
# along with FMTM. If not, see <https:#www.gnu.org/licenses/>.
#
import os
import csv
import base64
import csv
import io
import json
import os
import time
import uuid
import zipfile
from base64 import b64encode
from io import BytesIO
from io import BytesIO, StringIO
from json import dumps, loads
from typing import List
from zipfile import ZipFile
from zipfile import ZipFile as zf

import geoalchemy2
import geojson
Expand Down Expand Up @@ -70,9 +70,6 @@
from ..tasks import tasks_crud
from ..users import user_crud
from . import project_schemas
from zipfile import ZipFile as zf
from io import BytesIO, StringIO


QR_CODES_DIR = "QR_codes/"
TASK_GEOJSON_DIR = "geojson/"
Expand Down Expand Up @@ -2665,6 +2662,7 @@ def generate_appuser_files_for_janakpur(
db, background_task_id, 4
) # 4 is COMPLETED


def expand_geopoints(csv, geopoint_column_name):
"""Accepts a list representing a set of CSV ODK submissions and expands
a geopoint column to include lon, lat, ele, acc columns for easy
Expand Down Expand Up @@ -2694,38 +2692,42 @@ def expand_geopoints(csv, geopoint_column_name):

return newcsv


def project_submissions_unzipped(
pid, formsl, outdir, collate, expand_geopoint, odk_central_credentials
):
"""Downloads and unzips all of the submissions from a given ODK project."""
if collate:

collated_outfilepath = os.path.join(outdir, f'project_{pid}_submissions'
'_collated.csv')
c_outfile = open(collated_outfilepath, 'w')
collated_outfilepath = os.path.join(
outdir, f"project_{pid}_submissions" "_collated.csv"
)
c_outfile = open(collated_outfilepath, "w")
cw = csv.writer(c_outfile)

# create a single file to dump all repeat data lines
# TODO multiple collated files for multiple repeats
c_repeatfilepath = os.path.join(outdir, f'project_{pid}_repeats'
'_collated.csv')
c_repeatfile = open(c_repeatfilepath, 'w')
c_repeatfilepath = os.path.join(
outdir, f"project_{pid}_repeats" "_collated.csv"
)
c_repeatfile = open(c_repeatfilepath, "w")
cr = csv.writer(c_repeatfile)

for fidx, form in enumerate(formsl):
form_id = form['xmlFormId']
print(f'Checking submissions from {form_id}.')
form_id = form["xmlFormId"]
print(f"Checking submissions from {form_id}.")
# subs_zip = csv_submissions(url, aut, pid, form_id)

subs_zip = central_crud.download_submissions_media(pid, form_id, odk_central_credentials)
subs_zip = central_crud.download_submissions_media(
pid, form_id, odk_central_credentials
)

subs_bytes = BytesIO(subs_zip.content)
subs_bytes.seek(0)
subs_unzipped = zf(subs_bytes)
sub_namelist = subs_unzipped.namelist()

subcount = len(sub_namelist)
print(f'There are {subcount} files in submissions from {form_id}:')
print(f"There are {subcount} files in submissions from {form_id}:")
print(sub_namelist)

# Now save the rest of the files
Expand All @@ -2741,12 +2743,12 @@ def project_submissions_unzipped(
# If it is a csv, open it and see if it is more than one line
# This might go wrong if something is encoded in other than UTF-8

if os.path.splitext(sub_name)[1] == '.csv':
if os.path.splitext(sub_name)[1] == ".csv":
subs_stringio = StringIO(subs_bytes.decode())
subs_list = list(csv.reader(subs_stringio))
# Check if there are CSV lines after the headers
subs_len = len(subs_list)
print(f'{sub_name} has {subs_len - 1} submissions')
print(f"{sub_name} has {subs_len - 1} submissions")
if subs_len > 1:
subs_to_write = subs_list
if expand_geopoint:
Expand All @@ -2755,7 +2757,7 @@ def project_submissions_unzipped(
w = csv.writer(outfile)
w.writerows(subs_to_write)
if collate:
if not idx:
if not idx:
if not fidx:
# First form. Include header
cw.writerows(subs_to_write)
Expand All @@ -2767,7 +2769,7 @@ def project_submissions_unzipped(
# TODO actually create a separate collated
# CSV output for each repeat in the survey
cr.writerows(subs_to_write)

else:
with open(outfilename, "wb") as outfile:
outfile.write(subs_bytes)
26 changes: 13 additions & 13 deletions src/backend/app/projects/project_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
#
import json
import os
import uuid
import shutil
import uuid
from typing import List, Optional

from fastapi import (
Expand All @@ -38,7 +38,7 @@
from osm_fieldwork.xlsforms import xlsforms_path
from sqlalchemy.orm import Session

from ..central import central_crud, central_schemas
from ..central import central_crud
from ..db import database, db_models
from ..models.enums import TILES_SOURCE
from ..tasks import tasks_crud
Expand Down Expand Up @@ -382,10 +382,11 @@ async def upload_multi_project_boundary(
# Get the number of tasks in a project
task_count = await tasks_crud.get_task_count_in_project(db, project_id)

return {"message": "Project Boundary Uploaded",
"project_id": f"{project_id}",
"task_count": task_count
}
return {
"message": "Project Boundary Uploaded",
"project_id": f"{project_id}",
"task_count": task_count,
}


@router.post("/task_split")
Expand Down Expand Up @@ -1157,19 +1158,16 @@ async def generate_files_janakpur(

@router.get("/{project_id}/download_csv/")
def download_forms(
project_id: int,
db: Session = Depends(database.get_db)
project_id: int, db: Session = Depends(database.get_db)
) -> FileResponse:
"""
Download the submissions for a given project in CSV format.
"""Download the submissions for a given project in CSV format.
Parameters:
- project_id (int): The ID of the project.
Returns:
- FileResponse: The response object containing the downloaded CSV file.
"""

project = project_crud.get_project(db, project_id)
odkid = project.odkid

Expand All @@ -1187,10 +1185,12 @@ def download_forms(
shutil.rmtree(output_dir)
os.makedirs(output_dir)

project_crud.project_submissions_unzipped(odkid, forms, output_dir, True, False, odk_credentials)
project_crud.project_submissions_unzipped(
odkid, forms, output_dir, True, False, odk_credentials
)

temp_zip_file = "/tmp/submissions"
shutil.make_archive(temp_zip_file, 'zip', output_dir)
shutil.make_archive(temp_zip_file, "zip", output_dir)

headers = {
"Content-Disposition": f'attachment; filename="{os.path.basename(temp_zip_file)}.zip"'
Expand Down

0 comments on commit 498dae6

Please sign in to comment.