GeoPackage Export (#199)

* GeoPackge updates

* Create single locations table for all users.
* Create rtree index on location table.
* Remove user table which containted the last location for each user, this location is in the locations table.j
* Add user icon style to locations table.
* Add username, display name, accuracy, altitude, bearing and speed columns to locations table

* Don't use form field title for geopackage column name, as geopackage column name is required to be unique

* [service] exports: remove unsafe event name characters from geopackage export file name

* [service] remove only qualifier from test

---------

Co-authored-by: Robert St. John <[email protected]>

service/functionalTests/client.ts

@@ -266,8 +266,8 @@ export class MageClientSession {
     return this.http.get<ExportInfo[]>('/api/exports/myself').then(x => x.data)
   }
 
-  downloadExport(exportId: ExportId): Promise<buffer.Buffer> {
-    return this.http.get<buffer.Buffer>(`/api/exports/${exportId}`).then(x => x.data)
+  downloadExport(exportId: ExportId): Promise<NodeJS.ReadableStream> {
+    return this.http.get<NodeJS.ReadableStream>(`/api/exports/${exportId}`, { responseType: 'stream' }).then(x => x.data)
   }
 
   /**

service/functionalTests/exports/exports.test.ts

@@ -1,7 +1,11 @@
+import path from 'path'
+import fs from 'fs'
+import fs_async from 'fs/promises'
 import { expect } from 'chai'
 import { ExportInfo, ExportFormat, MageClientSession, ExportStatus, SignInResult } from '../client'
 import { ChildProcessTestStackRef, launchTestStack } from '../stack'
 import * as Fixture from './fixture'
+import Zip from 'jszip'
 
 
 describe('exports', function() {
@@ -103,4 +107,36 @@ describe('exports', function() {
     expect(finishedExport instanceof Error, 'geojson export error').to.be.false
     expect(finishedExport.status).to.equal(ExportStatus.Completed, 'geopackage export incomplete')
   })
+
+  for (const format of Object.values(ExportFormat)) {
+    it(`${format} export tolerates an event name with unsafe file system characters`, async function() {
+      const pendingExport = await rootSession.startExport(
+        fixture.eventWithUnsafeName.id,
+        { exportType: format, observations: true, locations: true, attachments: true }
+      )
+      const finishedExport = await rootSession.waitForMyExport(pendingExport.id, 5000) as ExportInfo
+
+      expect(finishedExport instanceof Error, `${format} export error`).to.be.false
+      expect(finishedExport.status).to.equal(ExportStatus.Completed, `${format} export incomplete`)
+    })
+  }
+
+  it('removes unsafe file system characters from geopackage export', async function() {
+
+    const pendingExport = await rootSession.startExport(
+      fixture.eventWithUnsafeName.id,
+      { exportType: ExportFormat.GeoPackage, observations: true, locations: true, attachments: true }
+    )
+    const finishedExport = await rootSession.waitForMyExport(pendingExport.id, 5000) as ExportInfo
+    const downloadDir = path.join(stack.baseDirPath, 'download')
+    const downloadPath = path.join(downloadDir, 'unsafe_gp_name.zip')
+    fs.mkdirSync(downloadDir)
+    const exportContent = await rootSession.downloadExport(finishedExport.id)
+    await fs_async.writeFile(downloadPath, exportContent)
+    const downloadZip = await Zip.loadAsync(fs.readFileSync(downloadPath))
+    const entryNames = Object.keys(downloadZip.files)
+
+    expect(entryNames).to.have.length(1)
+    expect(entryNames[0].replace(/\.gpkg$/, '')).to.match(/^[\w ]+$/)
+  })
 })

service/functionalTests/exports/fixture.ts

@@ -58,6 +58,11 @@ export const eventSeed: MageEventCreateRequest = {
   style: {},
 }
 
+export const eventUnsafeNameSeed: MageEventCreateRequest = {
+  name: '/tmp/etc/passwd',
+  style: {}
+}
+
 export const formsSeed: MageFormCreateRequest[] = [
   {
     name: 'form1',
@@ -123,6 +128,19 @@ export const formsSeed: MageFormCreateRequest[] = [
           { id: 3, value: 3, title: 'gold' },
         ]
       },
+      {
+        id: 5,
+        archived: true,
+        name: 'form1/dropdown3',
+        required: false,
+        title: 'Choice 2',
+        type: FormFieldType.Dropdown,
+        choices: [
+          { id: 1, value: 1, title: 'red' },
+          { id: 2, value: 2, title: 'green' },
+          { id: 3, value: 3, title: 'gold' },
+        ]
+      }
     ]
   },
   {
@@ -722,7 +740,7 @@ export async function populateFixtureData(stack: TestStack, rootSession: MageCli
 
   const now = Date.now()
   for (const obs of observations) {
-    expect(new Date(obs[1].createdAt).getTime()).to.be.closeTo(now, 250)
+    expect(new Date(obs[1].createdAt).getTime()).to.be.closeTo(now, 500)
   }
 
   const archivedForm = await rootSession.archiveForm(eventWithForms, formsByName.archivedForm.id).then(x => x.data)
@@ -829,11 +847,22 @@ export async function populateFixtureData(stack: TestStack, rootSession: MageCli
   expect(user4LocationNoDevice.data).to.have.length(1)
   expect(user4LocationNoDevice.data[0].properties.devicedId).to.not.exist
 
+  const eventWithUnsafeName = await rootSession.createEvent(eventUnsafeNameSeed)
+    .then(x => rootSession.readEvent(x.data.id)).then(x => x.data)
+    .then(unsafeEvent => rootSession.createForm(unsafeEvent.id, formsSeed[0]).then(() => unsafeEvent))
+    .then(unsafeEvent => rootSession.addParticipantToEvent(unsafeEvent, users[0].id).then(() => unsafeEvent))
+    .then(unsafeEvent => rootSession.readEvent(unsafeEvent.id).then(res => res.data))
+  await user1Session.postUserLocations(eventWithUnsafeName.id, [
+    [ 30, 40, Date.now() ]
+  ])
+
   return {
     event: finalEvent,
+    eventWithUnsafeName,
   }
 }
 
 export interface ExportTestFixture {
-  event: MageEventPopulated
+  event: MageEventPopulated,
+  eventWithUnsafeName: MageEventPopulated
 }