diff --git a/src/main/java/org/icatproject/topcat/IcatClient.java b/src/main/java/org/icatproject/topcat/IcatClient.java index 9702a5b9..f3ff6d12 100644 --- a/src/main/java/org/icatproject/topcat/IcatClient.java +++ b/src/main/java/org/icatproject/topcat/IcatClient.java @@ -97,29 +97,35 @@ public String getFullName() throws TopcatException { * Get all Datasets whose parent Investigation has the specified visitId. * * @param visitId ICAT Investigation.visitId - * @return JsonArray of Datasets, where each entry is a JsonArray of + * @return JsonArray of Dataset fields, where each entry is a JsonArray of * [dataset.id, dataset.fileCount]. * @throws TopcatException */ public JsonArray getDatasets(String visitId) throws TopcatException { - try { String query = "SELECT dataset.id, dataset.fileCount from Dataset dataset"; query += " WHERE dataset.investigation.visitId = '" + visitId + "' ORDER BY dataset.id"; - String encodedQuery = URLEncoder.encode(query, "UTF8"); + return submitQuery(query); + } - String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery; - Response response = httpClient.get(url, new HashMap()); - if (response.getCode() == 404) { - throw new NotFoundException("Could not run getEntities got a 404 response"); - } else if (response.getCode() >= 400) { - throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message")); - } - return Utils.parseJsonArray(response.toString()); - } catch (TopcatException e) { - throw e; - } catch (Exception e) { - throw new BadRequestException(e.getMessage()); - } + /** + * Get all Datafiles in the list of file locations. + * + * @param files List of ICAT Datafile.locations + * @return JsonArray of Datafile ids. + * @throws TopcatException + */ + public JsonArray getDatafiles(List files) throws TopcatException { + StringBuilder stringBuilder = new StringBuilder(); + ListIterator fileIterator = files.listIterator(); + stringBuilder.append("'" + fileIterator.next() + "'"); + fileIterator.forEachRemaining(file -> { + stringBuilder.append(","); + stringBuilder.append("'" + file + "'"); + }); + String formattedFiles = stringBuilder.toString(); + String query = "SELECT datafile.id from Datafile datafile"; + query += " WHERE datafile.location in (" + formattedFiles + ") ORDER BY datafile.id"; + return submitQuery(query); } /** @@ -132,10 +138,22 @@ public JsonArray getDatasets(String visitId) throws TopcatException { * @throws TopcatException */ public long getDatasetFileCount(long datasetId) throws TopcatException { - try { String query = "SELECT COUNT(datafile) FROM Datafile datafile WHERE datafile.dataset.id = " + datasetId; - String encodedQuery = URLEncoder.encode(query, "UTF8"); + JsonArray jsonArray = submitQuery(query); + return jsonArray.getJsonNumber(0).longValueExact(); + } + /** + * Utility method for submitting an unformatted query to the entityManager + * endpoint, and returning the resultant JsonArray. + * + * @param query Unformatted String query to submit + * @return JsonArray of results, contents will depend on the query. + * @throws TopcatException + */ + private JsonArray submitQuery(String query) throws TopcatException { + try { + String encodedQuery = URLEncoder.encode(query, "UTF8"); String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery; Response response = httpClient.get(url, new HashMap()); if (response.getCode() == 404) { @@ -143,8 +161,7 @@ public long getDatasetFileCount(long datasetId) throws TopcatException { } else if (response.getCode() >= 400) { throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message")); } - JsonArray jsonArray = Utils.parseJsonArray(response.toString()); - return jsonArray.getJsonNumber(0).longValueExact(); + return Utils.parseJsonArray(response.toString()); } catch (TopcatException e) { throw e; } catch (Exception e) { diff --git a/src/main/java/org/icatproject/topcat/web/rest/UserResource.java b/src/main/java/org/icatproject/topcat/web/rest/UserResource.java index a19f8b44..a07f4f8c 100644 --- a/src/main/java/org/icatproject/topcat/web/rest/UserResource.java +++ b/src/main/java/org/icatproject/topcat/web/rest/UserResource.java @@ -789,7 +789,7 @@ private long submitDownload(IdsClient idsClient, Download download, DownloadStat } /** - * Queue and entire visit for download, split by Dataset into part Downloads if + * Queue an entire visit for download, split by Dataset into part Downloads if * needed. * * @param facilityName ICAT Facility.name @@ -860,6 +860,75 @@ public Response queueVisitId(@PathParam("facilityName") String facilityName, return Response.ok(jsonArrayBuilder.build()).build(); } + /** + * Queue download of Datafiles by location, splitting into part Downloads if + * needed. + * + * @param facilityName ICAT Facility.name + * @param sessionId ICAT sessionId + * @param transport Transport mechanism to use + * @param email Optional email to notify upon completion + * @param files ICAT Datafile.locations to download + * @return Array of Download ids + * @throws TopcatException + */ + @POST + @Path("/queue/{facilityName}/files") + public Response queueFiles(@PathParam("facilityName") String facilityName, + @FormParam("sessionId") String sessionId, @FormParam("transport") String transport, + @FormParam("email") String email, @FormParam("files") List files) throws TopcatException { + + logger.info("queueVisitId called"); + validateTransport(transport); + if (files.size() == 0) { + throw new BadRequestException("At least one Datafile.location required"); + } + + String icatUrl = getIcatUrl(facilityName); + IcatClient icatClient = new IcatClient(icatUrl, sessionId); + String transportUrl = getDownloadUrl(facilityName, transport); + IdsClient idsClient = new IdsClient(transportUrl); + + // If we wanted to block the user, this is where we would do it + String userName = icatClient.getUserName(); + String fullName = icatClient.getFullName(); + JsonArray datafiles = icatClient.getDatafiles(files); + + long downloadId; + JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + + long part = 1; + long downloadFileCount = 0; + List downloadItems = new ArrayList(); + String filename = formatQueuedFilename(facilityName, "files", part); + Download download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email); + + for (JsonNumber datafileIdJsonNumber : datafiles.getValuesAs(JsonNumber.class)) { + long datafileId = datafileIdJsonNumber.longValueExact(); + + if (downloadFileCount >= queueMaxFileCount) { + download.setDownloadItems(downloadItems); + downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED); + jsonArrayBuilder.add(downloadId); + + part += 1; + downloadFileCount = 0; + downloadItems = new ArrayList(); + filename = formatQueuedFilename(facilityName, "files", part); + download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email); + } + + DownloadItem downloadItem = createDownloadItem(download, datafileId, EntityType.datafile); + downloadItems.add(downloadItem); + downloadFileCount += 1; + } + download.setDownloadItems(downloadItems); + downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED); + jsonArrayBuilder.add(downloadId); + + return Response.ok(jsonArrayBuilder.build()).build(); + } + /** * Format the filename for a queued Download, possibly one part of many. * diff --git a/src/test/java/org/icatproject/topcat/UserResourceTest.java b/src/test/java/org/icatproject/topcat/UserResourceTest.java index 2f9ba86a..bbd5e388 100644 --- a/src/test/java/org/icatproject/topcat/UserResourceTest.java +++ b/src/test/java/org/icatproject/topcat/UserResourceTest.java @@ -287,6 +287,43 @@ public void testQueueVisitId() throws Exception { } } + @Test + public void testQueueFiles() throws Exception { + List downloadIds = new ArrayList<>(); + try { + String facilityName = "LILS"; + String transport = "http"; + String email = ""; + List files = Arrays.asList("return/tax/free", "thus/upon/land", "hard/wife/five"); + Response response = userResource.queueFiles(facilityName, sessionId, transport, email, files); + assertEquals(200, response.getStatus()); + + JsonArray downloadIdsArray = Utils.parseJsonArray(response.getEntity().toString()); + assertEquals(3, downloadIdsArray.size()); + long part = 1; + for (JsonNumber downloadIdJson : downloadIdsArray.getValuesAs(JsonNumber.class)) { + long downloadId = downloadIdJson.longValueExact(); + downloadIds.add(downloadId); + Download download = downloadRepository.getDownload(downloadId); + assertNull(download.getPreparedId()); + assertEquals(DownloadStatus.PAUSED, download.getStatus()); + assertEquals(0, download.getInvestigationIds().size()); + assertEquals(0, download.getDatasetIds().size()); + assertEquals(1, download.getDatafileIds().size()); + assertEquals("LILS_files_000" + part, download.getFileName()); + assertEquals(transport, download.getTransport()); + assertEquals("simple/root", download.getUserName()); + assertEquals("simple/root", download.getFullName()); + assertEquals("", download.getEmail()); + part += 1; + } + } finally { + for (long downloadId : downloadIds) { + downloadRepository.removeDownload(downloadId); + } + } + } + @Test public void testGetDownloadTypeStatus() throws Exception { diff --git a/src/test/resources/run.properties b/src/test/resources/run.properties index 26196a71..c70b6242 100644 --- a/src/test/resources/run.properties +++ b/src/test/resources/run.properties @@ -7,5 +7,5 @@ anonUserName=anon/anon # Disable scheduled Download status checks (DO THIS FOR TESTS ONLY!) test.disableDownloadStatusChecks = true -# Test data has 100 files per Dataset, set this to 100 to ensure coverage of the batching logic -queue.maxFileCount = 100 +# Test data has 100 files per Dataset, set this to a small number to ensure coverage of the batching logic +queue.maxFileCount = 1