Implement queueFiles endpoint #50

src/main/java/org/icatproject/topcat/IcatClient.java

@@ -97,29 +97,35 @@ public String getFullName() throws TopcatException {
 	 * Get all Datasets whose parent Investigation has the specified visitId.
 	 * 
 	 * @param visitId ICAT Investigation.visitId
-	 * @return JsonArray of Datasets, where each entry is a JsonArray of
+	 * @return JsonArray of Dataset fields, where each entry is a JsonArray of
 	 *         [dataset.id, dataset.fileCount].
 	 * @throws TopcatException
 	 */
 	public JsonArray getDatasets(String visitId) throws TopcatException {
-		try {
 			String query = "SELECT dataset.id, dataset.fileCount from Dataset dataset";
 			query += " WHERE dataset.investigation.visitId = '" + visitId + "' ORDER BY dataset.id";
-			String encodedQuery = URLEncoder.encode(query, "UTF8");
+		return submitQuery(query);
+	}
 
-			String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery;
-			Response response = httpClient.get(url, new HashMap<String, String>());
-			if (response.getCode() == 404) {
-				throw new NotFoundException("Could not run getEntities got a 404 response");
-			} else if (response.getCode() >= 400) {
-				throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message"));
-			}
-			return Utils.parseJsonArray(response.toString());
-		} catch (TopcatException e) {
-            throw e;
-		} catch (Exception e) {
-			throw new BadRequestException(e.getMessage());
-		}
+	/**
+	 * Get all Datafiles in the list of file locations.
+	 * 
+	 * @param files List of ICAT Datafile.locations
+	 * @return JsonArray of Datafile ids.
+	 * @throws TopcatException
+	 */
+	public JsonArray getDatafiles(List<String> files) throws TopcatException {
+		StringBuilder stringBuilder = new StringBuilder();
+		ListIterator<String> fileIterator = files.listIterator();
+		stringBuilder.append("'" + fileIterator.next() + "'");
+		fileIterator.forEachRemaining(file -> {
+			stringBuilder.append(",");
+			stringBuilder.append("'" + file + "'");
+		});
+		String formattedFiles = stringBuilder.toString();
+		String query = "SELECT datafile.id from Datafile datafile";
+		query += " WHERE datafile.location in (" + formattedFiles + ") ORDER BY datafile.id";
+		return submitQuery(query);
 	}
 
 	/**
@@ -132,19 +138,30 @@ public JsonArray getDatasets(String visitId) throws TopcatException {
 	 * @throws TopcatException
 	 */
 	public long getDatasetFileCount(long datasetId) throws TopcatException {
-		try {
 			String query = "SELECT COUNT(datafile) FROM Datafile datafile WHERE datafile.dataset.id = " + datasetId;
-			String encodedQuery = URLEncoder.encode(query, "UTF8");
+		JsonArray jsonArray = submitQuery(query);
+		return jsonArray.getJsonNumber(0).longValueExact();
+	}
 
+	/**
+	 * Utility method for submitting an unformatted query to the entityManager
+	 * endpoint, and returning the resultant JsonArray.
+	 * 
+	 * @param query Unformatted String query to submit
+	 * @return JsonArray of results, contents will depend on the query.
+	 * @throws TopcatException
+	 */
+	private JsonArray submitQuery(String query) throws TopcatException {
+		try {
+			String encodedQuery = URLEncoder.encode(query, "UTF8");
 			String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery;
 			Response response = httpClient.get(url, new HashMap<String, String>());
 			if (response.getCode() == 404) {
 				throw new NotFoundException("Could not run getEntities got a 404 response");
 			} else if (response.getCode() >= 400) {
 				throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message"));
 			}
-			JsonArray jsonArray = Utils.parseJsonArray(response.toString());
-			return jsonArray.getJsonNumber(0).longValueExact();
+			return Utils.parseJsonArray(response.toString());
 		} catch (TopcatException e) {
             throw e;
 		} catch (Exception e) {

src/main/java/org/icatproject/topcat/web/rest/UserResource.java

@@ -789,7 +789,7 @@ private long submitDownload(IdsClient idsClient, Download download, DownloadStat
 	}
 
 	/**
-	 * Queue and entire visit for download, split by Dataset into part Downloads if
+	 * Queue an entire visit for download, split by Dataset into part Downloads if
 	 * needed.
 	 * 
 	 * @param facilityName ICAT Facility.name
@@ -860,6 +860,75 @@ public Response queueVisitId(@PathParam("facilityName") String facilityName,
 		return Response.ok(jsonArrayBuilder.build()).build();
 	}
 
+	/**
+	 * Queue download of Datafiles by location, splitting into part Downloads if
+	 * needed.
+	 * 
+	 * @param facilityName ICAT Facility.name
+	 * @param sessionId    ICAT sessionId
+	 * @param transport    Transport mechanism to use
+	 * @param email        Optional email to notify upon completion
+	 * @param files        ICAT Datafile.locations to download
+	 * @return Array of Download ids
+	 * @throws TopcatException
+	 */
+	@POST
+	@Path("/queue/{facilityName}/files")
+	public Response queueFiles(@PathParam("facilityName") String facilityName,
+			@FormParam("sessionId") String sessionId, @FormParam("transport") String transport,
+			@FormParam("email") String email, @FormParam("files") List<String> files) throws TopcatException {
+
+		logger.info("queueVisitId called");
+		validateTransport(transport);
+		if (files.size() == 0) {
+			throw new BadRequestException("At least one Datafile.location required");
+		}
+
+		String icatUrl = getIcatUrl(facilityName);
+		IcatClient icatClient = new IcatClient(icatUrl, sessionId);
+		String transportUrl = getDownloadUrl(facilityName, transport);
+		IdsClient idsClient = new IdsClient(transportUrl);
+
+		// If we wanted to block the user, this is where we would do it
+		String userName = icatClient.getUserName();
+		String fullName = icatClient.getFullName();
+		JsonArray datafiles = icatClient.getDatafiles(files);
+
+		long downloadId;
+		JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
+
+		long part = 1;
+		long downloadFileCount = 0;
+		List<DownloadItem> downloadItems = new ArrayList<DownloadItem>();
+		String filename = formatQueuedFilename(facilityName, "files", part);
+		Download download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email);
+
+		for (JsonNumber datafileIdJsonNumber : datafiles.getValuesAs(JsonNumber.class)) {
+			long datafileId = datafileIdJsonNumber.longValueExact();
+
+			if (downloadFileCount >= queueMaxFileCount) {
+				download.setDownloadItems(downloadItems);
+				downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
+				jsonArrayBuilder.add(downloadId);
+
+				part += 1;
+				downloadFileCount = 0;
+				downloadItems = new ArrayList<DownloadItem>();
+				filename = formatQueuedFilename(facilityName, "files", part);
+				download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email);
+			}
+
+			DownloadItem downloadItem = createDownloadItem(download, datafileId, EntityType.datafile);
+			downloadItems.add(downloadItem);
+			downloadFileCount += 1;
+		}
+		download.setDownloadItems(downloadItems);
+		downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
+		jsonArrayBuilder.add(downloadId);
+
+		return Response.ok(jsonArrayBuilder.build()).build();
+	}
+
 	/**
 	 * Format the filename for a queued Download, possibly one part of many.
 	 * 

src/test/java/org/icatproject/topcat/UserResourceTest.java

@@ -287,6 +287,43 @@ public void testQueueVisitId() throws Exception {
 		}
 	}
 
+	@Test
+	public void testQueueFiles() throws Exception {
+		List<Long> downloadIds = new ArrayList<>();
+		try {
+			String facilityName = "LILS";
+			String transport = "http";
+			String email = "";
+			List<String> files = Arrays.asList("return/tax/free", "thus/upon/land", "hard/wife/five");
+			Response response = userResource.queueFiles(facilityName, sessionId, transport, email, files);
+			assertEquals(200, response.getStatus());
+
+			JsonArray downloadIdsArray = Utils.parseJsonArray(response.getEntity().toString());
+			assertEquals(3, downloadIdsArray.size());
+			long part = 1;
+			for (JsonNumber downloadIdJson : downloadIdsArray.getValuesAs(JsonNumber.class)) {
+				long downloadId = downloadIdJson.longValueExact();
+				downloadIds.add(downloadId);
+				Download download = downloadRepository.getDownload(downloadId);
+				assertNull(download.getPreparedId());
+				assertEquals(DownloadStatus.PAUSED, download.getStatus());
+				assertEquals(0, download.getInvestigationIds().size());
+				assertEquals(0, download.getDatasetIds().size());
+				assertEquals(1, download.getDatafileIds().size());
+				assertEquals("LILS_files_000" + part, download.getFileName());
+				assertEquals(transport, download.getTransport());
+				assertEquals("simple/root", download.getUserName());
+				assertEquals("simple/root", download.getFullName());
+				assertEquals("", download.getEmail());
+				part += 1;
+			}
+		} finally {
+			for (long downloadId : downloadIds) {
+				downloadRepository.removeDownload(downloadId);
+			}
+		}
+	}
+
 	@Test
 	public void testGetDownloadTypeStatus() throws Exception {
 

src/test/resources/run.properties

@@ -7,5 +7,5 @@ anonUserName=anon/anon
 
 # Disable scheduled Download status checks (DO THIS FOR TESTS ONLY!)
 test.disableDownloadStatusChecks = true
-# Test data has 100 files per Dataset, set this to 100 to ensure coverage of the batching logic
-queue.maxFileCount = 100
+# Test data has 100 files per Dataset, set this to a small number to ensure coverage of the batching logic
+queue.maxFileCount = 1