Skip to content

Commit

Permalink
fileName -> part_x_of_y, facilityName -> formParam, initialise as longs
Browse files Browse the repository at this point in the history 
#48
  • Loading branch information
@patrick-austin
patrick-austin committed Jan 14, 2025
1 parent 742620c commit 38fcd0a
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 28 deletions.
4 changes: 2 additions & 2 deletions src/main/java/org/icatproject/topcat/IcatClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ public JsonArray getDatasets(String visitId) throws TopcatException {
String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery;
Response response = httpClient.get(url, new HashMap<String, String>());
if (response.getCode() == 404) {
throw new NotFoundException("Could not run getEntities got a 404 response");
throw new NotFoundException("Could not run getDatasets got a 404 response");
} else if (response.getCode() >= 400) {
throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message"));
}
Expand Down Expand Up @@ -162,7 +162,7 @@ public long getDatasetFileCount(long datasetId) throws TopcatException {
String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery;
Response response = httpClient.get(url, new HashMap<String, String>());
if (response.getCode() == 404) {
throw new NotFoundException("Could not run getEntities got a 404 response");
throw new NotFoundException("Could not run getDatasetFileCount got a 404 response");
} else if (response.getCode() >= 400) {
throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message"));
}
Expand Down
63 changes: 38 additions & 25 deletions src/main/java/org/icatproject/topcat/web/rest/UserResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ private String getCartUserName(String userName, String sessionId) {
* Login to create a session
*
* @param facilityName A facility name - properties must map this to a url to a valid ICAT REST api, if set.
* Can be null iff one Facility set in the config for the API.
* @param username ICAT username
* @param password Password for the specified authentication plugin
* @param plugin ICAT authentication plugin. If null, a default value will be used.
Expand Down Expand Up @@ -799,7 +798,7 @@ private static DownloadItem createDownloadItem(Download download, long entityId,
*
* @param idsClient Client for the IDS to use for the Download
* @param download Download to submit
* @param downloadStatus Initial DownloadStatus to set iff the IDS isTwoLevel
* @param downloadStatus Initial DownloadStatus to set if and only if the IDS isTwoLevel
* @return Id of the new Download
* @throws TopcatException
*/
Expand Down Expand Up @@ -829,7 +828,7 @@ private long submitDownload(IdsClient idsClient, Download download, DownloadStat
}

/**
* Queue and entire visit for download, split by Dataset into part Downloads if
* Queue an entire visit for download, split by Dataset into part Downloads if
* needed.
*
* @param facilityName ICAT Facility.name
Expand All @@ -841,8 +840,8 @@ private long submitDownload(IdsClient idsClient, Download download, DownloadStat
* @throws TopcatException
*/
@POST
@Path("/queue/{facilityName}/visit")
public Response queueVisitId(@PathParam("facilityName") String facilityName,
@Path("/queue/visit")
public Response queueVisitId(@FormParam("facilityName") String facilityName,
@FormParam("sessionId") String sessionId, @FormParam("transport") String transport,
@FormParam("email") String email, @FormParam("visitId") String visitId) throws TopcatException {

Expand All @@ -862,40 +861,50 @@ public Response queueVisitId(@PathParam("facilityName") String facilityName,
long downloadId;
JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();

long part = 1;
long downloadFileCount = 0;
long downloadFileCount = 0L;
List<DownloadItem> downloadItems = new ArrayList<DownloadItem>();
String filename = formatQueuedFilename(facilityName, visitId, part);
Download download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email);
List<Download> downloads = new ArrayList<Download>();
// String filename = formatQueuedFilename(facilityName, visitId, part);
Download newDownload = createDownload(sessionId, facilityName, "", userName, fullName, transport, email);

for (JsonValue dataset : datasets) {
JsonArray datasetArray = dataset.asJsonArray();
long datasetId = datasetArray.getJsonNumber(0).longValueExact();
long datasetFileCount = datasetArray.getJsonNumber(1).longValueExact();
if (datasetFileCount < 1) {
if (datasetFileCount < 1L) {
// Database triggers should set this, but check explicitly anyway
datasetFileCount = icatClient.getDatasetFileCount(datasetId);
}

if (downloadFileCount > 0 && downloadFileCount + datasetFileCount > queueMaxFileCount) {
download.setDownloadItems(downloadItems);
downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
jsonArrayBuilder.add(downloadId);
if (downloadFileCount > 0L && downloadFileCount + datasetFileCount > queueMaxFileCount) {
newDownload.setDownloadItems(downloadItems);
downloads.add(newDownload);
// downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
// jsonArrayBuilder.add(downloadId);

part += 1;
downloadFileCount = 0;
// part += 1L;
downloadFileCount = 0L;
downloadItems = new ArrayList<DownloadItem>();
filename = formatQueuedFilename(facilityName, visitId, part);
download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email);
// filename = formatQueuedFilename(facilityName, visitId, part);
newDownload = createDownload(sessionId, facilityName, "", userName, fullName, transport, email);
}

DownloadItem downloadItem = createDownloadItem(download, datasetId, EntityType.dataset);
DownloadItem downloadItem = createDownloadItem(newDownload, datasetId, EntityType.dataset);
downloadItems.add(downloadItem);
downloadFileCount += datasetFileCount;
}
download.setDownloadItems(downloadItems);
downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
jsonArrayBuilder.add(downloadId);
newDownload.setDownloadItems(downloadItems);
downloads.add(newDownload);
// downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
// jsonArrayBuilder.add(downloadId);
int part = 1;
for (Download download : downloads) {
String filename = formatQueuedFilename(facilityName, visitId, part, downloads.size());
download.setFileName(filename);
downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
jsonArrayBuilder.add(downloadId);
part += 1;
}

return Response.ok(jsonArrayBuilder.build()).build();
}
Expand All @@ -906,12 +915,14 @@ public Response queueVisitId(@PathParam("facilityName") String facilityName,
* @param facilityName ICAT Facility.name
* @param visitId ICAT Investigation.visitId
* @param part 1 indexed part of the overall request
* @param size Number of parts in the overall request
* @return Formatted filename
*/
private static String formatQueuedFilename(String facilityName, String visitId, long part) {
private static String formatQueuedFilename(String facilityName, String visitId, int part, int size) {
String partString = String.valueOf(part);
String sizeString = String.valueOf(size);
StringBuilder partBuilder = new StringBuilder();
while (partBuilder.length() + partString.length() < 4) {
while (partBuilder.length() + partString.length() < sizeString.length()) {
partBuilder.append("0");
}
partBuilder.append(partString);
Expand All @@ -920,8 +931,10 @@ private static String formatQueuedFilename(String facilityName, String visitId,
filenameBuilder.append(facilityName);
filenameBuilder.append("_");
filenameBuilder.append(visitId);
filenameBuilder.append("_");
filenameBuilder.append("_part_");
filenameBuilder.append(partBuilder);
filenameBuilder.append("_of_");
filenameBuilder.append(sizeString);
return filenameBuilder.toString();
}

Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/icatproject/topcat/UserResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ public void testQueueVisitId() throws Exception {
assertEquals(0, download.getInvestigationIds().size());
assertEquals(1, download.getDatasetIds().size());
assertEquals(0, download.getDatafileIds().size());
assertEquals("LILS_Proposal 0 - 0 0_000" + part, download.getFileName());
assertEquals("LILS_Proposal 0 - 0 0_part_" + part + "_of_3", download.getFileName());
assertEquals(transport, download.getTransport());
assertEquals("simple/root", download.getUserName());
assertEquals("simple/root", download.getFullName());
Expand Down

0 comments on commit 38fcd0a

Please sign in to comment.