Skip to content

Commit

Permalink
Implement endpoint for queuing visits #48
Browse files Browse the repository at this point in the history
  • Loading branch information
@patrick-austin
patrick-austin committed Nov 29, 2024
1 parent ff347ae commit 077923d
Show file tree
Hide file tree
Showing 5 changed files with 297 additions and 39 deletions.
7 changes: 6 additions & 1 deletion src/main/config/run.properties.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,4 +83,9 @@ maxCacheSize=100000
# ids.timeout=3m

# Username that corresponds with the anonymous user - this is used to make anonymous carts unique
anonUserName=anon/anon
anonUserName=anon/anon

# Limit the number files per queued Download part. Multiple Datasets will be combined into part
# Downloads based on their fileCount up to this limit. If a single Dataset has a fileCount
# greater than this limit, it will still be submitted in a part by itself.
queue.maxFileCount = 10000
59 changes: 59 additions & 0 deletions src/main/java/org/icatproject/topcat/IcatClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,65 @@ public String getFullName() throws TopcatException {
}
}

/**
* Get all Datasets whose parent Investigation has the specified visitId.
*
* @param visitId ICAT Investigation.visitId
* @return JsonArray of Datasets, where each entry is a JsonArray of
* [dataset.id, dataset.fileCount].
* @throws TopcatException
*/
public JsonArray getDatasets(String visitId) throws TopcatException {
try {
String query = "SELECT dataset.id, dataset.fileCount from Dataset dataset";
query += " WHERE dataset.investigation.visitId = '" + visitId + "' ORDER BY dataset.id";
String encodedQuery = URLEncoder.encode(query, "UTF8");

String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery;
Response response = httpClient.get(url, new HashMap<String, String>());
if (response.getCode() == 404) {
throw new NotFoundException("Could not run getEntities got a 404 response");
} else if (response.getCode() >= 400) {
throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message"));
}
return Utils.parseJsonArray(response.toString());
} catch (TopcatException e) {
throw e;
} catch (Exception e) {
throw new BadRequestException(e.getMessage());
}
}

/**
* Utility method to get the fileCount (not size) of a Dataset by COUNT of its
* child Datafiles. Ideally the fileCount field should be used, this is a
* fallback option if that field is not set.
*
* @param datasetId ICAT Dataset.id
* @return The number of Datafiles in the specified Dataset
* @throws TopcatException
*/
public long getDatasetFileCount(long datasetId) throws TopcatException {
try {
String query = "SELECT COUNT(datafile) FROM Datafile datafile WHERE datafile.dataset.id = " + datasetId;
String encodedQuery = URLEncoder.encode(query, "UTF8");

String url = "entityManager?sessionId=" + URLEncoder.encode(sessionId, "UTF8") + "&query=" + encodedQuery;
Response response = httpClient.get(url, new HashMap<String, String>());
if (response.getCode() == 404) {
throw new NotFoundException("Could not run getEntities got a 404 response");
} else if (response.getCode() >= 400) {
throw new BadRequestException(Utils.parseJsonObject(response.toString()).getString("message"));
}
JsonArray jsonArray = Utils.parseJsonArray(response.toString());
return jsonArray.getJsonNumber(0).longValueExact();
} catch (TopcatException e) {
throw e;
} catch (Exception e) {
throw new BadRequestException(e.getMessage());
}
}

public List<JsonObject> getEntities(String entityType, List<Long> entityIds) throws TopcatException {
List<JsonObject> out = new ArrayList<JsonObject>();
try {
Expand Down
231 changes: 193 additions & 38 deletions src/main/java/org/icatproject/topcat/web/rest/UserResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,15 @@ public class UserResource {
private CacheRepository cacheRepository;

private String anonUserName;
private long queueMaxFileCount;

@PersistenceContext(unitName = "topcat")
EntityManager em;

public UserResource() {
Properties properties = Properties.getInstance();
this.anonUserName = properties.getProperty("anonUserName", "");
this.queueMaxFileCount = Long.valueOf(properties.getProperty("queue.maxFileCount", "10000"));
}

/**
Expand Down Expand Up @@ -665,9 +667,7 @@ public Response submitCart(@PathParam("facilityName") String facilityName,
throw new BadRequestException("fileName is required");
}

if (transport == null || transport.trim().isEmpty()) {
throw new BadRequestException("transport is required");
}
validateTransport(transport);

String icatUrl = getIcatUrl( facilityName );
IcatClient icatClient = new IcatClient(icatUrl, sessionId);
Expand All @@ -685,50 +685,21 @@ public Response submitCart(@PathParam("facilityName") String facilityName,
if(email != null && email.equals("")){
email = null;
}


if (cart != null) {
em.refresh(cart);

Download download = new Download();
download.setSessionId(sessionId);
download.setFacilityName(cart.getFacilityName());
download.setFileName(fileName);
download.setUserName(cart.getUserName());
download.setFullName(fullName);
download.setTransport(transport);
download.setEmail(email);
download.setIsEmailSent(false);
download.setSize(0);

Download download = createDownload(sessionId, cart.getFacilityName(), fileName, cart.getUserName(),
fullName, transport, email);
List<DownloadItem> downloadItems = new ArrayList<DownloadItem>();

for (CartItem cartItem : cart.getCartItems()) {
DownloadItem downloadItem = new DownloadItem();
downloadItem.setEntityId(cartItem.getEntityId());
downloadItem.setEntityType(cartItem.getEntityType());
downloadItem.setDownload(download);
DownloadItem downloadItem = createDownloadItem(download, cartItem.getEntityId(),
cartItem.getEntityType());
downloadItems.add(downloadItem);
}

download.setDownloadItems(downloadItems);

Boolean isTwoLevel = idsClient.isTwoLevel();
download.setIsTwoLevel(isTwoLevel);

if(isTwoLevel){
download.setStatus(DownloadStatus.PREPARING);
} else {
String preparedId = idsClient.prepareData(download.getSessionId(), download.getInvestigationIds(), download.getDatasetIds(), download.getDatafileIds());
download.setPreparedId(preparedId);
download.setStatus(DownloadStatus.COMPLETE);
}

downloadId = submitDownload(idsClient, download, DownloadStatus.PREPARING);
try {
em.persist(download);
em.flush();
em.refresh(download);
downloadId = download.getId();
em.remove(cart);
em.flush();
} catch (Exception e) {
Expand All @@ -740,7 +711,191 @@ public Response submitCart(@PathParam("facilityName") String facilityName,
return emptyCart(facilityName, cartUserName, downloadId);
}

/**
* Create a new Download object and set basic fields, excluding data and status.
*
* @param sessionId ICAT sessionId
* @param facilityName ICAT Facility.name
* @param fileName Filename for the resultant Download
* @param userName ICAT User.name
* @param fullName ICAT User.fullName
* @param transport Transport mechanism to use
* @param email Optional email to send notification to on completion
* @return Download object with basic fields set
*/
private static Download createDownload(String sessionId, String facilityName, String fileName, String userName,
String fullName, String transport, String email) {
Download download = new Download();
download.setSessionId(sessionId);
download.setFacilityName(facilityName);
download.setFileName(fileName);
download.setUserName(userName);
download.setFullName(fullName);
download.setTransport(transport);
download.setEmail(email);
download.setIsEmailSent(false);
download.setSize(0);
return download;
}

/**
* Create a new DownloadItem.
*
* @param download Parent Download
* @param entityId ICAT Entity.id
* @param entityType EntityType
* @return DownloadItem with fields set
*/
private static DownloadItem createDownloadItem(Download download, long entityId, EntityType entityType) {
DownloadItem downloadItem = new DownloadItem();
downloadItem.setEntityId(entityId);
downloadItem.setEntityType(entityType);
downloadItem.setDownload(download);
return downloadItem;
}

/**
* Set the final fields and persist a new Download request.
*
* @param idsClient Client for the IDS to use for the Download
* @param download Download to submit
* @param downloadStatus Initial DownloadStatus to set iff the IDS isTwoLevel
* @return Id of the new Download
* @throws TopcatException
*/
private long submitDownload(IdsClient idsClient, Download download, DownloadStatus downloadStatus)
throws TopcatException {
Boolean isTwoLevel = idsClient.isTwoLevel();
download.setIsTwoLevel(isTwoLevel);

if (isTwoLevel) {
download.setStatus(downloadStatus);
} else {
String preparedId = idsClient.prepareData(download.getSessionId(), download.getInvestigationIds(),
download.getDatasetIds(), download.getDatafileIds());
download.setPreparedId(preparedId);
download.setStatus(DownloadStatus.COMPLETE);
}

try {
em.persist(download);
em.flush();
em.refresh(download);
return download.getId();
} catch (Exception e) {
logger.info("submitCart: exception during EntityManager operations: " + e.getMessage());
throw new BadRequestException("Unable to submit for cart for download");
}
}

/**
* Queue and entire visit for download, split by Dataset into part Downloads if
* needed.
*
* @param facilityName ICAT Facility.name
* @param sessionId ICAT sessionId
* @param transport Transport mechanism to use
* @param email Optional email to notify upon completion
* @param visitId ICAT Investigation.visitId to submit
* @return Array of Download ids
* @throws TopcatException
*/
@POST
@Path("/queue/{facilityName}/visit")
public Response queueVisitId(@PathParam("facilityName") String facilityName,
@FormParam("sessionId") String sessionId, @FormParam("transport") String transport,
@FormParam("email") String email, @FormParam("visitId") String visitId) throws TopcatException {

logger.info("queueVisitId called");
validateTransport(transport);

String icatUrl = getIcatUrl(facilityName);
IcatClient icatClient = new IcatClient(icatUrl, sessionId);
String transportUrl = getDownloadUrl(facilityName, transport);
IdsClient idsClient = new IdsClient(transportUrl);

// If we wanted to block the user, this is where we would do it
String userName = icatClient.getUserName();
String fullName = icatClient.getFullName();
JsonArray datasets = icatClient.getDatasets(visitId);

long downloadId;
JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();

long part = 1;
long downloadFileCount = 0;
List<DownloadItem> downloadItems = new ArrayList<DownloadItem>();
String filename = formatQueuedFilename(facilityName, visitId, part);
Download download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email);

for (JsonValue dataset : datasets) {
JsonArray datasetArray = dataset.asJsonArray();
long datasetId = datasetArray.getJsonNumber(0).longValueExact();
long datasetFileCount = datasetArray.getJsonNumber(1).longValueExact();
if (datasetFileCount < 1) {
// Database triggers should set this, but check explicitly anyway
datasetFileCount = icatClient.getDatasetFileCount(datasetId);
}

if (downloadFileCount > 0 && downloadFileCount + datasetFileCount > queueMaxFileCount) {
download.setDownloadItems(downloadItems);
downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
jsonArrayBuilder.add(downloadId);

part += 1;
downloadFileCount = 0;
downloadItems = new ArrayList<DownloadItem>();
filename = formatQueuedFilename(facilityName, visitId, part);
download = createDownload(sessionId, facilityName, filename, userName, fullName, transport, email);
}

DownloadItem downloadItem = createDownloadItem(download, datasetId, EntityType.dataset);
downloadItems.add(downloadItem);
downloadFileCount += datasetFileCount;
}
download.setDownloadItems(downloadItems);
downloadId = submitDownload(idsClient, download, DownloadStatus.PAUSED);
jsonArrayBuilder.add(downloadId);

return Response.ok(jsonArrayBuilder.build()).build();
}

/**
* Format the filename for a queued Download, possibly one part of many.
*
* @param facilityName ICAT Facility.name
* @param visitId ICAT Investigation.visitId
* @param part 1 indexed part of the overall request
* @return Formatted filename
*/
private static String formatQueuedFilename(String facilityName, String visitId, long part) {
String partString = String.valueOf(part);
StringBuilder partBuilder = new StringBuilder();
while (partBuilder.length() + partString.length() < 4) {
partBuilder.append("0");
}
partBuilder.append(partString);

StringBuilder filenameBuilder = new StringBuilder();
filenameBuilder.append(facilityName);
filenameBuilder.append("_");
filenameBuilder.append(visitId);
filenameBuilder.append("_");
filenameBuilder.append(partBuilder);
return filenameBuilder.toString();
}

/**
* Validate that the submitted transport mechanism is not null or empty.
*
* @param transport Transport mechanism to use
* @throws BadRequestException if null or empty
*/
private static void validateTransport(String transport) throws BadRequestException {
if (transport == null || transport.trim().isEmpty()) {
throw new BadRequestException("transport is required");
}
}

/**
* Retrieves the total file size (in bytes) for any investigation, datasets or datafiles.
Expand Down
37 changes: 37 additions & 0 deletions src/test/java/org/icatproject/topcat/UserResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,43 @@ public void testSubmitCart() throws Exception {
assertTrue(newDownload.getIsDeleted());
}

@Test
public void testQueueVisitId() throws Exception {
List<Long> downloadIds = new ArrayList<>();
try {
String facilityName = "LILS";
String transport = "http";
String email = "";
String visitId = "Proposal 0 - 0 0";
Response response = userResource.queueVisitId(facilityName, sessionId, transport, email, visitId);
assertEquals(200, response.getStatus());

JsonArray downloadIdsArray = Utils.parseJsonArray(response.getEntity().toString());
assertEquals(3, downloadIdsArray.size());
long part = 1;
for (JsonNumber downloadIdJson : downloadIdsArray.getValuesAs(JsonNumber.class)) {
long downloadId = downloadIdJson.longValueExact();
downloadIds.add(downloadId);
Download download = downloadRepository.getDownload(downloadId);
assertNull(download.getPreparedId());
assertEquals(DownloadStatus.PAUSED, download.getStatus());
assertEquals(0, download.getInvestigationIds().size());
assertEquals(1, download.getDatasetIds().size());
assertEquals(0, download.getDatafileIds().size());
assertEquals("LILS_Proposal 0 - 0 0_000" + part, download.getFileName());
assertEquals(transport, download.getTransport());
assertEquals("simple/root", download.getUserName());
assertEquals("simple/root", download.getFullName());
assertEquals("", download.getEmail());
part += 1;
}
} finally {
for (long downloadId : downloadIds) {
downloadRepository.removeDownload(downloadId);
}
}
}

@Test
public void testGetDownloadTypeStatus() throws Exception {

Expand Down
Loading

0 comments on commit 077923d

Please sign in to comment.