Refactor package cache + more concurrency improvements

.github/workflows/bidi-checker.yml

@@ -25,6 +25,6 @@ jobs:
         id: bidi_check
         uses: HL7/[email protected]
         env:
-          IGNORE: i18n-coverage-table\.png$|dummy-package.tgz$
+          IGNORE: i18n-coverage-table\.png$|dummy-package\.tgz$|dummy-package-no-index\.tgz$
       - name: Get the output time
         run: echo "The time was ${{ steps.bidi_check.outputs.time }}"

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java

@@ -69,7 +69,6 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS
 import org.apache.commons.io.FileUtils;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.utilities.FileNotifier.FileNotifier2;
-import org.hl7.fhir.utilities.Utilities.CaseInsensitiveSorter;
 import org.hl7.fhir.utilities.filesystem.CSFile;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
 import org.hl7.fhir.utilities.settings.FhirSettings;
@@ -451,6 +450,28 @@ public static String asHtmlBr(String prefix, List<String> strings) {
     return s.toString();
   }
 
+  /**
+   * Delete a directory atomically by first renaming it to a temp directory in
+   * its parent, and then deleting its contents.
+   *
+   * @param path The directory to delete.
+   */
+  public static void atomicDeleteDirectory(String path) throws IOException {
+
+      File directory = ManagedFileAccess.file(path);
+
+      String tempDirectoryPath = generateUniqueRandomUUIDPath(directory.getParent());
+      File tempDirectory = ManagedFileAccess.file(tempDirectoryPath);
+     if (!directory.renameTo(tempDirectory)) {
+       throw new IOException("Unable to rename directory " + path + " to " + tempDirectory +" for atomic delete");
+     }
+     clearDirectory(tempDirectory.getAbsolutePath());
+     if (!tempDirectory.delete()) {
+       throw new IOException("Unable to delete temp directory " + tempDirectory + " when atomically deleting " + path);
+     }
+  }
+
+
   public static void clearDirectory(String folder, String... exemptions) throws IOException {
     File dir = ManagedFileAccess.file(folder);
     if (dir.exists()) {
@@ -472,6 +493,21 @@ public static void clearDirectory(String folder, String... exemptions) throws IO
     }
   }
 
+  public static String generateUniqueRandomUUIDPath(String path) throws IOException {
+    String randomUUIDPath = null;
+
+    while (randomUUIDPath == null) {
+      final String uuid = UUID.randomUUID().toString().toLowerCase();
+      final String pathCandidate = Utilities.path(path, uuid);
+
+      if (!ManagedFileAccess.file(pathCandidate).exists()) {
+        randomUUIDPath = pathCandidate;
+      }
+    }
+
+    return randomUUIDPath;
+  }
+
   public static File createDirectory(String path) throws IOException {
     ManagedFileAccess.csfile(path).mkdirs();
     return ManagedFileAccess.file(path);

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java

@@ -186,6 +186,8 @@ public InputStreamWithSrc(InputStream stream, String url, String version) {
     }
   }
 
+
+  @Override
   public NpmPackage loadPackage(String idAndVer) throws FHIRException, IOException {
     return loadPackage(idAndVer, null);
   }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/CIBuildClient.java

@@ -0,0 +1,232 @@
+package org.hl7.fhir.utilities.npm;
+
+import lombok.Getter;
+import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.json.model.JsonArray;
+import org.hl7.fhir.utilities.json.model.JsonElement;
+import org.hl7.fhir.utilities.json.model.JsonObject;
+import org.hl7.fhir.utilities.json.parser.JsonParser;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+public class CIBuildClient {
+
+  private static final String DEFAULT_ROOT_URL = "https://build.fhir.org";
+  private static final long DEFAULT_CI_QUERY_INTERVAL = 1000 * 60 * 60;
+  private final long ciQueryInterval;
+
+  @Getter
+  private long ciLastQueriedTimeStamp = 0;
+
+  @Getter
+  private JsonArray ciBuildInfo;
+
+  @Getter
+  private final String rootUrl;
+
+  /**
+   * key = packageId
+   * value = url of built package on https://build.fhir.org/ig/
+   **/
+  private final Map<String, String> ciPackageUrls = new HashMap<>();
+
+  private final boolean silent;
+
+  public CIBuildClient() {
+   this(DEFAULT_ROOT_URL, DEFAULT_CI_QUERY_INTERVAL, false);
+  }
+
+  public CIBuildClient(String rootUrl, long ciQueryInterval, boolean silent) {
+    this.rootUrl = rootUrl;
+    this.ciQueryInterval = ciQueryInterval;
+    this.silent = silent;
+  }
+
+  String getPackageId(String canonical) {
+    if (canonical == null) {
+      return null;
+    }
+    checkCIServerQueried();
+    if (ciBuildInfo != null) {
+      for (JsonElement n : ciBuildInfo) {
+        JsonObject o = (JsonObject) n;
+        if (canonical.equals(o.asString("url"))) {
+          return o.asString("package-id");
+        }
+      }
+      for (JsonElement n : ciBuildInfo) {
+        JsonObject o = (JsonObject) n;
+        if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) {
+          return o.asString("package-id");
+        }
+      }
+    }
+    return null;
+  }
+
+  String getPackageUrl(String packageId) {
+    checkCIServerQueried();
+    for (JsonObject o : ciBuildInfo.asJsonObjects()) {
+      if (packageId.equals(o.asString("package-id"))) {
+        return o.asString("url");
+      }
+    }
+    return null;
+  }
+
+  public boolean isCurrent(String id, NpmPackage npmPackage) throws IOException {
+    checkCIServerQueried();
+    String packageManifestUrl = ciPackageUrls.get(id);
+    JsonObject packageManifestJson = JsonParser.parseObjectFromUrl(Utilities.pathURL(packageManifestUrl, "package.manifest.json"));
+    String currentDate = packageManifestJson.asString("date");
+    String packageDate = npmPackage.date();
+    return currentDate.equals(packageDate); // nup, we need a new copy
+  }
+
+  BasePackageCacheManager.InputStreamWithSrc loadFromCIBuild(String id, String branch) {
+    checkCIServerQueried();
+
+    if (ciPackageUrls.containsKey(id)) {
+      String packageBaseUrl = ciPackageUrls.get(id);
+      if (branch == null) {
+        InputStream stream;
+        try {
+          stream = fetchFromUrlSpecific(Utilities.pathURL(packageBaseUrl, "package.tgz"));
+        } catch (Exception e) {
+          stream = fetchFromUrlSpecific(Utilities.pathURL(packageBaseUrl, "branches", "main", "package.tgz"));
+        }
+        return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(packageBaseUrl, "package.tgz"), "current");
+      } else {
+        InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(packageBaseUrl, "branches", branch, "package.tgz"));
+        return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(packageBaseUrl, "branches", branch, "package.tgz"), "current$" + branch);
+      }
+    } else if (id.startsWith("hl7.fhir.r6")) {
+      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(rootUrl, id + ".tgz"));
+      return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(rootUrl, id + ".tgz"), "current");
+    } else if (id.startsWith("hl7.fhir.uv.extensions.")) {
+      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(rootUrl + "/ig/HL7/fhir-extensions/", id + ".tgz"));
+      return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(rootUrl + "/ig/HL7/fhir-extensions/", id + ".tgz"), "current");
+    } else {
+      throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciPackageUrls + ")");
+    }
+  }
+
+  private InputStream fetchFromUrlSpecific(String source) throws FHIRException {
+    try {
+      HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), source);
+      res.checkThrowException();
+      return new ByteArrayInputStream(res.getContent());
+    } catch (Exception e) {
+        throw new FHIRException("Unable to fetch: " + e.getMessage(), e);
+    }
+  }
+
+  private void checkCIServerQueried() {
+    if (System.currentTimeMillis() - ciLastQueriedTimeStamp > ciQueryInterval) {
+      try {
+        updateFromCIServer();
+      } catch (Exception e) {
+        try {
+          // we always pause a second and try again - the most common reason to be here is that the file was being changed on the server
+          Thread.sleep(1000);
+          updateFromCIServer();
+        } catch (Exception e2) {
+          if (!silent) {
+            System.out.println("Error connecting to build server - running without build (" + e2.getMessage() + ")");
+          }
+        }
+      }
+    }
+  }
+
+  private void updateFromCIServer() throws IOException {
+    HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), rootUrl + "/ig/qas.json?nocache=" + System.currentTimeMillis());
+    res.checkThrowException();
+
+    ciBuildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent()));
+
+    List<BuildRecord> builds = new ArrayList<>();
+
+    for (JsonElement n : ciBuildInfo) {
+      JsonObject j = (JsonObject) n;
+      if (j.has("url") && j.has("package-id") && j.asString("package-id").contains(".")) {
+        String packageUrl = j.asString("url");
+        if (packageUrl.contains("/ImplementationGuide/"))
+          packageUrl = packageUrl.substring(0, packageUrl.indexOf("/ImplementationGuide/"));
+        builds.add(new BuildRecord(packageUrl, j.asString("package-id"), getRepo(j.asString("repo")), readDate(j.asString("date"))));
+      }
+    }
+    Collections.sort(builds, new BuildRecordSorter());
+    for (BuildRecord build : builds) {
+      if (!ciPackageUrls.containsKey(build.getPackageId())) {
+        ciPackageUrls.put(build.getPackageId(), rootUrl + "/ig/" + build.getRepo());
+      }
+    }
+    ciLastQueriedTimeStamp = System.currentTimeMillis();
+  }
+
+  private String getRepo(String path) {
+    String[] p = path.split("/");
+    return p[0] + "/" + p[1];
+  }
+
+  private Date readDate(String s) {
+    SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US"));
+    try {
+      return sdf.parse(s);
+    } catch (ParseException e) {
+      e.printStackTrace();
+      return new Date();
+    }
+  }
+
+  public static class BuildRecord {
+
+    private final String url;
+    private final String packageId;
+    private final String repo;
+    private final Date date;
+
+    public BuildRecord(String url, String packageId, String repo, Date date) {
+      super();
+      this.url = url;
+      this.packageId = packageId;
+      this.repo = repo;
+      this.date = date;
+    }
+
+    public String getUrl() {
+      return url;
+    }
+
+    public String getPackageId() {
+      return packageId;
+    }
+
+    public String getRepo() {
+      return repo;
+    }
+
+    public Date getDate() {
+      return date;
+    }
+
+  }
+
+  public static class BuildRecordSorter implements Comparator<BuildRecord> {
+
+    @Override
+    public int compare(BuildRecord arg0, BuildRecord arg1) {
+      return arg1.date.compareTo(arg0.date);
+    }
+  }
+}