Improve archive file sizes. (#982)

This PR improves the growth of RRDP archive files. It tries to re-use
existing empty space when updating objects. It also pads all objects to a
multiple of 256 bytes to make it more likely that empty space can be reused.

The PR also adds a new command archive-stats that prints statistics for an
archive file.

Finally, the PR raises the Minimum Supported Rust Version to 1.73 to be able
to use u64::next_multiple_of.

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [1.70.0, stable, beta]
+        rust: [1.73.0, stable, beta]
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2

Cargo.toml

@@ -3,7 +3,7 @@
 name = "routinator"
 version = "0.14.1-dev"
 edition = "2021"
-rust-version = "1.70"
+rust-version = "1.73"
 authors = ["NLnet Labs <[email protected]>"]
 description = "An RPKI relying party software."
 repository = "https://github.com/NLnetLabs/routinator"

Dockerfile

@@ -44,7 +44,7 @@ ARG MODE=build
 # ========
 #
 # Only used when MODE=build.
-ARG BASE_IMG=alpine:3.18
+ARG BASE_IMG=alpine:3.21
 
 
 # CARGO_ARGS

src/collector/mod.rs

@@ -25,7 +25,7 @@
 //  `Collector`, `Run`, and `Repository` types.
 //
 pub use self::base::{Collector, Cleanup, Run, Repository};
-pub use self::rrdp::{HttpStatus, SnapshotReason};
+pub use self::rrdp::{HttpStatus, RrdpArchive, SnapshotReason};
 
 mod base;
 mod rrdp;

src/collector/rrdp/archive.rs

@@ -13,7 +13,7 @@ use crate::config::Config;
 use crate::error::RunFailed;
 use crate::utils::archive;
 use crate::utils::archive::{
-    Archive, ArchiveError, FetchError, OpenError, PublishError
+    Archive, ArchiveError, ArchiveStats, FetchError, OpenError, PublishError
 };
 use crate::utils::binio::{Compose, Parse};
 
@@ -81,10 +81,9 @@ impl RrdpArchive {
 }
 
 impl RrdpArchive {
-    pub fn verify(path: &Path) -> Result<(), OpenError> {
+    pub fn verify(path: &Path) -> Result<ArchiveStats, OpenError> {
         let archive = archive::Archive::<RrdpObjectMeta>::open(path, false)?;
-        archive.verify()?;
-        Ok(())
+        Ok(archive.verify()?)
     }
 
     /// Loads an object from the archive.

src/collector/rrdp/base.rs

@@ -108,7 +108,7 @@ impl Collector {
                     continue;
                 }
                 match RrdpArchive::verify(entry.path()) {
-                    Ok(()) | Err(OpenError::NotFound) => { }
+                    Ok(_) | Err(OpenError::NotFound) => { }
                     Err(OpenError::Archive(ArchiveError::Io(err))) => {
                         error!(
                             "Fatal: Failed to read RRDP repository archive\

src/collector/rrdp/mod.rs

@@ -1,5 +1,6 @@
 #![allow(dead_code)]
 
+pub use self::archive::RrdpArchive;
 pub use self::base::{Collector, LoadResult, ReadRepository, Run};
 pub use self::http::HttpStatus;
 pub use self::update::SnapshotReason;

src/operation.rs

@@ -70,6 +70,7 @@ pub enum Operation {
     Update(Update),
     PrintConfig(PrintConfig),
     Dump(Dump),
+    ArchiveStats(ArchiveStats),
     Man(Man),
 }
 
@@ -93,6 +94,7 @@ impl Operation {
         let app = Update::config_args(app);
         let app = PrintConfig::config_args(app);
         let app = Dump::config_args(app);
+        let app = ArchiveStats::config_args(app);
         Man::config_args(app)
     }
 
@@ -131,6 +133,11 @@ impl Operation {
             Some(("dump", matches)) => {
                 Operation::Dump( Dump::from_arg_matches(matches, cur_dir)?)
             }
+            Some(("archive-stats", matches)) => {
+                Operation::ArchiveStats(
+                    ArchiveStats::from_arg_matches(matches)?
+                )
+            }
             Some(("man", matches)) => {
                 Operation::Man(Man::from_arg_matches(matches)?)
             }
@@ -167,6 +174,7 @@ impl Operation {
             Operation::Update(cmd) => cmd.run(process),
             Operation::PrintConfig(cmd) => cmd.run(process),
             Operation::Dump(cmd) => cmd.run(process),
+            Operation::ArchiveStats(cmd) => cmd.run(process),
             Operation::Man(cmd) => cmd.run(process),
         }
     }
@@ -255,6 +263,7 @@ impl Server {
                 if let Some(log) = log.as_ref() {
                     log.start();
                 }
+
                 let timeout = match LocalExceptions::load(
                     process.config(), true
                 ) {
@@ -310,6 +319,11 @@ impl Server {
                 // to recalculate timeout.
                 let deadline = Instant::now() + timeout;
 
+                info!(
+                    "Next validation run scheduled in {} seconds",
+                    timeout.as_secs()
+                );
+
                 let end = loop {
                     let timeout = deadline.saturating_duration_since(
                         Instant::now()
@@ -1184,6 +1198,57 @@ impl Dump {
 }
 
 
+//------------ ArchiveStats --------------------------------------------------
+
+/// Prints archive statistics.
+#[derive(Clone, Debug, Parser)]
+pub struct ArchiveStats {
+    /// Archive file.
+    #[arg(value_name = "PATH")]
+    archive: PathBuf,
+}
+
+impl ArchiveStats {
+    /// Adds the command configuration to a clap app.
+    pub fn config_args<'a: 'b, 'b>(app: clap::Command) -> clap::Command {
+        // config
+        app.subcommand(
+            ArchiveStats::augment_args(
+                clap::Command::new("archive-stats")
+                    .about("Prints statics for an RRDP archive")
+                    .after_help(AFTER_HELP)
+            )
+        )
+    }
+
+    /// Creates a command from clap matches.
+    pub fn from_arg_matches(matches: &ArgMatches) -> Result<Self, Failed> {
+        Ok(
+            <ArchiveStats as FromArgMatches>::from_arg_matches(
+                matches
+            ).unwrap()
+        )
+    }
+
+    fn run(self, process: Process) -> Result<(), ExitError> {
+        use crate::collector::RrdpArchive;
+
+        process.switch_logging(false, false)?;
+        match RrdpArchive::verify(&self.archive) {
+            Ok(stats) => {
+                println!("RRDP archive {}:", self.archive.display());
+                stats.print();
+                Ok(())
+            }
+            Err(err) => {
+                eprintln!("Archive is corrupt: {err}");
+                Err(ExitError::Generic)
+            }
+        }
+    }
+}
+
+
 //------------ Man -----------------------------------------------------------
 
 /// Show the manual page.