From c4a5edf815cf54cab2e662758978d37bd8c7d710 Mon Sep 17 00:00:00 2001 From: Florian Guggi Date: Fri, 25 Oct 2024 21:55:54 +0200 Subject: [PATCH] Respect the maximum transfer size when building results With this commit, we now add the student programs result if it does not exceed 1 MB, then use the rest of the space for our own and then the student programs log with gzip compression. Closes: #148 #141 --- Cargo.lock | 132 +---------------------- scheduler/Cargo.toml | 1 + scheduler/src/command/execute_program.rs | 63 ++++++----- scheduler/src/communication/mod.rs | 7 +- 4 files changed, 46 insertions(+), 157 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24f9717..8d35db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,6 +22,7 @@ dependencies = [ "test-case", "thiserror", "toml", + "zopfli", ] [[package]] @@ -39,30 +40,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.3" @@ -72,12 +49,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" - [[package]] name = "anyhow" version = "1.0.86" @@ -103,7 +74,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", ] @@ -150,15 +121,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "crc" version = "3.2.1" @@ -208,12 +170,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dary_heap" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" - [[package]] name = "deranged" version = "0.3.11" @@ -266,16 +222,6 @@ dependencies = [ "serde", ] -[[package]] -name = "flate2" -version = "1.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" -dependencies = [ - "crc32fast", - "miniz_oxide 0.8.0", -] - [[package]] name = "fuzzy-matcher" version = "0.3.7" @@ -305,10 +251,6 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] [[package]] name = "heck" @@ -388,30 +330,6 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" -[[package]] -name = "libflate" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" -dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" -dependencies = [ - "core2", - "hashbrown", - "rle-decode-fast", -] - [[package]] name = "libudev" version = "0.3.0" @@ -478,15 +396,6 @@ dependencies = [ "adler", ] -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" -dependencies = [ - "adler2", -] - [[package]] name = "mio" version = "0.8.11" @@ -655,12 +564,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "rmp" version = "0.8.14" @@ -797,11 +700,6 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simple-archive" version = "0.1.0" -dependencies = [ - "flate2", - "libflate", - "zopfli", -] [[package]] name = "simplelog" @@ -1029,12 +927,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1229,26 +1121,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zopfli" version = "0.8.1" diff --git a/scheduler/Cargo.toml b/scheduler/Cargo.toml index d346508..e2afc53 100644 --- a/scheduler/Cargo.toml +++ b/scheduler/Cargo.toml @@ -17,6 +17,7 @@ strum = { version = "0.26.3", features = ["derive"] } subprocess = "0.2.9" thiserror = "1.0.63" toml = "0.8.19" +zopfli = "0.8.1" [features] mock = [] diff --git a/scheduler/src/command/execute_program.rs b/scheduler/src/command/execute_program.rs index 5cb25ed..afa7800 100644 --- a/scheduler/src/command/execute_program.rs +++ b/scheduler/src/command/execute_program.rs @@ -3,18 +3,16 @@ use crate::{ command::{ check_length, terminate_student_program, Event, ProgramStatus, ResultId, RetryEvent, }, - communication::{CEPPacket, CommunicationHandle}, + communication::{self, CEPPacket, CommunicationHandle}, }; use anyhow::anyhow; -use simple_archive::Compression; use std::{ - io::{ErrorKind, Write}, + io::ErrorKind, path::{Path, PathBuf}, time::Duration, }; use subprocess::Popen; - -const MAXIMUM_FILE_SIZE: usize = 1_000_000; +use zopfli::Options; /// Executes a students program and starts a watchdog for it. The watchdog also creates entries in the /// status and result queue found in `context`. The result, including logs, is packed into @@ -141,21 +139,30 @@ fn run_until_timeout( Err(()) } -/// The function uses `tar` to create an uncompressed archive that includes the result file specified, as well as -/// the programs stdout/stderr and the schedulers log file. If any of the files is missing, the archive -/// is created without them. +const RESULT_SIZE_LIMIT: usize = 1_000_000; + fn build_result_archive(res: ResultId) -> Result<(), std::io::Error> { let out_path = PathBuf::from(&format!("./data/{res}")); - let mut archive = simple_archive::Writer::new(std::fs::File::create(out_path)?); + let mut archive = simple_archive::Writer::new(Vec::new()); let res_path = PathBuf::from(format!("./archives/{}/results/{}", res.program_id, res.timestamp)); let student_log_path = PathBuf::from(format!("./data/{res}.log")); let log_path = PathBuf::from("./log"); - add_to_archive_if_exists(&mut archive, &res.to_string(), &res_path, Compression::None)?; - add_to_archive_if_exists(&mut archive, "student_log", &student_log_path, Compression::Zopfli)?; - add_to_archive_if_exists(&mut archive, "log", &log_path, Compression::Zopfli)?; + if let Some(d) = open_if_exists(&res_path)?.filter(|d| d.len() <= RESULT_SIZE_LIMIT) { + archive.append_data(&res.to_string(), &d)?; + } + + if let Some(d) = open_if_exists(&log_path)? { + compress_into_archive_if_it_fits(&mut archive, "log", &d)?; + } + + if let Some(d) = open_if_exists(&student_log_path)? { + compress_into_archive_if_it_fits(&mut archive, "student_log", &d)?; + } + + std::fs::write(out_path, archive.into_inner())?; let _ = std::fs::remove_file(res_path); let _ = std::fs::remove_file(student_log_path); @@ -164,19 +171,27 @@ fn build_result_archive(res: ResultId) -> Result<(), std::io::Error> { Ok(()) } -fn add_to_archive_if_exists( - archive: &mut simple_archive::Writer, - name: &str, - path: impl AsRef, - compression: simple_archive::Compression, -) -> std::io::Result<()> { +fn open_if_exists(path: impl AsRef) -> std::io::Result>> { match std::fs::read(path) { - Ok(mut data) => { - data.truncate(MAXIMUM_FILE_SIZE); - archive.append_data(name, &data, compression)?; - Ok(()) - } - Err(ref e) if e.kind() == ErrorKind::NotFound => Ok(()), + Ok(d) => Ok(Some(d)), + Err(ref e) if e.kind() == ErrorKind::NotFound => Ok(None), Err(e) => Err(e), } } + +fn compress_into_archive_if_it_fits( + archive: &mut simple_archive::Writer>, + path: &str, + data: &[u8], +) -> std::io::Result<()> { + let mut compressed = Vec::new(); + zopfli::compress(Options::default(), zopfli::Format::Gzip, data, &mut compressed)?; + + if compressed.len() + <= communication::MAXIMUM_DATA_LENGTH - archive.inner().len() - path.len() - 5 + { + archive.append_data(path, &compressed)?; + } + + Ok(()) +} diff --git a/scheduler/src/communication/mod.rs b/scheduler/src/communication/mod.rs index 7d4fd75..c4b22f7 100644 --- a/scheduler/src/communication/mod.rs +++ b/scheduler/src/communication/mod.rs @@ -9,13 +9,14 @@ use std::{ pub type ComResult = Result; +pub const MAXIMUM_MULTI_PACKETS: usize = 100; +pub const MAXIMUM_DATA_LENGTH: usize = MAXIMUM_MULTI_PACKETS * CEPPacket::MAXIMUM_DATA_LENGTH; + pub trait CommunicationHandle: Read + Write { const INTEGRITY_ACK_TIMEOUT: Duration; const UNLIMITED_TIMEOUT: Duration; const DATA_PACKET_RETRIES: usize = 4; - const MAXIMUM_MULTI_PACKETS: usize = 100; - const MAXIMUM_DATA_LENGTH: usize = Self::MAXIMUM_MULTI_PACKETS * CEPPacket::MAXIMUM_DATA_LENGTH; fn set_timeout(&mut self, timeout: Duration); @@ -45,7 +46,7 @@ pub trait CommunicationHandle: Read + Write { } fn send_multi_packet(&mut self, bytes: &[u8]) -> ComResult<()> { - if bytes.len() > Self::MAXIMUM_DATA_LENGTH { + if bytes.len() > MAXIMUM_DATA_LENGTH { return Err(CommunicationError::TooManyBytes); }