update: added proving time

.env.example

@@ -73,7 +73,7 @@ MADARA_ORCHESTRATOR_AWS_S3_BUCKET_NAME=      # S3 bucket name
 #### INSTRUMENTATION ####
 ## OTEL ##
 MADARA_ORCHESTRATOR_OTEL_SERVICE_NAME=       # OpenTelemetry service name
-MADARA_ORCHESTRATOR_OTES_COLLECTOR_ENDPOINT= # OpenTelemetry collector endpoint
+MADARA_ORCHESTRATOR_OTEL_COLLECTOR_ENDPOINT= # OpenTelemetry collector endpoint
 
 #### SERVER ####
 MADARA_ORCHESTRATOR_HOST=                    # Server host

crates/orchestrator/src/jobs/mod.rs

@@ -5,6 +5,7 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use async_trait::async_trait;
+use chrono::Utc;
 use color_eyre::eyre::{eyre, Context};
 use constants::JOB_METADATA_FAILURE_REASON;
 use conversion::parse_string;
@@ -234,6 +235,10 @@ pub async fn process_job(id: Uuid, config: Arc<Config>) -> Result<(), JobError>
     let external_id = match AssertUnwindSafe(job_handler.process_job(config.clone(), &mut job)).catch_unwind().await {
         Ok(Ok(external_id)) => {
             tracing::debug!(job_id = ?id, "Successfully processed job");
+            // Add the time of processing to the metadata.
+            if job.job_type == JobType::ProofCreation {
+                job.metadata.insert("processing_completed_at".to_string(), Utc::now().timestamp_millis().to_string());
+            }
             external_id
         }
         Ok(Err(e)) => {
@@ -352,9 +357,25 @@ pub async fn verify_job(id: Uuid, config: Arc<Config>) -> Result<(), JobError> {
     match verification_status {
         JobVerificationStatus::Verified => {
             tracing::info!(job_id = ?id, "Job verified successfully");
+            if job.job_type == JobType::ProofCreation {
+                match job
+                    .metadata
+                    .get("processing_completed_at")
+                    .and_then(|time| time.parse::<i64>().ok())
+                    .map(|start| Utc::now().timestamp_millis() - start)
+                {
+                    Some(time_taken) => ORCHESTRATOR_METRICS.proving_time.record(time_taken as f64, &[]),
+                    None => eprintln!("Failed to calculate proving time: Invalid or missing processing time"),
+                }
+            }
+            let mut metadata = job.metadata.clone();
+            metadata.remove("processing_completed_at");
             config
                 .database()
-                .update_job(&job, JobItemUpdates::new().update_status(JobStatus::Completed).build())
+                .update_job(
+                    &job,
+                    JobItemUpdates::new().update_metadata(metadata).update_status(JobStatus::Completed).build(),
+                )
                 .await
                 .map_err(|e| {
                     tracing::error!(job_id = ?id, error = ?e, "Failed to update job status to Completed");

crates/orchestrator/src/metrics.rs

@@ -12,6 +12,7 @@ pub struct OrchestratorMetrics {
     pub successful_job_operations: Counter<f64>,
     pub failed_job_operations: Counter<f64>,
     pub failed_jobs: Counter<f64>,
+    pub proving_time: Gauge<f64>,
     pub jobs_response_time: Gauge<f64>,
     pub db_calls_response_time: Gauge<f64>,
 }
@@ -57,25 +58,33 @@ impl Metrics for OrchestratorMetrics {
             "jobs".to_string(),
         );
 
+        let proving_time = register_gauge_metric_instrument(
+            &orchestrator_meter,
+            "proving_time".to_string(),
+            "A gauge to show the time taken for proving task".to_string(),
+            "ms".to_string(),
+        );
+
         let jobs_response_time = register_gauge_metric_instrument(
             &orchestrator_meter,
             "jobs_response_time".to_string(),
             "A gauge to show response time of jobs over time".to_string(),
-            "Time".to_string(),
+            "s".to_string(),
         );
 
         let db_calls_response_time = register_gauge_metric_instrument(
             &orchestrator_meter,
             "db_calls_response_time".to_string(),
             "A gauge to show response time of jobs over time".to_string(),
-            "Time".to_string(),
+            "s".to_string(),
         );
 
         Self {
             block_gauge,
             successful_job_operations,
             failed_job_operations,
             failed_jobs,
+            proving_time,
             jobs_response_time,
             db_calls_response_time,
         }