Skip to content

Commit

Permalink
chore: adding logging for retryable vs non retryable errors
Browse files Browse the repository at this point in the history
  • Loading branch information
eshwarprasadS committed Jan 10, 2025
1 parent 323ff80 commit 9c5103e
Showing 1 changed file with 18 additions and 3 deletions.
21 changes: 18 additions & 3 deletions src/instructlab/sdg/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@
import logging
import math
import os.path
import requests

# Third Party
from datasets import Dataset, concatenate_datasets
from openai import OpenAI
from openai import OpenAIError
import yaml

# First Party
Expand Down Expand Up @@ -203,9 +205,22 @@ def generate(self, dataset, checkpoint_name=None) -> Dataset:
output_splits.append(ds) # Store the successful result
checkpointer.checkpoint(ds) # Save progress
throttler.adjust_workers(success=True) # Increase workers on success
except Exception as err:
logger.error("Error in pipeline batch generation: %s", err)
throttler.adjust_workers(success=False)

except PipelineBlockError as err:
root_exception = err.exception # Access the underlying exception

if isinstance(root_exception, (requests.exceptions.RequestException, TimeoutError, OpenAIError)):
# Retryable errors
logger.warning("Retryable error in pipeline batch generation: %s", root_exception)
throttler.adjust_workers(success=False)
else:
# Non-retryable errors
logger.error("Non Retryable error in pipeline batch generation: %s", err)
throttler.adjust_workers(success=False)
except Exception as generic_err:
logger.error("Unexpected error in batch generation: %s", generic_err)
throttler.adjust_workers(success=False) # Adjust workers for unexpected errors


checkpointer.done()
if pre_generated_data:
Expand Down

0 comments on commit 9c5103e

Please sign in to comment.