ci/transformers: add baseline checks for test cases

Signed-off-by: Dmitry Rogozhkin <[email protected]>
intel · Dec 24, 2024 · a896289 · a896289
1 parent bc99386
commit a896289
Showing 1 changed file with 16 additions and 0 deletions.
diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
@@ -136,13 +136,19 @@ jobs:
           source activate huggingface_transformers_test
           cd transformers
           python3 -m pytest -rsf --make-reports=$TEST_CASE tests/*.py || true
+          (cat reports/$TEST_CASE/stats.txt | grep errors) && failed=1
+          (cat reports/$TEST_CASE/stats.txt | grep "10 failed") || failed=1
+          [[ "$failed" = 1 ]] && (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) || true
       - name: Run tests/benchmark
         env:
           TEST_CASE: 'tests_benchmark'
         run: |
           source activate huggingface_transformers_test
           cd transformers
           python3 -m pytest -rsf --make-reports=$TEST_CASE tests/benchmark || true
+          (cat reports/$TEST_CASE/stats.txt | grep errors) && failed=1
+          (cat reports/$TEST_CASE/stats.txt | grep "13 failed") || failed=1
+          [[ "$failed" = 1 ]] && (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) || true
       - name: Run tests/generation
         env:
           TEST_CASE: 'tests_generation'
@@ -153,6 +159,9 @@ jobs:
           # * torch.distributed.* not yet supported by XPU
           pattern="not TestFSDPGeneration"
           python3 -m pytest -rsf --make-reports=$TEST_CASE tests/generation -k "$pattern" || true
+          (cat reports/$TEST_CASE/stats.txt | grep errors) && failed=1
+          (cat reports/$TEST_CASE/stats.txt | grep "17 failed") || failed=1
+          [[ "$failed" = 1 ]] && (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) || true
       - name: Run tests/models
         env:
           TEST_CASE: 'tests_models'
@@ -168,6 +177,10 @@ jobs:
             not test_resize_embeddings_untied and \
             not test_resize_tokens_embeddings"
           python3 -m pytest -rsf --make-reports=$TEST_CASE tests/models -k "$pattern" || true
+          (cat reports/$TEST_CASE/stats.txt | grep errors) && failed=1
+          # TODO: establish baseline with failed tests after resolving random failures
+          #(cat reports/$TEST_CASE/stats.txt | grep "10 failed") || failed=1
+          [[ "$failed" = 1 ]] && (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) || true
       - name: Run tests/pipelines
         env:
           TEST_CASE: 'tests_pipelines'
@@ -177,6 +190,9 @@ jobs:
           # Some tests are known to fail w/o clear pattern
           # TODO: drop ||true after triage and fixes
           python3 -m pytest -rsf --make-reports=$TEST_CASE tests/pipelines || true
+          (cat reports/$TEST_CASE/stats.txt | grep errors) && failed=1
+          (cat reports/$TEST_CASE/stats.txt | grep "11 failed") || failed=1
+          [[ "$failed" = 1 ]] && (echo "FAILED_CASES=$FAILED_CASES,$TEST_CASE" >> $GITHUB_ENV) || true
       - name: Run tests/trainer
         env:
           TEST_CASE: 'tests_trainer'