Skip to content

Commit

Permalink
Merge pull request #369 from LeaYeh/ci-action-inputs
Browse files Browse the repository at this point in the history
  • Loading branch information
LeaYeh authored Aug 15, 2024
2 parents a584a34 + d38567e commit d9d490b
Show file tree
Hide file tree
Showing 8 changed files with 125 additions and 65 deletions.
21 changes: 17 additions & 4 deletions .github/actions/setup/action.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
# .github/actions/setup/action.yaml
name: Set Up Test Environment Action
description: Setup action

inputs:
tester_dir:
description: Directory where the tester will be installed to
required: false
scripts_dir:
description: Directory where the scripts for analyzing the test results will be copied to
required: false

runs:
using: composite
steps:
Expand All @@ -17,9 +26,13 @@ runs:
shell: bash
- run: env --list-signal-handling bash
shell: bash
- run: |
git clone https://github.com/LeaYeh/42_minishell_tester.git ${{ env.TESTER_DIR }}
chmod +x ${{ env.TESTER_DIR }}/tester.sh
- if: inputs.tester_dir
run: |
git clone https://github.com/LeaYeh/42_minishell_tester.git ${{ inputs.tester_dir }}
chmod +x ${{ inputs.tester_dir }}/tester.sh
shell: bash
- if: inputs.scripts_dir
run: |
find .github/scripts -type f -name "*.sh" -exec chmod +x {} \;
cp -r .github/scripts ${{ env.SCRIPTS_DIR }}
cp -r .github/scripts ${{ inputs.scripts_dir }}
shell: bash
48 changes: 40 additions & 8 deletions .github/actions/summary_test_result/action.yaml
Original file line number Diff line number Diff line change
@@ -1,16 +1,42 @@
# .github/actions/summary_test_result/action.yaml
name: Test Result Summary Action
description: Compares failed counts between source and target branches

inputs:
source_failed_count:
description: Number of failed test cases on source branch
required: true
target_failed_count:
description: Number of failed test cases on target branch
required: true
source_log_file:
description: Path to the log file for the source branch
required: true
target_log_file:
description: Path to the log file for the target branch
required: true
tester_dir:
description: Directory containing the tester
required: true
tester_output_dir:
description: Directory containing the tester output files
required: true
scripts_dir:
description: Directory containing the scripts for analyzing the test results
required: true

runs:
using: composite
steps:
# Requires SOURCE_FAILED_COUNT and TARGET_FAILED_COUNT environment variables
- name: 🆚 Compare failed count
id: comparison
env:
SOURCE_FAILED_COUNT: ${{ inputs.source_failed_count }}
TARGET_FAILED_COUNT: ${{ inputs.target_failed_count }}
run: |
echo -e "\e[93m🌱 SOURCE_FAILED_COUNT: $SOURCE_FAILED_COUNT\e[0m"
echo -e "\e[94m🎯 TARGET_FAILED_COUNT: $TARGET_FAILED_COUNT\e[0m"
if [ $SOURCE_FAILED_COUNT -gt $TARGET_FAILED_COUNT ]; then
echo -e "\e[93m🌱 SOURCE_FAILED_COUNT: ${{ inputs.source_failed_count }}\e[0m"
echo -e "\e[94m🎯 TARGET_FAILED_COUNT: ${{ inputs.target_failed_count }}\e[0m"
if [ ${{ inputs.source_failed_count }} -gt ${{ inputs.target_failed_count }} ]; then
echo -e "\e[1;31mSOURCE_FAILED_COUNT is greater than TARGET_FAILED_COUNT\e[0m"
exit 1
else
Expand All @@ -19,13 +45,19 @@ runs:
fi
continue-on-error: true
shell: bash
# Requires TESTER_OUTPUT_DIR environment variable
- name: 📈 Show the regressions between source and target branch
if: steps.comparison.outcome == 'failure'
run: ${{ env.SCRIPTS_DIR }}/print_changed_test_cases.sh
run: ${{ inputs.scripts_dir }}/print_changed_test_cases.sh
env:
TESTER_DIR: ${{ inputs.tester_dir }}
TESTER_OUTPUT_DIR: ${{ inputs.tester_output_dir }}
SCRIPTS_DIR: ${{ inputs.scripts_dir }}
shell: bash
# Requires TESTER_OUTPUT_DIR environment variable
- name: 📉 Show the improvements between source and target branch
if: steps.comparison.outcome == 'success'
run: ${{ env.SCRIPTS_DIR }}/print_changed_test_cases.sh || true
run: ${{ inputs.scripts_dir }}/print_changed_test_cases.sh || true
env:
TESTER_DIR: ${{ inputs.tester_dir }}
TESTER_OUTPUT_DIR: ${{ inputs.tester_output_dir }}
SCRIPTS_DIR: ${{ inputs.scripts_dir }}
shell: bash
6 changes: 3 additions & 3 deletions .github/scripts/print_all_failed_test_cases.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/bin/bash

test_output=$(cat "$RESULT_FILE")
test_log=$(cat "$LOG_FILE")
result=0

# Extract line numbers and file paths from test output
# Extract line numbers and file paths from test log
while IFS= read -r line; do
if [[ $line == *""* ]]; then
echo "$line"
Expand All @@ -19,4 +19,4 @@ while IFS= read -r line; do
if [[ $result -ne 0 ]]; then
echo "$line"
fi
done <<< "$test_output"
done <<< "$test_log"
2 changes: 1 addition & 1 deletion .github/scripts/print_changed_test_cases.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

diff_output=$(diff "$HOME/target_test_result.txt" "$HOME/source_test_result.txt" || true)
diff_output=$(diff "$TARGET_LOG_FILE" "$SOURCE_LOG_FILE" || true)

# Extract line numbers and file paths from diff output
while IFS= read -r line; do
Expand Down
6 changes: 3 additions & 3 deletions .github/scripts/print_leak_test_cases.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
#!/bin/bash

test_output=$(cat "$RESULT_FILE")
test_log=$(cat "$LOG_FILE")
leaks=0
result=0

# Extract line numbers and file paths from test output
# Extract line numbers and file paths from test log
while IFS= read -r line; do
stripped_line=$(echo "$line" | sed 's/\x1b\[[0-9;]*m//g')
if [[ $stripped_line == *"LEAKS: ❌"* ]]; then
Expand All @@ -22,7 +22,7 @@ while IFS= read -r line; do
if [[ $result -ne 0 ]]; then
echo "$line"
fi
done <<< "$test_output"
done <<< "$test_log"

if [[ $leaks -ne 0 ]] ; then
exit 1
Expand Down
63 changes: 33 additions & 30 deletions .github/workflows/delivery.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,39 @@ on:
- 'v*'

jobs:
update_evaluation_branch:
check_tag_version:
name: Check Tag Version
runs-on: ubuntu-latest
env:
proceed:
main_tag:
outputs:
main_tag: ${{ steps.check_new_version.outputs.main_tag }}
is_new_version: ${{ steps.check_new_version.outputs.is_new_version }}
steps:
- name: Checkout tag
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Check if new version
id: check_new_version
run: |
main_tag=$(git tag --sort=-creatordate --merged origin/main 2>/dev/null | head -n 1) || true
eval_tag=$(git tag --sort=-creatordate --no-merged origin/main 2>/dev/null | head -n 1) || true
echo "MAIN_TAG: $main_tag"
echo "EVAL_TAG: $eval_tag"
echo "main_tag=$main_tag" >> $GITHUB_OUTPUT
if [[ "$main_tag" > "$eval_tag" ]]; then
echo "is_new_version=true" >> $GITHUB_OUTPUT
else
echo "is_new_version=false" >> $GITHUB_OUTPUT
fi
update_evaluation_branch:
name: Update Evaluation Branch
needs: check_tag_version
if: needs.check_tag_version.outputs.is_new_version == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout code
- name: Checkout tag
uses: actions/checkout@v4
with:
fetch-depth: 0
Expand All @@ -24,63 +49,41 @@ jobs:
git config --local user.email "[email protected]"
git config --local user.name "Lea Yeh"
- name: Check tag version
run: |
MAIN_TAG=$(git tag --sort=-creatordate --merged origin/main 2>/dev/null | head -n 1) || true
EVAL_TAG=$(git tag --sort=-creatordate --no-merged origin/main 2>/dev/null | head -n 1) || true
echo "MAIN_TAG: $MAIN_TAG"
echo "EVAL_TAG: $EVAL_TAG"
echo main_tag=$MAIN_TAG >> "$GITHUB_ENV"
if [[ "$MAIN_TAG" > "$EVAL_TAG" ]]; then
echo "proceed=true" >> "$GITHUB_ENV"
else
echo "proceed=false" >> "$GITHUB_ENV"
fi
- name: Save .github directory
if: env.proceed == 'true'
run: |
cp -rf .github ~/.github
- name: Setup evaluation branch
if: env.proceed == 'true'
run: |
git checkout -b evaluation origin/evaluation || git checkout -b evaluation
git merge origin/main || (git checkout --theirs . && git add -A)
- name: Restore .github directory
if: env.proceed == 'true'
run: |
rm -rf .github
mv ~/.github .github
- name: Partial cleanup of forbidden files
if: env.proceed == 'true'
uses: ./.github/actions/cleanup_partial

- name: Commit partial cleanup to evaluation branch
if: env.proceed == 'true'
run: |
git add -A
git commit -m "[GH-BOT] Remove forbidden files and empty folders"
- name: 📏 Check norminette
if: env.proceed == 'true'
uses: ./.github/actions/norminette

- name: Full cleanup of forbidden files
if: env.proceed == 'true'
uses: ./.github/actions/cleanup_full

- name: Amend full cleanup to evaluation branch
if: env.proceed == 'true'
run: |
git add -A
git commit --amend --no-edit
- name: 🏷️ Set version tag on evaluation branch
if: env.proceed == 'true'
run: |
EVAL_TAG="${{ env.main_tag }}-eval"
git tag "$EVAL_TAG"
git push origin evaluation "$EVAL_TAG"
eval_tag="${{ needs.check_tag_version.outputs.main_tag }}-eval"
git tag "$eval_tag"
git push origin evaluation "$eval_tag"
29 changes: 19 additions & 10 deletions .github/workflows/regression_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@ on:
env:
HOME: /home/runner
TESTER_DIR: /home/runner/42_minishell_tester
TESTER_OUTPUT_DIR: /home/runner/tester_output
SCRIPTS_DIR: /home/runner/scripts
SOURCE_FAILED_COUNT: 0
TARGET_FAILED_COUNT: 0
SOURCE_LOG_FILE: /home/runner/source_test.log
TARGET_LOG_FILE: /home/runner/target_test.log

jobs:
regression_test:
Expand All @@ -41,22 +44,24 @@ jobs:

- name: Set up test environment
uses: ./.github/actions/setup
with:
tester_dir: ${{ env.TESTER_DIR }}
scripts_dir: ${{ env.SCRIPTS_DIR }}

- name: 🌱 Test source branch of pull request
run: |
make re CC=clang-12
${{ env.TESTER_DIR }}/tester.sh --no-update ${{ inputs.test_mode }} > ${{ env.HOME }}/source_test_result.txt
${{ env.TESTER_DIR }}/tester.sh --no-update ${{ inputs.test_mode }} > ${{ env.SOURCE_LOG_FILE }}
env:
GH_BRANCH: SOURCE_FAILED_COUNT

- name: Save tester output to home directory
run: mv -f tester_output* ${{ env.HOME }}/tester_output || true
- name: Save tester output
run: mv -f tester_output* ${{ env.TESTER_OUTPUT_DIR }} || true

- name: 📝 Print all test cases that failed on source branch
run: ${{ env.SCRIPTS_DIR }}/print_all_failed_test_cases.sh
env:
RESULT_FILE: ${{ env.HOME }}/source_test_result.txt
TESTER_OUTPUT_DIR: ${{ env.HOME }}/tester_output
LOG_FILE: ${{ env.SOURCE_LOG_FILE }}

- name: Checkout target branch of pull request
uses: actions/checkout@v4
Expand All @@ -66,7 +71,7 @@ jobs:
- name: 🎯 Test target branch of pull request
run: |
make re CC=clang-12
${{ env.TESTER_DIR }}/tester.sh --no-update ${{ inputs.test_mode }} > ${{ env.HOME }}/target_test_result.txt
${{ env.TESTER_DIR }}/tester.sh --no-update ${{ inputs.test_mode }} > ${{ env.TARGET_LOG_FILE }}
env:
GH_BRANCH: TARGET_FAILED_COUNT

Expand All @@ -75,7 +80,11 @@ jobs:

- name: 📜 Summarize regression test result
uses: ./.github/actions/summary_test_result
env:
SOURCE_FAILED_COUNT: ${{ env.SOURCE_FAILED_COUNT }}
TARGET_FAILED_COUNT: ${{ env.TARGET_FAILED_COUNT }}
TESTER_OUTPUT_DIR: ${{ env.HOME }}/tester_output
with:
source_failed_count: ${{ env.SOURCE_FAILED_COUNT }}
target_failed_count: ${{ env.TARGET_FAILED_COUNT }}
source_log_file: ${{ env.SOURCE_LOG_FILE }}
target_log_file: ${{ env.TARGET_LOG_FILE }}
tester_dir: ${{ env.TESTER_DIR }}
tester_output_dir: ${{ env.TESTER_OUTPUT_DIR }}
scripts_dir: ${{ env.SCRIPTS_DIR }}
15 changes: 9 additions & 6 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,17 @@ jobs:
fail-fast: false
matrix:
test_script: ${{ fromJson(needs.prepare_test_matrix.outputs.test_matrix) }}
env:
TESTER_OUTPUT_DIR: /home/runner/tester_output
LOG_FILE: /home/runner/leak_test.log
steps:
- name: Checkout source branch of pull request
uses: actions/checkout@v4
- name: Set up test environment
uses: ./.github/actions/setup
with:
tester_dir: ${{ env.TESTER_DIR }}
scripts_dir: ${{ env.SCRIPTS_DIR }}
- name: Delete all but one test script
run: |
for file in ${{ env.TESTER_DIR }}/cmds/**/*.sh; do
Expand All @@ -79,17 +85,14 @@ jobs:
- name: 🔍 Check memory leaks
run: |
make re CC=clang-12
${{ env.TESTER_DIR }}/tester.sh --no-update va > ${{ env.HOME }}/leak_result.txt
${{ env.TESTER_DIR }}/tester.sh --no-update va > ${{ env.LOG_FILE }}
env:
GH_BRANCH: IGNORE
continue-on-error: true
- name: Save tester output to home directory
run: mv -f tester_output* ${{ env.HOME }}/tester_output || true
- name: Save tester output
run: mv -f tester_output* ${{ env.TESTER_OUTPUT_DIR }} || true
- name: 📝 Print all test cases that leaked on source branch
run: ${{ env.SCRIPTS_DIR }}/print_leak_test_cases.sh
env:
RESULT_FILE: ${{ env.HOME }}/leak_result.txt
TESTER_OUTPUT_DIR: ${{ env.HOME }}/tester_output

combine_memory_leak_test_results:
name: Combine Memory Leak Test Results
Expand Down

0 comments on commit d9d490b

Please sign in to comment.