test: add prompt eval test #703
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "CI" | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - dev | |
| push: | |
| branches: | |
| - main | |
| - dev | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: lts/* | |
| - name: Setup project | |
| run: | | |
| npm install | |
| npm run package | |
| - name: Check lint | |
| run: | | |
| npm run lint | |
| semver-title: | |
| runs-on: ubuntu-latest | |
| if: (github.event_name == 'pull_request') | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: amannn/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| unit-test: | |
| runs-on: windows-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: checkout PR | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Merge Branches | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "bot" | |
| git remote add ${{ github.event.pull_request.base.repo.owner.login }} ${{ github.event.pull_request.base.repo.clone_url }} | |
| git fetch ${{ github.event.pull_request.base.repo.owner.login }} | |
| git merge ${{ github.event.pull_request.base.repo.owner.login }}/${{ github.event.pull_request.base.ref }} | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: lts/* | |
| - name: run test | |
| run: | | |
| npm install | |
| npm run test | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODE_COV }} | |
| fail_ci_if_error: true | |
| verbose: true | |
| prompt-evaluation: | |
| runs-on: ubuntu-latest | |
| environment: engineering | |
| permissions: | |
| contents: read # Required for actions/checkout | |
| pull-requests: write # Ability to post comments on Pull Requests | |
| id-token: write # Required for Azure login | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: lts/* | |
| - name: Setup project | |
| run: | | |
| npm install | |
| - name: Azure login | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.EVALUATION_TEST_CLIENT_ID }} | |
| tenant-id: ${{ secrets.EVALUATION_TEST_TENANT_ID }} | |
| subscription-id: ${{ secrets.EVALUATION_TEST_SUB_ID }} | |
| enable-AzPSSession: true | |
| - name: Set up promptfoo cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/promptfoo | |
| key: ${{ runner.os }}-promptfoo-v1 | |
| restore-keys: | | |
| ${{ runner.os }}-promptfoo- | |
| - name: Prompt Evaluation | |
| continue-on-error: true | |
| env: | |
| PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo | |
| run: | | |
| for dir in evalprompt/*/; do | |
| if [ -d "$dir" ] && [ "$(basename "$dir")" != "example" ]; then | |
| cd "$dir" || continue | |
| npx promptfoo eval -o output.json --share || true | |
| cd - > /dev/null || exit | |
| fi | |
| done | |
| - name: generate comments doc | |
| run: | | |
| node .github/workflows/scripts/collectPromptRes.js | |
| cat ./commentContent.txt | |
| - uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const fs = require('fs'); | |
| const content = fs.readFileSync('./commentContent.txt', 'utf8'); | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: content | |
| }) | |