diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74ee548..1731ee2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,8 @@ on: jobs: publish: runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Create a GitHub release diff --git a/.github/workflows/sync-purls.yml b/.github/workflows/sync-purls.yml new file mode 100644 index 0000000..b1261dc --- /dev/null +++ b/.github/workflows/sync-purls.yml @@ -0,0 +1,101 @@ +name: Collect latest PURLs from FederatedCode and create release with latest FST + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +permissions: + contents: write + +jobs: + collect-purls: + strategy: + max-parallel: 1 + matrix: + include: + - ecosystem: apk + - ecosystem: cargo + - ecosystem: composer + - ecosystem: conan + - ecosystem: cpan + - ecosystem: cran + - ecosystem: debain + - ecosystem: maven + - ecosystem: npm + - ecosystem: nuget + - ecosystem: pypi + - ecosystem: swift + + uses: aboutcode-org/purl-validator.rs/.github/workflows/collect-purls_template.yml + with: + ecosystem: ${{ matrix.ecosystem }} + path: "cmd/data/${{ matrix.ecosystem }}.txt" + + regen-fst-and-release: + name: Regenerate FST and create release using new FST + needs: collect-purls + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_TAG_RELEASE_TOKEN }} + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: 'stable' + + - name: Install dependencies + run: make dev + + - name: Regenerate FST + id: regen_fst + run: |- + git pull + make build-fst + git diff --name-only | grep -q 'purls.fst' && echo "changed=true" >> $GITHUB_OUTPUT \ + || echo "changed=false" >> $GITHUB_OUTPUT + + # Commit latest FST + git config user.name "AboutCode Automation" + git config user.email "automation@aboutcode.org" + git add purls.fst + git commit -m "$(echo -e "Regenerate FST using latest PURLs\n\nSigned-off-by: AboutCode Automation ")" || exit 0 + git push + + - name: Bump minor version + id: bump + if: steps.regen_fst.outputs.changed == 'true' + run: |- + new_version=$(git tag --sort=version:refname | tail -n1 | awk -F'[v.]' '{ printf "v%d.%d.0\n", $2, $3+1 }') + + echo "new_version=$new_version" >> $GITHUB_OUTPUT + + - name: Add Changelog + if: steps.regen_fst.outputs.changed == 'true' + run: |- + # Add Changelog + today=$(date +%Y-%m-%d) + awk -v ver="${{ steps.bump.outputs.new_version }}" -v date="$today" ' + NR==1{ + print "# Changelog\n\n## v" ver " (" date ")\n\n - Update FST with latest PURLs" + next + }1' CHANGELOG.md > .tmp.CHANGELOG.md && mv .tmp.CHANGELOG.md CHANGELOG.md + + - name: Commit Changelog + if: steps.regen_fst.outputs.changed == 'true' + run: |- + git config user.name "AboutCode Automation" + git config user.email "automation@aboutcode.org" + git add -A + git commit -m "$(echo -e "Bump version for v${{ steps.bump.outputs.new_version }} release\n\nSigned-off-by: AboutCode Automation ")" || exit 0 + git push + + - name: Push tag + if: steps.regen_fst.outputs.changed == 'true' + run: |- + # Push tag + git tag -a "v${{ steps.bump.outputs.new_version }}" -m "Release v${{ steps.bump.outputs.new_version }}" + git push origin "v${{ steps.bump.outputs.new_version }}" diff --git a/README.md b/README.md index 9bfe287..f5ba270 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,18 @@ ## Currently Supported Ecosystems -- **nuget**: [https://www.nuget.org/](https://www.nuget.org/) +- **apk** +- **cargo** +- **composer** +- **conan** +- **cpan** +- **cran** +- **debain** +- **maven** +- **npm** +- **nuget** +- **pypi** +- **swift** ## Usage diff --git a/cmd/data/testpurls.txt b/cmd/data/testpurls.txt deleted file mode 100644 index a29c6af..0000000 --- a/cmd/data/testpurls.txt +++ /dev/null @@ -1,4 +0,0 @@ -pkg:nuget/FluentUtils.EnumExtensions -pkg:nuget/FluentUtils.FromCompositeAttribute -pkg:nuget/FluentUtils.MediatR.Pagination -pkg:nuget/FluentUtils.MediatR.Pagination.AspNetCore \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index 77a2fd9..fd09a09 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -14,6 +14,7 @@ package main import ( "log" "os" + "path/filepath" "sort" "strings" @@ -21,32 +22,27 @@ import ( ) func main() { - data, _ := os.ReadFile("cmd/data/purls.txt") - lines := strings.FieldsFunc(string(data), func(r rune) bool { - return r == '\n' - }) - sort.Strings(lines) - output := strings.Join(lines, "\n") - - // #nosec G306 - err := os.WriteFile("cmd/data/purls.txt", []byte(output), 0644) + f, err := os.Create("purls.fst") + var purlCount int if err != nil { - panic(err) + log.Fatal(err) } - f, err := os.Create("purls.fst") + dirname := "cmd/data/" + entries, err := os.ReadDir(dirname) if err != nil { log.Fatal(err) } + builder, err := vellum.New(f, nil) if err != nil { log.Fatal(err) } - for _, line := range lines { - err = builder.Insert([]byte(line), 0) - if err != nil { - log.Fatal(err) + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(strings.ToLower(entry.Name()), ".txt") { + fullPath := filepath.Join(dirname, entry.Name()) + purlCount += insert_purls(builder, fullPath) } } @@ -54,4 +50,25 @@ func main() { if err != nil { log.Fatal(err) } + log.Printf("FST generated with %d base PackageURLs", purlCount) + log.Printf("FST generated at %s", f.Name()) +} + +func insert_purls(builder *vellum.Builder, file string) int { + var err error + // #nosec G304 + data, _ := os.ReadFile(file) + lines := strings.FieldsFunc(string(data), func(r rune) bool { + return r == '\n' + }) + sort.Strings(lines) + + log.Printf("Insert PURLs from %s in FST", file) + for _, line := range lines { + err = builder.Insert([]byte(line), 0) + if err != nil { + log.Fatal(err) + } + } + return len(lines) }