diff --git a/AUTHORS.rst b/AUTHORS.rst index 16e2046..8bd7b6f 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,4 @@ The following organizations or individuals have contributed to this repo: -- +- AboutCode.org contributors +- Kaushik Kumar diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fc2b6e3..ed95994 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,4 +5,7 @@ Changelog v0.0.0 ------ -*xxxx-xx-xx* -- Initial release. +*Unreleased* + +- Add required phrase dataset extraction. +- Add composite rule required phrase updates. diff --git a/MANIFEST.in b/MANIFEST.in index 0f19707..8767f1e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ graft src graft docs +prune docs/_build graft etc include *.LICENSE diff --git a/Makefile b/Makefile index 3041547..e8e9ea9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/README.rst b/README.rst index 11a4dfb..c2e65c0 100644 --- a/README.rst +++ b/README.rst @@ -1,107 +1,51 @@ -A Simple Python Project Skeleton -================================ +ScanCode Required Phrases +========================= -This repo attempts to standardize the structure of the Python-based project's -repositories using modern Python packaging and configuration techniques. -Using this `blog post`_ as inspiration, this repository serves as the base for -all new Python projects and is mergeable in existing repositories as well. +``scancode-required-phrases`` provides commands for working with required +phrases in ScanCode license rules. ScanCode Toolkit provides the rule models, +tokenization, matching, and validation used by these commands. -.. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ +Installation +============ +Install this package in a Python 3.10 or newer environment: -Usage -===== +.. code-block:: console -A brand new project -------------------- + python -m pip install . -.. code-block:: bash +Build a dataset +=============== - git init my-new-repo - cd my-new-repo - git pull git@github.com:nexB/skeleton +.. code-block:: console - # Create the new repo on GitHub, then update your remote - git remote set-url origin git@github.com:nexB/your-new-repo.git + build-required-phrases-dataset --rules-dir path/to/rules --output-dir dataset-output -From here, you can make the appropriate changes to the files for your specific project. +The command writes ``train.jsonl``, ``val.jsonl``, and ``test.jsonl``. If +``--rules-dir`` is omitted, the installed ScanCode rules directory is used. -Update an existing project ---------------------------- +Update composite rules +====================== -.. code-block:: bash +.. code-block:: console - cd my-existing-project - git remote add skeleton git@github.com:nexB/skeleton - git fetch skeleton - git merge skeleton/main --allow-unrelated-histories + add-composite-required-phrases --dry-run --verbose -This is also the workflow to use when updating the skeleton files in any given repository. +The command uses existing required phrases from single-key rules to update +composite rules. A rule is updated only when every relevant license key has a +non-overlapping match. It operates on the installed ScanCode rules directory. -More usage instructions can be found in ``docs/skeleton-usage.rst``. +Development +=========== +Create a development environment and run the tests: -Release Notes -============= +.. code-block:: console -- 2025-03-31: + configure --dev + venv\Scripts\pytest - - Use ruff as the main code formatting tool, add ruff rules to pyproject.toml +On POSIX systems, run ``./configure --dev`` and ``venv/bin/pytest``. -- 2025-03-29: - - - Add support for beta macOS-15 - - Add support for beta windows-2025 - -- 2025-02-14: - - - Drop support for Python 3.8, add support in CI for Python 3.13, use Python 3.12 as default - version. - -- 2025-01-17: - - - Drop support for macOS-12, add support for macOS-14 - - Add support in CI for ubuntu-24.04 - - Add support in CI for Python 3.12 - -- 2024-08-20: - - - Update references of ownership from nexB to aboutcode-org - -- 2024-07-01: - - - Drop support for Python 3.8 - - Drop support for macOS-11, add support for macOS-14 - -- 2024-02-19: - - - Replace support in CI of default ubuntu-20.04 by ubuntu-22.04 - -- 2023-10-18: - - - Add dark mode support in documentation - -- 2023-07-18: - - - Add macOS-13 job in azure-pipelines.yml - -- 2022-03-04: - - - Synchronize configure and configure.bat scripts for sanity - - Update CI operating system support with latest Azure OS images - - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party - dependencies. There are now fewer scripts. See etc/scripts/README.rst for details - -- 2021-09-03: - - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` - and ``requirements-dev.txt`` - - ``configure`` can now accept multiple options at once - - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - - Rename virtual environment directory from ``tmp`` to ``venv`` - - Update README.rst with instructions for generating ``requirements.txt`` - and ``requirements-dev.txt``, as well as collecting dependencies as wheels and generating - ABOUT files for them. - -- 2021-05-11: - - Adopt new configure scripts from ScanCode TK that allows correct configuration of which - Python version is used. +See ``docs/source/dataset.rst`` and ``docs/source/composite_rules.rst`` for +details. diff --git a/docs/source/composite_rules.rst b/docs/source/composite_rules.rst new file mode 100644 index 0000000..a43fa97 --- /dev/null +++ b/docs/source/composite_rules.rst @@ -0,0 +1,34 @@ +Update composite rules +====================== + +The ``add-composite-required-phrases`` command adds existing required phrases +to ScanCode rules with multi-key license expressions. + +Run the command +--------------- + +Preview changes without saving rules: + +.. code-block:: console + + add-composite-required-phrases --dry-run --verbose + +Use ``--license-expression`` to process one expression. Use +``--write-phrase-source`` to record the source required phrase rules in updated +rules. + +Matching +-------- + +Candidates come from existing ``is_required_phrase`` rules with one +non-generic license key. A composite rule is updated only when every relevant +key has a non-overlapping phrase match. Existing required phrase markers are +kept, and filenames and URLs are not changed. + +Writing and validation +---------------------- + +The command operates on the installed ScanCode rules directory. Without +``--dry-run``, each changed rule is written once after all required phrases are +added. Use ``--validate`` to validate the rules and licenses and ``--reindex`` +to rebuild the cached license index after updating. diff --git a/docs/source/conf.py b/docs/source/conf.py index 056ca6e..409b8e8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,7 @@ # -- Project information ----------------------------------------------------- -project = "nexb-skeleton" +project = "ScanCode Required Phrases" copyright = "nexB Inc., AboutCode and others." author = "AboutCode.org authors and contributors" @@ -79,9 +79,9 @@ html_context = { "display_github": True, - "github_user": "nexB", - "github_repo": "nexb-skeleton", - "github_version": "develop", # branch + "github_user": "aboutcode-org", + "github_repo": "scancode-required-phrases", + "github_version": "main", # branch "conf_py_path": "/docs/source/", # path in the checkout to the docs root } diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 2a719a5..316f81f 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -10,9 +10,9 @@ Setup Local Build To get started, check out and configure the repository for development:: - git clone https://github.com/aboutcode-org/.git + git clone https://github.com/aboutcode-org/scancode-required-phrases.git - cd your-repo + cd scancode-required-phrases ./configure --dev (Or use "make dev") @@ -55,7 +55,7 @@ documentation standards are enforced. We checks for these aspects of the documen You myst run these scripts locally before creating a pull request:: make doc8 - make check-docs + make docs-check .. _doc_style_docs8: diff --git a/docs/source/dataset.rst b/docs/source/dataset.rst new file mode 100644 index 0000000..89d5edf --- /dev/null +++ b/docs/source/dataset.rst @@ -0,0 +1,43 @@ +Build a required phrase dataset +=============================== + +The ``build-required-phrases-dataset`` command reads ScanCode ``.RULE`` files +and creates a BIOES-labelled JSONL dataset from required phrases marked with +``{{ }}``. + +Run the command +--------------- + +.. code-block:: console + + build-required-phrases-dataset \ + --rules-dir path/to/rules \ + --output-dir dataset-output + +``--rules-dir`` defaults to the installed ScanCode rules directory. +``--output-dir`` defaults to ``dataset-output``. + +Output +------ + +The output directory contains: + +* ``train.jsonl`` +* ``val.jsonl`` +* ``test.jsonl`` + +Each record contains the rule identifier, license expression, rule type, +unmarked text, tokens, and BIOES labels. + +Splitting +--------- + +Rules with common license expressions are assigned by a deterministic hash of +their identifiers. Rules with rarer expressions stay together in one split. +The target proportions are 80 percent training, 10 percent validation, and +10 percent test. + +Only eligible rules containing a marked required phrase are included. +Dedicated ``is_required_phrase`` rules, deprecated rules, false positives, +license clues, license introductions, and rules without a license expression +are excluded. diff --git a/docs/source/index.rst b/docs/source/index.rst index eb63717..daf7c60 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,11 +1,15 @@ -Welcome to nexb-skeleton's documentation! -========================================= +ScanCode Required Phrases +========================= + +Commands for building required phrase datasets and updating composite ScanCode +license rules. .. toctree:: :maxdepth: 2 :caption: Contents: - skeleton-usage + dataset + composite_rules contribute/contrib_doc Indices and tables diff --git a/docs/source/skeleton-usage.rst b/docs/source/skeleton-usage.rst deleted file mode 100644 index 6cb4cc5..0000000 --- a/docs/source/skeleton-usage.rst +++ /dev/null @@ -1,160 +0,0 @@ -Usage -===== -A brand new project -------------------- -.. code-block:: bash - - git init my-new-repo - cd my-new-repo - git pull git@github.com:nexB/skeleton - - # Create the new repo on GitHub, then update your remote - git remote set-url origin git@github.com:nexB/your-new-repo.git - -From here, you can make the appropriate changes to the files for your specific project. - -Update an existing project ---------------------------- -.. code-block:: bash - - cd my-existing-project - git remote add skeleton git@github.com:nexB/skeleton - git fetch skeleton - git merge skeleton/main --allow-unrelated-histories - -This is also the workflow to use when updating the skeleton files in any given repository. - -Customizing ------------ - -You typically want to perform these customizations: - -- remove or update the src/README.rst and tests/README.rst files -- set project info and dependencies in setup.cfg -- check the configure and configure.bat defaults - -Initializing a project ----------------------- - -All projects using the skeleton will be expected to pull all of it dependencies -from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using -requirements.txt and/or requirements-dev.txt to determine what version of a -package to collect. By default, PyPI will not be used to find and collect -packages from. - -In the case where we are starting a new project where we do not have -requirements.txt and requirements-dev.txt and whose dependencies are not yet on -thirdparty.aboutcode.org/pypi, we run the following command after adding and -customizing the skeleton files to your project: - -.. code-block:: bash - - ./configure - -This will initialize the virtual environment for the project, pull in the -dependencies from PyPI and add them to the virtual environment. - - -Generating requirements.txt and requirements-dev.txt ----------------------------------------------------- - -After the project has been initialized, we can generate the requirements.txt and -requirements-dev.txt files. - -Ensure the virtual environment is enabled. - -.. code-block:: bash - - source venv/bin/activate - -To generate requirements.txt: - -.. code-block:: bash - - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ - -Replace \ with the version number of the Python being used, for example: -``venv/lib/python3.6/site-packages/`` - -To generate requirements-dev.txt after requirements.txt has been generated: - -.. code-block:: bash - - ./configure --dev - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ - -Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` - -.. code-block:: bash - - python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ - .\configure --dev - python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ - - -Collecting and generating ABOUT files for dependencies ------------------------------------------------------- - -Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed: - -.. code-block:: bash - - pip install -r etc/scripts/requirements.txt - -Once we have requirements.txt and requirements-dev.txt, we can fetch the project -dependencies as wheels and generate ABOUT files for them: - -.. code-block:: bash - - python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt - -There may be issues with the generated ABOUT files, which will have to be -corrected. You can check to see if your corrections are valid by running: - -.. code-block:: bash - - python etc/scripts/check_thirdparty.py -d thirdparty - -Once the wheels are collected and the ABOUT files are generated and correct, -upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT -files from the thirdparty directory to the pypi directory at -https://github.com/aboutcode-org/thirdparty-packages - - -Usage after project initialization ----------------------------------- - -Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated -and the project dependencies and their ABOUT files have been uploaded to -thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically -when you update dependencies or use a new checkout. - -If the virtual env for the project becomes polluted, or you would like to remove -it, use the ``--clean`` option: - -.. code-block:: bash - - ./configure --clean - -Then you can run ``./configure`` again to set up the project virtual environment. - -To set up the project for development use: - -.. code-block:: bash - - ./configure --dev - -To update the project dependencies (adding, removing, updating packages, etc.), -update the dependencies in ``setup.cfg``, then run: - -.. code-block:: bash - - ./configure --clean # Remove existing virtual environment - source venv/bin/activate # Ensure virtual environment is activated - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt - pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py - python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files - -Ensure that the generated ABOUT files are valid, then take the dependency wheels -and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 65ae595..521b234 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 76a19a6..5c64efb 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 1b87944..7e6b6d0 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import argparse diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 8548205..384fdd4 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import argparse diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py index 374c06f..9f2865b 100644 --- a/etc/scripts/update_skeleton.py +++ b/etc/scripts/update_skeleton.py @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index b6bff51..b64ea87 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import io diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index f377578..c5d0987 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index bc68ac7..89eb52e 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import email diff --git a/pyproject.toml b/pyproject.toml index f106e69..39bc302 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] force-single-line = true lines-after-imports = 1 default-section = "first-party" -known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-first-party = ["scancode_required_phrases", "tests", "etc/scripts/**/*.py"] known-third-party = ["click", "pytest"] sections = { django = ["django"] } diff --git a/setup.cfg b/setup.cfg index a0f2985..fe37819 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,18 +1,19 @@ [metadata] -name = skeleton +name = scancode-required-phrases +version = 0.0.0 license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 -description = skeleton +description = Work with required phrases in ScanCode license rules long_description = file:README.rst long_description_content_type = text/x-rst -url = https://github.com/aboutcode-org/skeleton +url = https://github.com/aboutcode-org/scancode-required-phrases author = nexB. Inc. and others author_email = info@aboutcode.org classifiers = - Development Status :: 5 - Production/Stable + Development Status :: 3 - Alpha Intended Audience :: Developers Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only @@ -20,7 +21,9 @@ classifiers = Topic :: Utilities keywords = - utilities + scancode + license + required phrases license_files = apache-2.0.LICENSE @@ -40,12 +43,21 @@ include_package_data = true zip_safe = false install_requires = + click == 8.3.2 + license-expression == 30.4.4 + scancode-toolkit == 32.5.0 [options.packages.find] where = src +[options.entry_points] +console_scripts = + add-composite-required-phrases = scancode_required_phrases.composite_rules:add_composite_required_phrases + build-required-phrases-dataset = scancode_required_phrases.dataset:main + + [options.extras_require] dev = pytest >= 7.0.1 diff --git a/src/README.rst b/src/README.rst deleted file mode 100644 index ec651fc..0000000 --- a/src/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python source code (and installable data) in this directory. - diff --git a/src/scancode_required_phrases/__init__.py b/src/scancode_required_phrases/__init__.py new file mode 100644 index 0000000..3eb3464 --- /dev/null +++ b/src/scancode_required_phrases/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tools for working with required phrases in ScanCode license rules.""" diff --git a/src/scancode_required_phrases/composite_rules.py b/src/scancode_required_phrases/composite_rules.py new file mode 100644 index 0000000..0b844f3 --- /dev/null +++ b/src/scancode_required_phrases/composite_rules.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +"""Add existing required phrases to composite ScanCode license rules.""" + +import click +from license_expression import Licensing + +from licensedcode.cache import get_licenses_db +from licensedcode.models import rules_data_dir +from licensedcode.required_phrases import add_required_phrase_to_rule +from licensedcode.required_phrases import collect_is_required_phrase_from_rules +from licensedcode.required_phrases import find_phrase_spans_in_text +from licensedcode.required_phrases import get_base_rules_by_expression +from licensedcode.required_phrases import get_ignorable_spans +from licensedcode.required_phrases import get_non_overlapping_spans +from licensedcode.required_phrases import get_updatable_rules_by_expression +from licensedcode.required_phrases import validate_and_reindex +from licensedcode.tokenize import get_existing_required_phrase_spans + + +def get_required_phrases_by_key(rules_by_expression, licenses_by_key): + """ + Return required phrase candidates grouped by license key. + Only collect from required phrase rules with a single non-generic key. + """ + licensing = Licensing() + required_phrases_by_key = {} + required_phrases_by_expression = collect_is_required_phrase_from_rules( + rules_by_expression=rules_by_expression, + ) + + for expression, required_phrases in required_phrases_by_expression.items(): + license_keys = licensing.license_keys(expression, unique=True) + if len(license_keys) != 1: + continue + + license_key = license_keys[0] + if licenses_by_key[license_key].is_generic: + continue + + if required_phrases: + required_phrases_by_key[license_key] = required_phrases + + return required_phrases_by_key + + +def _get_required_phrase_matches(rule, license_keys, required_phrases_by_key): + """ + Return one non-overlapping required phrase match for every license key. + Prefer phrases already marked in the rule and return None if no complete match exists. + """ + existing_spans = get_existing_required_phrase_spans(rule.text) + unavailable_spans = existing_spans + get_ignorable_spans(rule) + matches_by_key = {} + + for license_key in license_keys: + marked_matches = [] + new_matches = [] + + for candidate in required_phrases_by_key.get(license_key, []): + phrase_spans = find_phrase_spans_in_text( + rule.text, + candidate.required_phrase_text, + ) + marked_spans = [ + span + for span in phrase_spans + if any(span in existing for existing in existing_spans) + ] + if marked_spans: + marked_matches.extend((candidate, True, [span]) for span in marked_spans) + continue + + spans_to_add = list( + get_non_overlapping_spans( + old_required_phrase_spans=unavailable_spans, + new_required_phrase_spans=phrase_spans, + ) + ) + if spans_to_add: + new_matches.append((candidate, False, spans_to_add)) + + matches_by_key[license_key] = marked_matches + new_matches + if not matches_by_key[license_key]: + return + + def find_matches(remaining_keys, matched_spans): + if not remaining_keys: + return [] + + license_key = remaining_keys[0] + for required_phrase, is_marked, phrase_spans in matches_by_key[license_key]: + if any(span.overlap(matched) for span in phrase_spans for matched in matched_spans): + continue + + remaining_matches = find_matches( + remaining_keys=remaining_keys[1:], + matched_spans=matched_spans + phrase_spans, + ) + if remaining_matches is not None: + return [ + (required_phrase, is_marked), + *remaining_matches, + ] + + return find_matches( + remaining_keys=license_keys, + matched_spans=[], + ) + + +def add_required_phrases_to_composite_rules( + rules, + license_keys, + required_phrases_by_key, + write_phrase_source=False, + dry_run=False, +): + """Add required phrases to rules when every license key has a matching phrase.""" + for rule in rules: + matched_required_phrases = _get_required_phrase_matches( + rule=rule, + license_keys=license_keys, + required_phrases_by_key=required_phrases_by_key, + ) + if not matched_required_phrases: + continue + + original_text = rule.text + original_source = rule.source + updated = False + + for required_phrase, is_marked in matched_required_phrases: + if is_marked: + continue + + source = rule.source + if write_phrase_source: + phrase_source = required_phrase.rule.identifier + source = f"{source} {phrase_source}" if source else phrase_source + + added = add_required_phrase_to_rule( + rule=rule, + required_phrase=required_phrase.required_phrase_text, + source=source, + dry_run=True, + ) + if not added: + rule.text = original_text + rule.source = original_source + updated = False + break + + updated = True + + if updated and not dry_run: + rule.dump(rules_data_dir) + + +def update_composite_rules_using_required_phrases( + license_expression=None, + write_phrase_source=False, + verbose=False, + dry_run=False, +): + """ + Add existing required phrases to composite rules when every non-generic + license key has a non-overlapping match. + """ + licensing = Licensing() + licenses_by_key = get_licenses_db() + rules_by_expression = get_base_rules_by_expression() + required_phrases_by_key = get_required_phrases_by_key( + rules_by_expression=rules_by_expression, + licenses_by_key=licenses_by_key, + ) + updatable_rules_by_expression = get_updatable_rules_by_expression( + license_expression=license_expression, + simple_expression=False, + ) + + for expression, updatable_rules in updatable_rules_by_expression.items(): + license_keys = licensing.license_keys(expression, unique=True) + if len(license_keys) < 2: + continue + + license_keys = [ + license_key + for license_key in license_keys + if not licenses_by_key[license_key].is_generic + ] + if not license_keys: + continue + + if verbose: + click.echo(f"Annotating required phrases for expression: {expression}") + + add_required_phrases_to_composite_rules( + rules=updatable_rules, + license_keys=license_keys, + required_phrases_by_key=required_phrases_by_key, + write_phrase_source=write_phrase_source, + dry_run=dry_run, + ) + + +@click.command(name="add-composite-required-phrases") +@click.option( + "-l", + "--license-expression", + type=str, + default=None, + metavar="STRING", + help="Only update rules using this license expression.", +) +@click.option( + "-w", + "--write-phrase-source", + is_flag=True, + default=False, + help="Record the source required phrase rule in modified rules.", +) +@click.option( + "--validate", + is_flag=True, + default=False, + help="Validate all rules and licenses after updating.", +) +@click.option( + "-r", + "--reindex", + is_flag=True, + default=False, + help="Rebuild and cache the license index after updating.", +) +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Do not save rules.", +) +@click.option( + "-v", + "--verbose", + is_flag=True, + default=False, + help="Print progress information.", +) +@click.help_option("-h", "--help") +def add_composite_required_phrases( + license_expression, + write_phrase_source, + validate, + reindex, + dry_run, + verbose, +): + """Add existing required phrases to composite license rules.""" + click.echo("Updating composite rules from required phrases.") + update_composite_rules_using_required_phrases( + license_expression=license_expression, + write_phrase_source=write_phrase_source, + dry_run=dry_run, + verbose=verbose, + ) + validate_and_reindex( + validate=validate, + reindex=reindex, + verbose=verbose, + ) + + +if __name__ == "__main__": + add_composite_required_phrases() diff --git a/src/scancode_required_phrases/dataset.py b/src/scancode_required_phrases/dataset.py new file mode 100644 index 0000000..4313688 --- /dev/null +++ b/src/scancode_required_phrases/dataset.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/scancode-required-phrases for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +"""Build a BIOES dataset from required phrases marked in license rules.""" + +from collections import Counter +import hashlib +import json +from pathlib import Path +import unicodedata + +import click + +from licensedcode.models import load_rules +from licensedcode.models import rules_data_dir as default_rules_data_dir +from licensedcode.required_phrases import get_required_phrase_verbatim +from licensedcode.tokenize import get_existing_required_phrase_spans +from licensedcode.tokenize import required_phrase_splitter + + +def get_rule_type(rule): + """Return the first license rule type set on ``rule``.""" + for flag in rule.license_flag_names: + if getattr(rule, flag): + return flag + if rule.is_false_positive: + return "is_false_positive" + return "unknown" + + +def tag_tokens(text): + """Return rule text tokens and their required phrase BIOES labels.""" + tokens = [] + labels = [] + in_phrase = False + phrase_length = 0 + + for token in required_phrase_splitter(text): + if token == "{{": + in_phrase = True + phrase_length = 0 + continue + + if token == "}}": + if in_phrase and phrase_length: + labels[-1] = "S-REQ" if phrase_length == 1 else "E-REQ" + in_phrase = False + phrase_length = 0 + continue + + tokens.append(token) + if in_phrase: + labels.append("B-REQ" if phrase_length == 0 else "I-REQ") + phrase_length += 1 + else: + labels.append("O") + + return tokens, labels + + +def build_record(rule): + """Return a dataset record for an eligible annotated rule, or None.""" + if ( + rule.is_required_phrase + or rule.is_false_positive + or rule.is_license_intro + or rule.is_license_clue + or rule.is_deprecated + or not rule.license_expression + or not rule.text + ): + return + + text = rule.text.replace("\r\n", "\n").replace("\r", "\n") + text = unicodedata.normalize("NFKC", text) + + # Fail on invalid nested, empty, or dangling required phrase markers. + get_existing_required_phrase_spans(text) + if not any(get_required_phrase_verbatim(text)): + return + + tokens, bioes_labels = tag_tokens(text) + return { + "identifier": rule.identifier, + "license_expression": rule.license_expression or "", + "rule_type": get_rule_type(rule), + "text": text.replace("{{", "").replace("}}", ""), + "tokens": tokens, + "bioes_labels": bioes_labels, + } + + +def split_records(records, common_expression_threshold=50): + """ + Return train, validation, and test records using a hybrid split. + + Keep rare license expressions in one split. Distribute records from common + expressions by identifier so each split represents their varied rule text. + """ + expression_counts = Counter(record["license_expression"] for record in records) + common_expressions = { + expression + for expression, count in expression_counts.items() + if count >= common_expression_threshold + } + + rare_expressions = sorted( + (expression for expression in expression_counts if expression not in common_expressions), + key=lambda expression: (-expression_counts[expression], expression), + ) + rare_record_count = sum(expression_counts[expression] for expression in rare_expressions) + targets = { + "train": 0.8 * rare_record_count, + "val": 0.1 * rare_record_count, + "test": 0.1 * rare_record_count, + } + assigned_counts = {name: 0 for name in targets} + rare_assignments = {} + + for expression in rare_expressions: + split = min( + targets, + key=lambda name: assigned_counts[name] / targets[name], + ) + rare_assignments[expression] = split + assigned_counts[split] += expression_counts[expression] + + splits = {name: [] for name in targets} + for record in records: + expression = record["license_expression"] + if expression in common_expressions: + identifier = record["identifier"].encode("utf-8") + bucket = int(hashlib.md5(identifier).hexdigest(), 16) % 100 + if bucket < 80: + split = "train" + elif bucket < 90: + split = "val" + else: + split = "test" + else: + split = rare_assignments[expression] + + splits[split].append(record) + + return splits + + +@click.command() +@click.option( + "--rules-dir", + type=click.Path(exists=True, file_okay=False), + default=None, + help="Path to rules directory (defaults to the ScanCode rules directory).", +) +@click.option( + "--output-dir", + type=click.Path(file_okay=False), + default="dataset-output", + help="Output directory for train, validation, and test JSONL files.", +) +def main(rules_dir, output_dir): + """Extract marked required phrases into a BIOES training dataset.""" + rules_path = Path(rules_dir or default_rules_data_dir) + rule_files = sorted(rules_path.glob("*.RULE")) + records = [] + + click.echo(f"scanning rules from: {rules_path}") + for rule in load_rules(rules_data_dir=str(rules_path)): + record = build_record(rule) + if record: + records.append(record) + + splits = split_records(records) + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + for split_name, records_in_split in splits.items(): + split_file = output_path / f"{split_name}.jsonl" + with split_file.open("w", encoding="utf-8") as output: + for record in records_in_split: + output.write(json.dumps(record, ensure_ascii=False) + "\n") + + click.echo("\ndone") + click.echo(f" rules scanned: {len(rule_files)}") + click.echo(f" annotated: {len(records)}") + click.echo( + f" train: {len(splits['train'])} val: {len(splits['val'])} test: {len(splits['test'])}" + ) + click.echo(f" output: {output_path}") + + +if __name__ == "__main__": + main() diff --git a/tests/README.rst b/tests/README.rst deleted file mode 100644 index d94783e..0000000 --- a/tests/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python test modules in this directory. - diff --git a/tests/test_composite_rules.py b/tests/test_composite_rules.py new file mode 100644 index 0000000..f2ab4a2 --- /dev/null +++ b/tests/test_composite_rules.py @@ -0,0 +1,525 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +from click.testing import CliRunner +import pytest + +from licensedcode.models import Rule +from licensedcode.required_phrases import IsRequiredPhrase + +from scancode_required_phrases import composite_rules +from scancode_required_phrases.composite_rules import add_composite_required_phrases +from scancode_required_phrases.composite_rules import add_required_phrases_to_composite_rules +from scancode_required_phrases.composite_rules import get_required_phrases_by_key +from scancode_required_phrases.composite_rules import update_composite_rules_using_required_phrases + + +def make_required_phrase_rule(expression, text, identifier): + return SimpleNamespace( + license_expression=expression, + text=text, + identifier=identifier, + is_required_phrase=True, + ) + + +def make_license(is_generic=False): + return SimpleNamespace(is_generic=is_generic) + + +def make_candidate(expression, text, identifier): + rule = make_required_phrase_rule(expression, text, identifier) + return IsRequiredPhrase(rule=rule, required_phrase_text=text) + + +@pytest.fixture +def required_phrases_by_key(): + return { + "mit": [make_candidate("mit", "MIT License", "mit_1.RULE")], + "apache-2.0": [ + make_candidate("apache-2.0", "Apache License", "apache-2.0_1.RULE"), + ], + "bsd-new": [make_candidate("bsd-new", "BSD License", "bsd-new_1.RULE")], + } + + +def test_get_required_phrases_by_key_collects_single_key_phrases_longest_first(): + rules_by_expression = { + "mit": [ + make_required_phrase_rule("mit", "MIT", "mit_1.RULE"), + make_required_phrase_rule("mit", "MIT License", "mit_2.RULE"), + SimpleNamespace(is_required_phrase=False), + ], + } + + required_phrases = get_required_phrases_by_key( + rules_by_expression=rules_by_expression, + licenses_by_key={"mit": make_license()}, + ) + + assert [phrase.required_phrase_text for phrase in required_phrases["mit"]] == [ + "MIT License", + "MIT", + ] + + +def test_get_required_phrases_by_key_skips_composite_expressions(): + rules_by_expression = { + "mit AND apache-2.0": [ + make_required_phrase_rule( + "mit AND apache-2.0", + "MIT and Apache", + "mit_and_apache_1.RULE", + ), + ], + } + licenses_by_key = { + "mit": make_license(), + "apache-2.0": make_license(), + } + + required_phrases = get_required_phrases_by_key( + rules_by_expression=rules_by_expression, + licenses_by_key=licenses_by_key, + ) + + assert required_phrases == {} + + +def test_get_required_phrases_by_key_skips_generic_licenses(): + rules_by_expression = { + "unknown": [ + make_required_phrase_rule("unknown", "Unknown License", "unknown_1.RULE"), + ], + } + + required_phrases = get_required_phrases_by_key( + rules_by_expression=rules_by_expression, + licenses_by_key={"unknown": make_license(is_generic=True)}, + ) + + assert required_phrases == {} + + +def test_add_required_phrases_marks_every_key(required_phrases_by_key): + rule = Rule( + license_expression="mit AND apache-2.0", + identifier="mit_and_apache_test.RULE", + text="Licensed under the MIT License and the Apache License.", + is_license_notice=True, + ) + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert "{{MIT License}}" in rule.text + assert "{{Apache License}}" in rule.text + + +def test_add_required_phrases_requires_every_key(required_phrases_by_key): + text = "Licensed under the MIT License." + rule = Rule( + license_expression="mit AND apache-2.0", + identifier="missing_apache_test.RULE", + text=text, + is_license_notice=True, + ) + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert rule.text == text + assert rule.source is None + + +def test_add_required_phrases_marks_three_keys(required_phrases_by_key): + rule = Rule( + license_expression="mit AND apache-2.0 AND bsd-new", + identifier="three_key_test.RULE", + text="MIT License, Apache License, and BSD License apply.", + is_license_notice=True, + ) + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0", "bsd-new"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert "{{MIT License}}" in rule.text + assert "{{Apache License}}" in rule.text + assert "{{BSD License}}" in rule.text + + +def test_add_required_phrases_keeps_existing_markers(required_phrases_by_key): + rule = Rule( + license_expression="mit AND apache-2.0", + identifier="existing_marker_test.RULE", + text="Licensed under the {{MIT License}} and the Apache License.", + is_license_notice=True, + ) + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert rule.text.count("{{MIT License}}") == 1 + assert "{{Apache License}}" in rule.text + + +def test_add_required_phrases_prefers_an_existing_marker(required_phrases_by_key): + text = "Licensed under {{Apache License}} {{or the MIT License}} (LICENSE.mit)." + rule = Rule( + license_expression="mit OR apache-2.0", + identifier="existing_markers_test.RULE", + text=text, + is_license_notice=True, + ) + required_phrases_by_key["mit"] = [ + make_candidate("mit", "License: MIT", "mit_2.RULE"), + required_phrases_by_key["mit"][0], + ] + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert rule.text == text + + +def test_add_required_phrases_writes_once(required_phrases_by_key, tmp_path, monkeypatch): + rule = Rule( + license_expression="mit AND apache-2.0", + identifier="write_once_test.RULE", + text="Licensed under the MIT License and the Apache License.", + is_license_notice=True, + ) + original_dump = Rule.dump + dump_calls = [] + + def dump(rule, rules_data_dir): + dump_calls.append(rule.identifier) + original_dump(rule, rules_data_dir) + + monkeypatch.setattr(Rule, "dump", dump) + monkeypatch.setattr(composite_rules, "rules_data_dir", str(tmp_path)) + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0"], + required_phrases_by_key=required_phrases_by_key, + write_phrase_source=True, + ) + + saved_rule = Rule.from_file(str(tmp_path / rule.identifier)) + assert dump_calls == [rule.identifier] + assert "{{MIT License}}" in saved_rule.text + assert "{{Apache License}}" in saved_rule.text + assert saved_rule.source == "mit_1.RULE apache-2.0_1.RULE" + + +def test_add_required_phrases_dry_run_does_not_write( + required_phrases_by_key, + monkeypatch, +): + rule = Rule( + license_expression="mit AND apache-2.0", + identifier="dry_run_test.RULE", + text="Licensed under the MIT License and the Apache License.", + is_license_notice=True, + ) + + def dump(*args, **kwargs): + pytest.fail("Rule.dump() called during a dry run") + + monkeypatch.setattr(Rule, "dump", dump) + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert "{{MIT License}}" in rule.text + assert "{{Apache License}}" in rule.text + + +def test_add_required_phrases_rolls_back_a_failed_update( + required_phrases_by_key, + monkeypatch, +): + text = "Licensed under the MIT License and the Apache License." + source = "existing.RULE" + rule = Rule( + license_expression="mit AND apache-2.0", + identifier="rollback_test.RULE", + text=text, + source=source, + is_license_notice=True, + ) + original_add = composite_rules.add_required_phrase_to_rule + calls = [] + + def add_required_phrase(*args, **kwargs): + calls.append(kwargs["required_phrase"]) + if len(calls) == 2: + return False + return original_add(*args, **kwargs) + + monkeypatch.setattr(composite_rules, "add_required_phrase_to_rule", add_required_phrase) + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["mit", "apache-2.0"], + required_phrases_by_key=required_phrases_by_key, + write_phrase_source=True, + dry_run=True, + ) + + assert calls == ["MIT License", "Apache License"] + assert rule.text == text + assert rule.source == source + + +def test_add_required_phrases_uses_a_non_overlapping_candidate(): + rule = Rule( + license_expression="gpl-2.0 AND gpl-2.0-plus", + identifier="overlapping_candidate_test.RULE", + text="GNU General Public License version 2, or any later version.", + is_license_notice=True, + ) + required_phrases_by_key = { + "gpl-2.0": [ + make_candidate( + "gpl-2.0", + "GNU General Public License version 2", + "gpl-2.0_1.RULE", + ), + ], + "gpl-2.0-plus": [ + make_candidate( + "gpl-2.0-plus", + "General Public License version 2", + "gpl-2.0-plus_1.RULE", + ), + make_candidate( + "gpl-2.0-plus", + "any later version", + "gpl-2.0-plus_2.RULE", + ), + ], + } + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["gpl-2.0", "gpl-2.0-plus"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert "{{GNU General Public License version 2}}" in rule.text + assert "{{any later version}}" in rule.text + + +def test_add_required_phrases_backtracks_to_an_earlier_key_candidate(): + rule = Rule( + license_expression="license-a AND license-b", + identifier="candidate_backtracking_test.RULE", + text="Alpha Long License and Backup Terms.", + is_license_notice=True, + ) + required_phrases_by_key = { + "license-a": [ + make_candidate("license-a", "Alpha Long License", "license-a_1.RULE"), + make_candidate("license-a", "Backup Terms", "license-a_2.RULE"), + ], + "license-b": [ + make_candidate("license-b", "Long License", "license-b_1.RULE"), + ], + } + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["license-a", "license-b"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert "{{Long License}}" in rule.text + assert "{{Backup Terms}}" in rule.text + assert "{{Alpha Long License}}" not in rule.text + + +def test_add_required_phrases_does_not_partially_mark_repeated_overlaps(): + text = "Alpha License terms and Alpha License." + rule = Rule( + license_expression="license-a AND license-b", + identifier="repeated_overlap_test.RULE", + text=text, + is_license_notice=True, + ) + required_phrases_by_key = { + "license-a": [ + make_candidate("license-a", "Alpha License", "license-a_1.RULE"), + ], + "license-b": [ + make_candidate("license-b", "License terms", "license-b_1.RULE"), + ], + } + + add_required_phrases_to_composite_rules( + rules=[rule], + license_keys=["license-a", "license-b"], + required_phrases_by_key=required_phrases_by_key, + dry_run=True, + ) + + assert rule.text == text + assert rule.source is None + + +def test_update_composite_rules_uses_single_key_required_phrases(monkeypatch): + required_rules = { + "mit": [make_required_phrase_rule("mit", "MIT License", "mit_1.RULE")], + "apache-2.0": [ + make_required_phrase_rule( + "apache-2.0", + "Apache License", + "apache-2.0_1.RULE", + ), + ], + } + target = Rule( + license_expression="mit AND apache-2.0", + identifier="mit_and_apache_test.RULE", + text="Licensed under the MIT License and the Apache License.", + is_license_notice=True, + ) + + licenses_by_key = { + "mit": make_license(), + "apache-2.0": make_license(), + } + + monkeypatch.setattr(composite_rules, "get_licenses_db", lambda: licenses_by_key) + monkeypatch.setattr( + composite_rules, + "get_base_rules_by_expression", + lambda license_expression=None: required_rules, + ) + monkeypatch.setattr( + composite_rules, + "get_updatable_rules_by_expression", + lambda license_expression=None, simple_expression=True: { + "mit AND apache-2.0": [target], + }, + ) + + update_composite_rules_using_required_phrases(dry_run=True) + + assert "{{MIT License}}" in target.text + assert "{{Apache License}}" in target.text + + +def test_update_composite_rules_skips_generic_keys(monkeypatch): + required_rules = { + "mit": [make_required_phrase_rule("mit", "MIT License", "mit_1.RULE")], + "unknown": [ + make_required_phrase_rule("unknown", "Unknown License", "unknown_1.RULE"), + ], + } + target = Rule( + license_expression="mit AND unknown", + identifier="mit_and_unknown_test.RULE", + text="Licensed under the MIT License and an Unknown License.", + is_license_notice=True, + ) + licenses_by_key = { + "mit": make_license(), + "unknown": make_license(is_generic=True), + } + + monkeypatch.setattr(composite_rules, "get_licenses_db", lambda: licenses_by_key) + monkeypatch.setattr( + composite_rules, + "get_base_rules_by_expression", + lambda license_expression=None: required_rules, + ) + monkeypatch.setattr( + composite_rules, + "get_updatable_rules_by_expression", + lambda license_expression=None, simple_expression=True: { + "mit AND unknown": [target], + }, + ) + + update_composite_rules_using_required_phrases(dry_run=True) + + assert "{{MIT License}}" in target.text + assert "{{Unknown License}}" not in target.text + + +def test_composite_command_calls_update_and_validation(monkeypatch): + update_calls = [] + validation_calls = [] + + monkeypatch.setattr( + composite_rules, + "update_composite_rules_using_required_phrases", + lambda **kwargs: update_calls.append(kwargs), + ) + monkeypatch.setattr( + composite_rules, + "validate_and_reindex", + lambda **kwargs: validation_calls.append(kwargs), + ) + + result = CliRunner().invoke( + add_composite_required_phrases, + [ + "--license-expression", + "mit AND apache-2.0", + "--write-phrase-source", + "--validate", + "--reindex", + "--dry-run", + "--verbose", + ], + ) + + assert result.exit_code == 0, result.output + assert update_calls == [ + { + "license_expression": "mit AND apache-2.0", + "write_phrase_source": True, + "dry_run": True, + "verbose": True, + }, + ] + assert validation_calls == [ + { + "validate": True, + "reindex": True, + "verbose": True, + }, + ] diff --git a/tests/test_dataset.py b/tests/test_dataset.py new file mode 100644 index 0000000..8e4ac0c --- /dev/null +++ b/tests/test_dataset.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +from click.testing import CliRunner +import pytest + +from licensedcode.models import InvalidRule +from licensedcode.models import Rule +from licensedcode.tokenize import InvalidRuleRequiredPhrase + +from scancode_required_phrases.dataset import build_record +from scancode_required_phrases.dataset import main +from scancode_required_phrases.dataset import split_records +from scancode_required_phrases.dataset import tag_tokens + + +def make_rule( + identifier="mit_test.RULE", + license_expression="mit", + text="Licensed under the {{MIT License}}.", + **kwargs, +): + is_license_notice = kwargs.pop("is_license_notice", True) + return Rule( + identifier=identifier, + license_expression=license_expression, + text=text, + is_license_notice=is_license_notice, + **kwargs, + ) + + +def test_tag_tokens_assigns_bioes_labels(): + tokens, labels = tag_tokens("Use {{MIT}} or the {{Apache License Version}} terms.") + + assert tokens == ["Use", "MIT", "or", "the", "Apache", "License", "Version", "terms"] + assert labels == ["O", "S-REQ", "O", "O", "B-REQ", "I-REQ", "E-REQ", "O"] + + +def test_build_record_returns_normalized_rule_data(): + rule = make_rule(text="Licensed under the {{ＭＩＴ License}}.\rTerms") + + record = build_record(rule) + + assert record == { + "identifier": "mit_test.RULE", + "license_expression": "mit", + "rule_type": "is_license_notice", + "text": "Licensed under the MIT License.\nTerms", + "tokens": ["Licensed", "under", "the", "MIT", "License", "Terms"], + "bioes_labels": ["O", "O", "O", "B-REQ", "E-REQ", "O"], + } + + +def test_build_record_skips_unannotated_rules_and_rejects_invalid_markers(): + assert build_record(make_rule(text="Licensed under the MIT License.")) is None + assert build_record(make_rule(text="")) is None + assert build_record(make_rule(is_required_phrase=True)) is None + + invalid_texts = ( + "Empty {{}} marker", + "Opening {{dangling marker", + "Closing dangling}} marker", + "Valid {{MIT}} and {{dangling", + ) + for text in invalid_texts: + with pytest.raises(InvalidRuleRequiredPhrase): + build_record(make_rule(text=text)) + + +@pytest.mark.parametrize( + "rule", + [ + make_rule( + license_expression=None, + is_license_notice=False, + is_false_positive=True, + ), + make_rule(is_license_notice=False, is_license_intro=True), + make_rule(is_license_notice=False, is_license_clue=True), + make_rule(is_deprecated=True), + ], +) +def test_build_record_skips_rules_that_are_not_training_targets(rule): + assert build_record(rule) is None + + +def test_split_records_uses_the_hybrid_split_deterministically(): + records = [ + { + "identifier": f"common_{index}.RULE", + "license_expression": "common", + } + for index in range(50) + ] + for expression in ("rare-a", "rare-b", "rare-c"): + records.extend( + { + "identifier": f"{expression}_{index}.RULE", + "license_expression": expression, + } + for index in range(5) + ) + + splits = split_records(records) + + assert splits == split_records(records) + assert sum(len(split) for split in splits.values()) == len(records) + split_by_identifier = { + record["identifier"]: name for name, split in splits.items() for record in split + } + assert split_by_identifier["common_20.RULE"] == "train" + assert split_by_identifier["common_43.RULE"] == "val" + assert split_by_identifier["common_3.RULE"] == "test" + assert split_by_identifier["rare-a_0.RULE"] == "train" + assert split_by_identifier["rare-b_0.RULE"] == "val" + assert split_by_identifier["rare-c_0.RULE"] == "test" + + for expression in ("rare-a", "rare-b", "rare-c"): + containing_splits = [ + name + for name, split in splits.items() + if any(record["license_expression"] == expression for record in split) + ] + assert len(containing_splits) == 1 + + +def test_main_writes_the_complete_dataset(tmp_path): + rules_dir = tmp_path / "rules" + output_dir = tmp_path / "dataset" + rules_dir.mkdir() + + rules = [ + make_rule( + identifier="mit_test.RULE", + license_expression="mit", + text="Licensed under the {{MIT License}}.", + ), + make_rule( + identifier="apache_test.RULE", + license_expression="apache-2.0", + text="Licensed under the {{Apache License}}.", + ), + make_rule( + identifier="bsd_test.RULE", + license_expression="bsd-new", + text="Licensed under the {{BSD License}}.", + ), + make_rule( + identifier="unmarked_test.RULE", + text="Licensed under the MIT License.", + ), + make_rule( + identifier="required_phrase_test.RULE", + text="MIT License", + is_required_phrase=True, + ), + ] + for rule in rules: + rule.dump(str(rules_dir)) + + result = CliRunner().invoke( + main, + ["--rules-dir", str(rules_dir), "--output-dir", str(output_dir)], + ) + + assert result.exit_code == 0, result.output + split_files = sorted(path.name for path in output_dir.glob("*.jsonl")) + assert split_files == ["test.jsonl", "train.jsonl", "val.jsonl"] + + records = [] + for split_file in output_dir.glob("*.jsonl"): + records.extend( + json.loads(line) for line in split_file.read_text(encoding="utf-8").splitlines() + ) + + assert {record["identifier"] for record in records} == { + "apache_test.RULE", + "bsd_test.RULE", + "mit_test.RULE", + } + assert all(len(record["tokens"]) == len(record["bioes_labels"]) for record in records) + assert all("{{" not in record["text"] and "}}" not in record["text"] for record in records) + + +def test_main_fails_when_a_rule_cannot_be_loaded(tmp_path): + rules_dir = tmp_path / "rules" + output_dir = tmp_path / "dataset" + rules_dir.mkdir() + (rules_dir / "broken.RULE").write_text("", encoding="utf-8") + + result = CliRunner().invoke( + main, + ["--rules-dir", str(rules_dir), "--output-dir", str(output_dir)], + ) + + assert isinstance(result.exception, InvalidRule) + assert "broken.RULE" in str(result.exception) + assert not output_dir.exists()