From 9dc45bf6b16207616f6a224e94465ed67b09f566 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 11:31:01 -0400 Subject: [PATCH 01/10] adding pydantic generation and jsonld generation to release --- .github/workflows/validate_and_release.yml | 26 +++++- scripts/fix_pydantic.py | 98 ++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 scripts/fix_pydantic.py diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index 30657fb88..b2180e338 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -6,6 +6,7 @@ on: push: branches: - main + - ga_rel_generate pull_request: branches: ['*'] # Allow to trigger the generation of release files automatically @@ -36,6 +37,28 @@ jobs: python scripts/jsonParser.py reproschema validate examples + generate other formats: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + python install linkml astor pre-commit + - name: generate pydantic and fixing it + run: | + gen-pydantic --pydantic-version 2 linkml-schema/reproschema.yaml > reproschema_model_autogen.py + python scripts/fix_pydantic.py reproschema_model_autogen.py reproschema_model.py + pre-commit run --files reproschema_model.py + - name: generate jsonld + run: | + gen-jsonld --context contexts/reproschema linkml-schema/reproschema.yaml > reproschema.jsonld + + release: needs: [validate] if: github.event_name == 'workflow_dispatch' @@ -51,7 +74,8 @@ jobs: run: | echo "Making a release ${{ inputs.version }}" mkdir releases/${{ inputs.version }} - cp contexts/reproschema releases/${{ inputs.version }}/base + cp contexts/reproschema releases/${{ inputs.version }}/reproschema + cp reproschema_model.py releases/${{ inputs.version }}/reproschema_model.py # python scripts/makeRelease.py ${{ inputs.version }} - name: Open pull requests to add files diff --git a/scripts/fix_pydantic.py b/scripts/fix_pydantic.py new file mode 100644 index 000000000..8ff226967 --- /dev/null +++ b/scripts/fix_pydantic.py @@ -0,0 +1,98 @@ +""" Using ast transformer to fix some issues with automatic pydantic generation""" + +import sys +import ast +import astor + + +class ClassRemover(ast.NodeTransformer): + def __init__(self, class_name): + self.class_name = class_name + + def visit_ClassDef(self, node): + # Remove the class if its name matches the class_to_remove + if node.name == self.class_name: + return None + return node + + def visit_Expr(self, node): + # Check if the node is a call expression + if isinstance(node.value, ast.Call): + # Check if the call expression is an attribute (method call) + if isinstance(node.value.func, ast.Attribute): + # Check if the method call matches the specified class and method name + if ( + isinstance(node.value.func.value, ast.Name) + and node.value.func.value.id == self.class_name + ): + return None # Remove this node + return self.generic_visit(node) + + +class TypeReplacer(ast.NodeTransformer): + def __init__(self, old_type, new_type): + self.old_type = old_type + self.new_type = new_type + + def visit_FunctionDef(self, node): + # Check all arguments in the function definition + for arg in node.args.args: + if arg.annotation: + arg.annotation = self.visit(arg.annotation) + return self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node): + # Handle async function definitions similarly + for arg in node.args.args: + if arg.annotation: + arg.annotation = self.visit(arg.annotation) + return self.generic_visit(node) + + def visit_Name(self, node): + # Replace the old type with the new type + if node.id == self.old_type: + node.id = self.new_type + return node + + def visit_Subscript(self, node): + # Handle Union, Optional, and other subscripted types + node.value = self.visit(node.value) + node.slice = self.visit(node.slice) + return node + + def visit_Index(self, node): + # Handle the index part of subscripted types + node.value = self.visit(node.value) + return node + + def visit_Tuple(self, node): + # Handle tuples in type annotations + node.elts = [self.visit(elt) for elt in node.elts] + return node + + +def edit_pydantic(input_file, output_file): + + with open(input_file, "r") as file: + tree = ast.parse(file.read()) + + transformer_class = ClassRemover(class_name="LangString") + tree_modclass = transformer_class.visit(tree) + + transformer_tp = TypeReplacer(old_type="LangString", new_type="Dict[str, str]") + tree_modclass_modtype = transformer_tp.visit(tree_modclass) + + with open(output_file, "w") as file: + file.write(astor.to_source(tree_modclass_modtype)) + + +if __name__ == "__main__": + input_file = sys.argv[1] + if len(sys.argv) < 3: + output_file = input_file + else: + output_file = sys.argv[2] + print( + f"Fixing automatically generated pydantic file {input_file} and saving to {output_file}" + ) + edit_pydantic(input_file, output_file) From 5fb26fa44b0f9c72bbcda159c87317799394a1cb Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Mon, 17 Jun 2024 10:00:18 -0400 Subject: [PATCH 02/10] removing creator slot that is not used --- linkml-schema/reproschema.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/linkml-schema/reproschema.yaml b/linkml-schema/reproschema.yaml index 8afcaf3fd..0eedfc152 100644 --- a/linkml-schema/reproschema.yaml +++ b/linkml-schema/reproschema.yaml @@ -101,9 +101,6 @@ slots: contentUrl: slot_uri: schema:contentUrl range: uriorcurie - creator: - slot_uri: schema:creator - range: Person cronTable: title: cronTable description: TODO not described in reproschema From d604256c045528088f02339dc8a0c8409160c63d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 08:16:45 -0400 Subject: [PATCH 03/10] adding preamble to Protocol, removing toods --- linkml-schema/reproschema.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/linkml-schema/reproschema.yaml b/linkml-schema/reproschema.yaml index 0eedfc152..912d981c8 100644 --- a/linkml-schema/reproschema.yaml +++ b/linkml-schema/reproschema.yaml @@ -55,7 +55,7 @@ slots: slot_uri: schema:associatedMedia audio: title: audio - description: TODO + description: An audio object. slot_uri: schema:audio any_of: - range: uri @@ -103,7 +103,6 @@ slots: range: uriorcurie cronTable: title: cronTable - description: TODO not described in reproschema slot_uri: reproschema:cronTable datumType: title: datumType @@ -261,7 +260,7 @@ slots: range: OverrideProperty preamble: title: Preamble - description: The preamble for an assessment + description: The preamble for an assessment. slot_uri: reproschema:preamble multivalued: true range: langString @@ -323,7 +322,7 @@ slots: range: datetime slot_uri: prov:startedAtTime subject_id: - slot_uri: nidm:subject_id #TODO check this @type:rdf:Property + slot_uri: nidm:subject_id range: string ui: title: UI @@ -499,7 +498,7 @@ classes: class_uri: rdf:langString MediaObject: title: Media Object - description: Add description #TODO + description: A media object, such as an image, video, audio, or text object embedded in a web page or a downloadable dataset. is_a: Thing class_uri: schema:MediaObject slots: @@ -539,7 +538,7 @@ classes: - id - subject_id class_uri: reproschema:Participant - Protocol: # TODO multiple types + Protocol: title: Protocol description: A representation of a study which comprises one or more assessments. is_a: Thing @@ -552,6 +551,7 @@ classes: - description - landingPage - messages + - preamble - prefLabel - schemaVersion - ui @@ -593,7 +593,7 @@ classes: - unitOptions - valueType class_uri: reproschema:ResponseOption - SoftwareAgent: # TODO multiple types + SoftwareAgent: title: Software Agent description: Captures information about some action that took place. It also links to information @@ -613,9 +613,8 @@ classes: - category class_uri: schema:Thing UI: - title: todo - description: - - todo + title: UI properties + description: A group of properties related to UI. slots: - order - addProperties From f88980f0e97c68f2daca87d21aa97603760e1a5c Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 11:39:05 -0400 Subject: [PATCH 04/10] adding nt and ttl to release --- .github/workflows/validate_and_release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index b2180e338..4ad23da9a 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -49,6 +49,7 @@ jobs: run: | python -m pip install --upgrade pip setuptools python install linkml astor pre-commit + pip install git+https://github.com/ReproNim/reproschema-py.git - name: generate pydantic and fixing it run: | gen-pydantic --pydantic-version 2 linkml-schema/reproschema.yaml > reproschema_model_autogen.py @@ -57,6 +58,10 @@ jobs: - name: generate jsonld run: | gen-jsonld --context contexts/reproschema linkml-schema/reproschema.yaml > reproschema.jsonld + - name: generate n-triples and turtle formats using reproschema + run: | + reproschema convert --format n-triples reproschema.jsonld > reproschema.nt + reproschema convert --format turtle reproschema.jsonld > reproschema.ttl release: @@ -76,6 +81,9 @@ jobs: mkdir releases/${{ inputs.version }} cp contexts/reproschema releases/${{ inputs.version }}/reproschema cp reproschema_model.py releases/${{ inputs.version }}/reproschema_model.py + cp reproschema.jsonld releases/${{ inputs.version }}/reproschema.jsonld + cp reproschema.nt releases/${{ inputs.version }}/reproschema.nt + cp reproschema.ttl releases/${{ inputs.version }}/reproschema.ttl # python scripts/makeRelease.py ${{ inputs.version }} - name: Open pull requests to add files From fddd258626338c3a780088c9118a82436a3d2db6 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 11:40:26 -0400 Subject: [PATCH 05/10] fixing job name --- .github/workflows/validate_and_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index 4ad23da9a..80fbcac1c 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -37,7 +37,7 @@ jobs: python scripts/jsonParser.py reproschema validate examples - generate other formats: + generate_other_formats: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From d6816d691a827f2dca62b69cc9f723e7802e0f94 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 11:43:12 -0400 Subject: [PATCH 06/10] fixing ga --- .github/workflows/validate_and_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index 80fbcac1c..e26b9e5b9 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -48,7 +48,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - python install linkml astor pre-commit + pip install linkml astor pre-commit pip install git+https://github.com/ReproNim/reproschema-py.git - name: generate pydantic and fixing it run: | From e1ac0d11b566cffce5b598993d49b9bc6593e7db Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 11:45:32 -0400 Subject: [PATCH 07/10] removing testing branch from ga --- .github/workflows/validate_and_release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index e26b9e5b9..37a01b59c 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -6,7 +6,6 @@ on: push: branches: - main - - ga_rel_generate pull_request: branches: ['*'] # Allow to trigger the generation of release files automatically From 2bf12d6cb03ec00b52584edfb88567d9e640bdcf Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 12:22:03 -0400 Subject: [PATCH 08/10] modify the pre-commit in GA so it doesnt stop when issue found --- .github/workflows/validate_and_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index 37a01b59c..2ef35ac8d 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -53,7 +53,7 @@ jobs: run: | gen-pydantic --pydantic-version 2 linkml-schema/reproschema.yaml > reproschema_model_autogen.py python scripts/fix_pydantic.py reproschema_model_autogen.py reproschema_model.py - pre-commit run --files reproschema_model.py + pre-commit run --files reproschema_model.py || true - name: generate jsonld run: | gen-jsonld --context contexts/reproschema linkml-schema/reproschema.yaml > reproschema.jsonld From 59702412a0f125fcf38a0c0a82f88dc305443106 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 12:25:31 -0400 Subject: [PATCH 09/10] moving format generations to release --- .github/workflows/validate_and_release.yml | 45 ++++++++-------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index 2ef35ac8d..c1411e6c7 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -36,33 +36,6 @@ jobs: python scripts/jsonParser.py reproschema validate examples - generate_other_formats: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install linkml astor pre-commit - pip install git+https://github.com/ReproNim/reproschema-py.git - - name: generate pydantic and fixing it - run: | - gen-pydantic --pydantic-version 2 linkml-schema/reproschema.yaml > reproschema_model_autogen.py - python scripts/fix_pydantic.py reproschema_model_autogen.py reproschema_model.py - pre-commit run --files reproschema_model.py || true - - name: generate jsonld - run: | - gen-jsonld --context contexts/reproschema linkml-schema/reproschema.yaml > reproschema.jsonld - - name: generate n-triples and turtle formats using reproschema - run: | - reproschema convert --format n-triples reproschema.jsonld > reproschema.nt - reproschema convert --format turtle reproschema.jsonld > reproschema.ttl - - release: needs: [validate] if: github.event_name == 'workflow_dispatch' @@ -73,7 +46,23 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.12 - + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install linkml astor pre-commit + pip install git+https://github.com/ReproNim/reproschema-py.git + - name: Generate pydantic using linml and fixing it with reproschema specific script + run: | + gen-pydantic --pydantic-version 2 linkml-schema/reproschema.yaml > reproschema_model_autogen.py + python scripts/fix_pydantic.py reproschema_model_autogen.py reproschema_model.py + pre-commit run --files reproschema_model.py || true + - name: Generate jsonld format using linkml + run: | + gen-jsonld --context contexts/reproschema linkml-schema/reproschema.yaml > reproschema.jsonld + - name: Generate n-triples and turtle formats using reproschema + run: | + reproschema convert --format n-triples reproschema.jsonld > reproschema.nt + reproschema convert --format turtle reproschema.jsonld > reproschema.ttl - name: Make a release run: | echo "Making a release ${{ inputs.version }}" From ccba7f9dea8e27297af379eed6aa84154482c75b Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Tue, 18 Jun 2024 12:29:25 -0400 Subject: [PATCH 10/10] running new pre-commit --- scripts/fix_pydantic.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/fix_pydantic.py b/scripts/fix_pydantic.py index 8ff226967..b82f5d295 100644 --- a/scripts/fix_pydantic.py +++ b/scripts/fix_pydantic.py @@ -1,7 +1,8 @@ -""" Using ast transformer to fix some issues with automatic pydantic generation""" +""" Using ast transformer to fix issues with automatic pydantic generation""" -import sys import ast +import sys + import astor @@ -20,7 +21,7 @@ def visit_Expr(self, node): if isinstance(node.value, ast.Call): # Check if the call expression is an attribute (method call) if isinstance(node.value.func, ast.Attribute): - # Check if the method call matches the specified class and method name + # Check if the method call matches the specified class if ( isinstance(node.value.func.value, ast.Name) and node.value.func.value.id == self.class_name @@ -79,7 +80,9 @@ def edit_pydantic(input_file, output_file): transformer_class = ClassRemover(class_name="LangString") tree_modclass = transformer_class.visit(tree) - transformer_tp = TypeReplacer(old_type="LangString", new_type="Dict[str, str]") + transformer_tp = TypeReplacer( + old_type="LangString", new_type="Dict[str, str]" + ) tree_modclass_modtype = transformer_tp.visit(tree_modclass) with open(output_file, "w") as file: @@ -93,6 +96,7 @@ def edit_pydantic(input_file, output_file): else: output_file = sys.argv[2] print( - f"Fixing automatically generated pydantic file {input_file} and saving to {output_file}" + f"Fixing automatically generated pydantic file {input_file} " + f"and saving to {output_file}" ) edit_pydantic(input_file, output_file)