diff --git a/.flake8 b/.flake8 index bb93c8a93a..b7de0e021e 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,2 @@ [flake8] max-line-length=79 -exclude = - scripts/editProperties.py diff --git a/.github/workflows/publishdocs.yaml b/.github/workflows/publishdocs.yaml index 45459add27..e621765234 100644 --- a/.github/workflows/publishdocs.yaml +++ b/.github/workflows/publishdocs.yaml @@ -12,6 +12,19 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install the required python packages + run: python -m pip install linkml + + - name: Generating docs + run: | + gen-doc -d docs/schema/doc-linkml-autogen linkml-schema/reproschema.yaml + gen-erdiagram linkml-schema/reproschema.yaml > docs/schema/erdiagram-autogen.md + - name: Deploy docs uses: mhausenblas/mkdocs-deploy-gh-pages@master env: diff --git a/.github/workflows/validate_and_release.yml b/.github/workflows/validate_and_release.yml index 8d7632e3df..f3b713eaa3 100644 --- a/.github/workflows/validate_and_release.yml +++ b/.github/workflows/validate_and_release.yml @@ -72,7 +72,6 @@ jobs: mv reproschema.jsonld releases/${{ inputs.version }}/reproschema.jsonld mv reproschema.nt releases/${{ inputs.version }}/reproschema.nt mv reproschema.ttl releases/${{ inputs.version }}/reproschema.ttl - # python scripts/makeRelease.py ${{ inputs.version }} - name: Open pull requests to add files uses: peter-evans/create-pull-request@v6 diff --git a/.gitignore b/.gitignore index d4c3c92cf9..ddbe44f355 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,12 @@ __pycache__ .idea/ -node_modules local_data +# ignore linkml auto generated doc +docs/schema/doc-linkml-autogen +docs/schema/erdiagram-autogen.md + +node_modules package-lock.json package.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8cf1e3ea20..f8d06d530a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,11 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + exclude: | + (?x)^( + docs/specification/.* + | mkdocs.yml + )$ - id: check-json - id: check-ast - id: check-added-large-files diff --git a/README.md b/README.md index 50b5195ad9..d4ee7ccbc9 100644 --- a/README.md +++ b/README.md @@ -19,37 +19,49 @@ This repository contains: - a example of [a protocol based on the reproschema](./examples) - the [documentation](./docs) +## Developing ReproSchema -## Licenses +### Updating the schema -### Code +As of release 1.0.0, a linked data modeling language, [LinkML](https://linkml.io/linkml/), is used to create +a [YAML file](linkml-schema/reproschema.yaml) with the schema. -The content of this repository is distributed under the [Apache 2.0 license](./LICENSE). +The [context file](contexts/reproschema) was automatically generated using LinkML, +and then manually curated in order to support all the reproschema feature. -### Documentation +#### Style -Creative Commons License
The corresponding documentation is licensed under a Creative Commons Attribution 4.0 International License. +This repo uses pre-commit to check styling. +- Install pre-commit with pip: `pip install pre-commit` +- In order to use it with the repository, you have to run `run pre-commit install` in the root directory the first time you use it. -## Contributors -https://github.com/ReproNim/reproschema/graphs/contributors +### Release +Upon release, there are additional formats, `jsonsld`, `turtle`, `n-triples` +and `pydantic` that are created using `LinkML` tools, `reproschema-py`, +and [reproschema-specific script](./scripts/fix_pydantic.py) to "fix" the `pydantic` format. +The entire process is automated in the GitHub Action Workflow: +[Validate and Release](.github/workflows/validate_and_release.yml). +This workflow must be manually triggered by the core developers once a new release is ready. +All the releases can be found in [releases directory](./releases). + +### Updating model in reproschema-py +Another GitHub Action Workflow: [ Create Pull Request to reproschema-py](.github/workflows/push_reproschema_py.yml) +is responsible for creating pull request to the `reproschema-py` Python library with +the new version of pydantic model and context. +The workflow is currently also triggered manually by the core developers. -### Developer notes -To run the Python scripts in the scripts directory, you will need to install the -following libraries via pip -- reproschema (makeRelease.py) -- pytablewriter (editProperties.py) +## Licenses + +### Code -To make a new release: +The content of this repository is distributed under the [Apache 2.0 license](./LICENSE). -```bash -python scripts/makeRelease.py -python scripts/editProperties.py -``` +### Documentation -In addition, this repo uses pre-commit to check styling. +Creative Commons License
The corresponding documentation is licensed under a Creative Commons Attribution 4.0 International License. -Install: `pip install pre-commit` +## Contributors -Use: run `pre-commit install` in the root directory of the repo. +https://github.com/ReproNim/reproschema/graphs/contributors diff --git a/docs/index.md b/docs/index.md index f33964d600..464cfe3f25 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,7 @@ things are to use the ReproSchema but some "big picture" conceptual understandin could save you from a lot of confusion. 😉 - Not sure how the project is organized? Check out the [project structure](./project-structure.md) page. -- Want more details on how the `Reproschema` itself is structured: check out our [schema page](./schema.md) +- Want more details on how the `Reproschema` itself is structured: check out our [schema page](./schema/schema.md) - - -### Protocol -A representation of a study which comprises one or more assessments. - -**URI:** [http://schema.repronim.org/Protocol](http://schema.repronim.org/Protocol) - -| Name | Description | URI | -| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| addProperties | An array of objects to describe the various properties added to assessments and fields. | [http://schema.repronim.org/addProperties](http://schema.repronim.org/addProperties) | -| allow | An array of items indicating properties allowed on an activity or protocol | [http://schema.repronim.org/allow](http://schema.repronim.org/allow) | -| compute | An array of objects indicating computations in an activity or protocol and maps it to the corresponding Field item. scoring logic is a subset of all computations that could be performed and not all computations will be scoring. For example, one may want to do conversion from one unit to another. | [http://schema.repronim.org/compute](http://schema.repronim.org/compute) | -| landingPage | An element (by URL) to point to the protocol readme or landing page. | [http://schema.repronim.org/landingPage](http://schema.repronim.org/landingPage) | -| messages | An array of objects to define conditional messages in an activity or protocol. | [http://schema.repronim.org/messages](http://schema.repronim.org/messages) | -| order | An ordered list to describe the order in which the items of an assessment or protocol appear in the user interface. | [http://schema.repronim.org/order](http://schema.repronim.org/order) | -| overrideProperties | An array of objects to override the various properties added to assessments and fields. | [http://schema.repronim.org/overrideProperties](http://schema.repronim.org/overrideProperties) | -| preamble | The preamble for an assessment | [http://schema.repronim.org/preamble](http://schema.repronim.org/preamble) | -| shuffle | An element (bool) to determine if the list of items is shuffled or in order. | [http://schema.repronim.org/shuffle](http://schema.repronim.org/shuffle) | -| about | The subject matter of the Field. | [http://schema.org/about](http://schema.org/about) | -| audio | An embedded audio object. | [http://schema.org/audio](http://schema.org/audio) | -| description | A description of the item. | [http://schema.org/description](http://schema.org/description) | -| image | An image of the item. This can be a URL or a fully described ImageObject. | [http://schema.org/image](http://schema.org/image) | -| schemaVersion | Indicates (by URL or string) a particular version of a schema used in some CreativeWork. For example, a document could declare a schemaVersion using an URL such as http://schema.org/version/2.0/ if precise indication of schema version was required by some application. | [http://schema.org/schemaVersion](http://schema.org/schemaVersion) | -| version | The version of the CreativeWork embodied by a specified resource. | [http://schema.org/version](http://schema.org/version) | -| video | An embedded video object. | [http://schema.org/video](http://schema.org/video) | -| altLabel | The alternate label. | [http://www.w3.org/2004/02/skos/core#altLabel](http://www.w3.org/2004/02/skos/core#altLabel) | -| prefLabel | The preferred label. | [http://www.w3.org/2004/02/skos/core#prefLabel](http://www.w3.org/2004/02/skos/core#prefLabel) | - -### Activity -An assessment in a protocol. - -**URI:** [http://schema.repronim.org/Activity](http://schema.repronim.org/Activity) - -| Name | Description | URI | -| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| addProperties | An array of objects to describe the various properties added to assessments and fields. | [http://schema.repronim.org/addProperties](http://schema.repronim.org/addProperties) | -| allow | An array of items indicating properties allowed on an activity or protocol | [http://schema.repronim.org/allow](http://schema.repronim.org/allow) | -| compute | An array of objects indicating computations in an activity or protocol and maps it to the corresponding Field item. scoring logic is a subset of all computations that could be performed and not all computations will be scoring. For example, one may want to do conversion from one unit to another. | [http://schema.repronim.org/compute](http://schema.repronim.org/compute) | -| cronTable | None | [http://schema.repronim.org/cronTable](http://schema.repronim.org/cronTable) | -| messages | An array of objects to define conditional messages in an activity or protocol. | [http://schema.repronim.org/messages](http://schema.repronim.org/messages) | -| order | An ordered list to describe the order in which the items of an assessment or protocol appear in the user interface. | [http://schema.repronim.org/order](http://schema.repronim.org/order) | -| overrideProperties | An array of objects to override the various properties added to assessments and fields. | [http://schema.repronim.org/overrideProperties](http://schema.repronim.org/overrideProperties) | -| preamble | The preamble for an assessment | [http://schema.repronim.org/preamble](http://schema.repronim.org/preamble) | -| shuffle | An element (bool) to determine if the list of items is shuffled or in order. | [http://schema.repronim.org/shuffle](http://schema.repronim.org/shuffle) | -| about | The subject matter of the Field. | [http://schema.org/about](http://schema.org/about) | -| audio | An embedded audio object. | [http://schema.org/audio](http://schema.org/audio) | -| citation | A citation or reference to another creative work, such as another publication, web page, scholarly article, etc. | [http://schema.org/citation](http://schema.org/citation) | -| description | A description of the item. | [http://schema.org/description](http://schema.org/description) | -| image | An image of the item. This can be a URL or a fully described ImageObject. | [http://schema.org/image](http://schema.org/image) | -| schemaVersion | Indicates (by URL or string) a particular version of a schema used in some CreativeWork. For example, a document could declare a schemaVersion using an URL such as http://schema.org/version/2.0/ if precise indication of schema version was required by some application. | [http://schema.org/schemaVersion](http://schema.org/schemaVersion) | -| version | The version of the CreativeWork embodied by a specified resource. | [http://schema.org/version](http://schema.org/version) | -| video | An embedded video object. | [http://schema.org/video](http://schema.org/video) | -| altLabel | The alternate label. | [http://www.w3.org/2004/02/skos/core#altLabel](http://www.w3.org/2004/02/skos/core#altLabel) | -| prefLabel | The preferred label. | [http://www.w3.org/2004/02/skos/core#prefLabel](http://www.w3.org/2004/02/skos/core#prefLabel) | - -### Field -An item in an assessment. - -**URI:** [http://schema.repronim.org/Field](http://schema.repronim.org/Field) - -| Name | Description | URI | -| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| additionalNotesObj | A set of objects to define notes in a field. For example, most Redcap and NDA data dictionaries have notes for each item which needs to be captured in reproschema. | [http://schema.repronim.org/additionalNotesObj](http://schema.repronim.org/additionalNotesObj) | -| inputType | An element to describe the input type of a Field item. | [http://schema.repronim.org/inputType](http://schema.repronim.org/inputType) | -| preamble | The preamble for an assessment | [http://schema.repronim.org/preamble](http://schema.repronim.org/preamble) | -| responseOptions | An element (object or by URL)to describe the properties of response of the Field item. | [http://schema.repronim.org/responseOptions](http://schema.repronim.org/responseOptions) | -| about | The subject matter of the Field. | [http://schema.org/about](http://schema.org/about) | -| audio | An embedded audio object. | [http://schema.org/audio](http://schema.org/audio) | -| description | A description of the item. | [http://schema.org/description](http://schema.org/description) | -| image | An image of the item. This can be a URL or a fully described ImageObject. | [http://schema.org/image](http://schema.org/image) | -| isPartOf | Indicates an item or CreativeWork that this item, or CreativeWork (in some sense), is part of. | [http://schema.org/isPartOf](http://schema.org/isPartOf) | -| question | A sub property of object. A question. | [http://schema.org/question](http://schema.org/question) | -| readonlyValue | Whether or not a property is mutable. Default is false. Specifying this for a property that also has a value makes it act similar to a "hidden" input in an HTML form. | [http://schema.org/readonlyValue](http://schema.org/readonlyValue) | -| schemaVersion | Indicates (by URL or string) a particular version of a schema used in some CreativeWork. For example, a document could declare a schemaVersion using an URL such as http://schema.org/version/2.0/ if precise indication of schema version was required by some application. | [http://schema.org/schemaVersion](http://schema.org/schemaVersion) | -| version | The version of the CreativeWork embodied by a specified resource. | [http://schema.org/version](http://schema.org/version) | -| video | An embedded video object. | [http://schema.org/video](http://schema.org/video) | -| altLabel | The alternate label. | [http://www.w3.org/2004/02/skos/core#altLabel](http://www.w3.org/2004/02/skos/core#altLabel) | -| prefLabel | The preferred label. | [http://www.w3.org/2004/02/skos/core#prefLabel](http://www.w3.org/2004/02/skos/core#prefLabel) | - -### AdditionalProperty -An object to describe the various properties added to assessments and fields. - -**URI:** [http://schema.repronim.org/AdditionalProperty](http://schema.repronim.org/AdditionalProperty) - -| Name | Description | URI | -| -------------- | -------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| allow | An array of items indicating properties allowed on an activity or protocol | [http://schema.repronim.org/allow](http://schema.repronim.org/allow) | -| isAbout | A pointer to the node describing the item. | [http://schema.repronim.org/isAbout](http://schema.repronim.org/isAbout) | -| isVis | An element to describe (by boolean or conditional statement) visibility conditions of items in an assessment. | [http://schema.repronim.org/isVis](http://schema.repronim.org/isVis) | -| limit | An element to limit the duration (uses ISO 8601) this activity is allowed to be completed by once activity is available. | [http://schema.repronim.org/limit](http://schema.repronim.org/limit) | -| maxRetakes | Defines number of times the item is allowed to be redone. | [http://schema.repronim.org/maxRetakes](http://schema.repronim.org/maxRetakes) | -| randomMaxDelay | Present activity/item within some random offset of activity available time up to the maximum specified by this ISO 8601 duration | [http://schema.repronim.org/randomMaxDelay](http://schema.repronim.org/randomMaxDelay) | -| schedule | An element to set make activity available/repeat info using ISO 8601 repeating interval format. | [http://schema.repronim.org/schedule](http://schema.repronim.org/schedule) | -| variableName | The name used to represent an item. | [http://schema.repronim.org/variableName](http://schema.repronim.org/variableName) | -| valueRequired | Whether the property must be filled in to complete the action. Default is false. | [http://schema.org/valueRequired](http://schema.org/valueRequired) | -| prefLabel | The preferred label. | [http://www.w3.org/2004/02/skos/core#prefLabel](http://www.w3.org/2004/02/skos/core#prefLabel) | - -### OverrideProperty -An object to override the various properties added to assessments and fields. - -**URI:** [http://schema.repronim.org/OverrideProperty](http://schema.repronim.org/OverrideProperty) - -| Name | Description | URI | -| -------------- | -------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| isAbout | A pointer to the node describing the item. | [http://schema.repronim.org/isAbout](http://schema.repronim.org/isAbout) | -| isVis | An element to describe (by boolean or conditional statement) visibility conditions of items in an assessment. | [http://schema.repronim.org/isVis](http://schema.repronim.org/isVis) | -| limit | An element to limit the duration (uses ISO 8601) this activity is allowed to be completed by once activity is available. | [http://schema.repronim.org/limit](http://schema.repronim.org/limit) | -| maxRetakes | Defines number of times the item is allowed to be redone. | [http://schema.repronim.org/maxRetakes](http://schema.repronim.org/maxRetakes) | -| randomMaxDelay | Present activity/item within some random offset of activity available time up to the maximum specified by this ISO 8601 duration | [http://schema.repronim.org/randomMaxDelay](http://schema.repronim.org/randomMaxDelay) | -| schedule | An element to set make activity available/repeat info using ISO 8601 repeating interval format. | [http://schema.repronim.org/schedule](http://schema.repronim.org/schedule) | -| variableName | The name used to represent an item. | [http://schema.repronim.org/variableName](http://schema.repronim.org/variableName) | -| valueRequired | Whether the property must be filled in to complete the action. Default is false. | [http://schema.org/valueRequired](http://schema.org/valueRequired) | -| prefLabel | The preferred label. | [http://www.w3.org/2004/02/skos/core#prefLabel](http://www.w3.org/2004/02/skos/core#prefLabel) | - -### UnitOption -An object to represent a human displayable name alongside the more formal value for units. - -**URI:** [http://schema.repronim.org/UnitOption](http://schema.repronim.org/UnitOption) - -| Name | Description | URI | -| --------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| value | The value for each option in choices or in additionalNotesObj | [http://schema.repronim.org/value](http://schema.repronim.org/value) | -| prefLabel | The preferred label. | [http://www.w3.org/2004/02/skos/core#prefLabel](http://www.w3.org/2004/02/skos/core#prefLabel) | - -### ResponseOption -An element (object or by URL)to describe the properties of response of the Field item. - -**URI:** [http://schema.repronim.org/ResponseOption](http://schema.repronim.org/ResponseOption) - -| Name | Description | URI | -| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| choices | An array to list the available options for response of the Field item. | [http://schema.repronim.org/choices](http://schema.repronim.org/choices) | -| datumType | Indicates what type of datum the response is (e.g. range,count,scalar etc.) for the Field item. | [http://schema.repronim.org/datumType](http://schema.repronim.org/datumType) | -| multipleChoice | Indicates (by bool) if response for the Field item has one or more answer. | [http://schema.repronim.org/multipleChoice](http://schema.repronim.org/multipleChoice) | -| unitOptions | A list of objects to represent a human displayable name alongside the more formal value for units. | [http://schema.repronim.org/unitOptions](http://schema.repronim.org/unitOptions) | -| valueType | The type of the response of an item. For example, string, integer, etc. | [http://schema.repronim.org/valueType](http://schema.repronim.org/valueType) | -| maxValue | The upper value of some characteristic or property. | [http://schema.org/maxValue](http://schema.org/maxValue) | -| minValue | The lower value of some characteristic or property. | [http://schema.org/minValue](http://schema.org/minValue) | -| unitCode | The unit of measurement given using the UN/CEFACT Common Code (3 characters) or a URL. Other codes than the UN/CEFACT Common Code may be used with a prefix followed by a colon. | [http://schema.org/unitCode](http://schema.org/unitCode) | - -### Choice -An object to describe a response option. - -**URI:** [http://schema.repronim.org/Choice](http://schema.repronim.org/Choice) - -| Name | Description | URI | -| ----- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | -| value | The value for each option in choices or in additionalNotesObj | [http://schema.repronim.org/value](http://schema.repronim.org/value) | -| image | An image of the item. This can be a URL or a fully described ImageObject. | [http://schema.org/image](http://schema.org/image) | -| name | The name of the item. | [http://schema.org/name](http://schema.org/name) | - -### ComputeSpecification -An object to define computations in an activity or protocol. - -**URI:** [http://schema.repronim.org/ComputeSpecification](http://schema.repronim.org/ComputeSpecification) - -| Name | Description | URI | -| ------------ | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| jsExpression | A JavaScript expression to compute a score from other variables. | [http://schema.repronim.org/jsExpression](http://schema.repronim.org/jsExpression) | -| variableName | The name used to represent an item. | [http://schema.repronim.org/variableName](http://schema.repronim.org/variableName) | - -### MessageSpecification -An object to define messages in an activity or protocol. - -**URI:** [http://schema.repronim.org/MessageSpecification](http://schema.repronim.org/MessageSpecification) - -| Name | Description | URI | -| ------------ | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| jsExpression | A JavaScript expression to compute a score from other variables. | [http://schema.repronim.org/jsExpression](http://schema.repronim.org/jsExpression) | -| message | The message to be conditionally displayed for an item. | [http://schema.repronim.org/message](http://schema.repronim.org/message) | - -### AdditionalNoteObj -A set of objects to define notes in a field. For example, most Redcap and NDA data dictionaries have notes for each item which needs to be captured in reproschema - -**URI:** [http://schema.repronim.org/AdditionalNoteObj](http://schema.repronim.org/AdditionalNoteObj) - -| Name | Description | URI | -| ------ | -------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| column | An element to define the column name where the note was taken from. | [http://schema.repronim.org/column](http://schema.repronim.org/column) | -| source | An element to define the source (eg. RedCap, NDA) where the note was taken from. | [http://schema.repronim.org/source](http://schema.repronim.org/source) | -| value | The value for each option in choices or in additionalNotesObj | [http://schema.repronim.org/value](http://schema.repronim.org/value) | - -### ResponseActivity -Captures information about some action that took place. It also links to information (entities) that were used during the activity - -**URI:** [http://schema.repronim.org/ResponseActivity](http://schema.repronim.org/ResponseActivity) - -| Name | Description | URI | -| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| endedAtTime | None | [http://www.w3.org/ns/prov#endedAtTime](http://www.w3.org/ns/prov#endedAtTime) | -| generated | None | [http://www.w3.org/ns/prov#generated](http://www.w3.org/ns/prov#generated) | -| startedAtTime | None | [http://www.w3.org/ns/prov#startedAtTime](http://www.w3.org/ns/prov#startedAtTime) | -| used | None | [http://www.w3.org/ns/prov#used](http://www.w3.org/ns/prov#used) | -| wasAssociatedWith | None | [http://www.w3.org/ns/prov#wasAssociatedWith](http://www.w3.org/ns/prov#wasAssociatedWith) | -| inLanguage | The language of the content or performance or used in an action. Please use one of the language codes from the [IETF BCP 47 standard](http://tools.ietf.org/html/bcp47). See also [[availableLanguage]]. | [http://schema.org/inLanguage](http://schema.org/inLanguage) | - -### Response -Describes the response of an item. - -**URI:** [http://schema.repronim.org/Response](http://schema.repronim.org/Response) - -| Name | Description | URI | -| --------------- | ------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| wasAttributedTo | None | [http://www.w3.org/ns/prov#wasAttributedTo](http://www.w3.org/ns/prov#wasAttributedTo) | -| isAbout | A pointer to the node describing the item. | [http://schema.repronim.org/isAbout](http://schema.repronim.org/isAbout) | -| value | The value for each option in choices or in additionalNotesObj | [http://schema.repronim.org/value](http://schema.repronim.org/value) | - -### Participant -An Agent describing characteristics associated with a participant. - -**URI:** [http://schema.repronim.org/Participant](http://schema.repronim.org/Participant) - -| Name | Description | URI | -| ---------- | ------------------------------- | -------------------------------------------------------------------------------- | -| subject_id | The identifier of a participant | [http://purl.org/nidash/nidm#subject_id](http://purl.org/nidash/nidm#subject_id) | - -### SoftwareAgent -Captures information about some action that took place. It also links to information (entities) that were used during the activity - -**URI:** [http://schema.repronim.org/SoftwareAgent](http://schema.repronim.org/SoftwareAgent) - -| Name | Description | URI | -| ------- | ----------------------------------------------------------------- | ------------------------------------------------------ | -| url | URL of the item. | [http://schema.org/url](http://schema.org/url) | -| version | The version of the CreativeWork embodied by a specified resource. | [http://schema.org/version](http://schema.org/version) | diff --git a/docs/schema/schema.md b/docs/schema/schema.md new file mode 100644 index 0000000000..9bb6ef5202 --- /dev/null +++ b/docs/schema/schema.md @@ -0,0 +1,94 @@ +# The schema + +## Overview + +A simplistic way to describe the Reprochema is to say it is organized in a hierarchical manner with roughly 3 levels with a schema describing each level. + +1. The highest level is the `protocol` level that originally define a set of assessments or questionnaires to be + included in a given study. + This schema is defined by the [Protocol schema](https://raw.githubusercontent.com/ReproNim/reproschema/master/terms/Protocol). + +1. The second level is the `activity` level that describe a given questionnaire. This level would describe all the questions of this assessment: for example all the items of the Edinburgh handedness inventory would constitute one activity. + This schema is defined by the [Activity schema](https://raw.githubusercontent.com/ReproNim/reproschema/master/terms/Activity). + +1. At a lower level we have the `item` level where each `item` represents a question from a given assessment. + On top of containing the text of the actual question, the schema at this level can contain additional + information such as the expected format of the item for the user interface: a boolean + (if this is a yes/no question), a multiple choice (with a list of the response choices), + a float or an integer (if a numerical value is expected)... + This schema is defined by the [Field schema](https://raw.githubusercontent.com/ReproNim/reproschema/master/terms/Field). + + + +You can see an example of those in the [examples folder](https://github.com/ReproNim/reproschema/tree/master/examples) + +There are in fact more levels than this each and each level has its own schema: + +- all of the schemas can be found in the [`terms` folder](https://github.com/ReproNim/reproschema/tree/master/terms) +- the Reproschema actually allows for a more complex level nesting than the one described above (e.g you can have an `activity` within an `activity`) +- all the properties of each level are described below in the [Properties of ReproSchema objects section](#properties-of-reproschema-objects) + +## Detailed description + +The core model of ReproSchema was initially derived from the [CEDAR Metadata Model](https://more.metadatacenter.org/tools-training/outreach/cedar-template-model). +To accommodate the needs of neuroimaging and other clinical and behavioral +protocols and assessments the schema has evolved significantly. These changes +include: + +1. Alignment with [schema.org](https://schema.org) and [NIDM](https://nidm.nidash.org). + We have used schema.org classes and properties where it maps on to the needs of + the model and extended the model with NIDM elements to harmonize across ReproNim projects. + +1. Allowing for structured nested elements in a schema + `Protocol > Activity > [Activity | Field > ResponseOption]`. This nested + structure provides a flexible schema to represent nested activities, which are + common in biomedical and other domains. + +```text +Protocol +├── Activity +│ ├── Field +│ │ └── ResponseOption +│ └── Field +│ │ └── ResponseOption +│ ├── Activity +│ │ ├── Field +│ │ │ └── ResponseOption +│ │ ... +│ ... +├── Activity +│ ... +... +``` + +1. Interaction with Git/Github or another Web service to deliver a new `protocol`, + `ativity` or `field` with a persistent URI, while tracking changes associated with + any of these elements. By making every `field` represented through a persistent URI, + a data collection instrument can link the responses to the exact `field` that was + used. + +1. Addition of computable elements that are derived from the values entered by a participant. + +1. Allowing for user interface (UI) elements that allow interaction between the schema + and any implementation that allows collecting data using the schema. By providing + some additional UI elements the provider can guide an implementer to allow for + more complex data collection behavior. + +The [ReproSchema-UI](https://repronim.org/reproschema-ui) is a prototype implementation of an UI that leverages these +different elements of the schema. + +## ReproSchema Model + +The model was written using a linked data modeling language [LinkML](https://linkml.io/linkml/). + +There are multiple ways of accessing the content: + +- [The LinkML YAML file that defines the model](https://github.com/ReproNim/reproschema/blob/main/linkml-schema/reproschema.yaml) +- [Automatically generated documentation](./doc-linkml-autogen/index.md) using LinkML tools +- [Automatically generated Entity-Relationship (ER) Diagrams](./erdiagram-autogen.md)) to visualize the model. + +The context was created semi-automatically: the initial version was generated using LinkML tools, +but manual curation was necessary to support all Reeproschema features. diff --git a/mkdocs.yml b/mkdocs.yml index ddbfeb6536..3c1628d2ea 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -39,7 +39,10 @@ nav: - Welcome: "index.md" - Introduction: "introduction.md" - Project structure: "project-structure.md" - - Schema: "schema.md" + - Schema: + - Schema overview: "schema/schema.md" + - Schema documentation: "schema/doc-linkml-autogen/index.md" + - Schema diagram: schema/erdiagram-autogen.md - User Guide: - Create a research protocol: "user-guide/create-new-protocol.md" - Adopt assessments from the library: "user-guide/adopt-assessments.md" @@ -81,6 +84,11 @@ markdown_extensions: - tables - toc: anchorlink: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format watch: - includes diff --git a/mlc_config.json b/mlc_config.json index c3325eda53..f6ae6860bd 100644 --- a/mlc_config.json +++ b/mlc_config.json @@ -14,6 +14,15 @@ }, { "pattern": "^https://doi.org/10.1080/1357650X.2013.783045" + }, + { + "pattern": "^./doc-linkml-autogen/index.md" + }, + { + "pattern": "^./erdiagram-autogen.md" + }, + { + "pattern": "http://www.cogpo.org/" } ] } diff --git a/releases/1.0.0/base b/releases/1.0.0/base new file mode 100644 index 0000000000..ac6fb734d8 --- /dev/null +++ b/releases/1.0.0/base @@ -0,0 +1,18 @@ +{ + "@context": { + "@version": 1.1, + "dct": "http://purl.org/dc/terms/", + "owl": "http://www.w3.org/2002/07/owl#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfa": "http://www.w3.org/ns/rdfa#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "schema": "http://schema.org/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "prov": "http://www.w3.org/ns/prov#", + "pav": "http://purl.org/pav/", + "nidm": "http://purl.org/nidash/nidm#", + "uuid": "http://uuid.repronim.org/", + "reproschema": "http://schema.repronim.org/" + } +} diff --git a/requirements.txt b/requirements.txt index 044ec90ae1..5c6a3a7020 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ mkdocs-material pymdown-extensions rdflib==5.0.0 ruamel.yaml +mkdocs-mermaid2-plugin diff --git a/scripts/editProperties.py b/scripts/editProperties.py deleted file mode 100644 index b6f1ce9217..0000000000 --- a/scripts/editProperties.py +++ /dev/null @@ -1,104 +0,0 @@ -import sys - -import rdflib as rl -from pytablewriter import MarkdownTableWriter - -AUTO_LINE = "" - - -def edit_properties(version): - targetClasses = [ - "Protocol", - "Activity", - "Field", - "AdditionalProperty", - "OverrideProperty", - "UnitOption", - "ResponseOption", - "Choice", - "ComputeSpecification", - "MessageSpecification", - "AdditionalNoteObj", - "ResponseActivity", - "Response", - "Participant", - "SoftwareAgent", - ] - - with open("docs/schema.md") as fp: - doc = fp.readlines() - for idx, line in enumerate(doc): - if line.startswith(AUTO_LINE): - break - doc = doc[:idx] - doc.append(AUTO_LINE) - doc.append( - """\n -""" - ) - - # read all the terms - g = rl.Graph() - g.parse( - "https://raw.githubusercontent.com/schemaorg/schemaorg/main/data/releases/9.0/schema.ttl", - format="turtle", - ) - g.parse(f"releases/{version}/reproschema.ttl", format="turtle") - g.parse(f"releases/{version}/reproschema-shacl.ttl", format="turtle") - - query = """PREFIX sh: -SELECT ?obj ?prop ?desc ?objdesc -WHERE { - ?obj1 a sh:NodeShape; - sh:property [ sh:path ?prop ]; - sh:targetClass ?obj . - OPTIONAL { ?prop rdfs:comment ?desc .} - OPTIONAL { ?obj rdfs:comment ?objdesc . } -} -""" - results = list(g.query(query)) - - objects = {k: {} for k in targetClasses} - for res in results: - s = g.compute_qname(res[0]) - p = g.compute_qname(res[1]) - if s[2] not in objects: - raise ValueError(f"Unknown class: {s[2]}. Source: {res}") - objects[s[2]][p] = str(res[2]) - objects[s[2]]["klass_desc"] = str(res[3]) - for klass, props in objects.items(): - writer = MarkdownTableWriter() - writer.table_name = ( - f"{klass}\n" - f"{props['klass_desc']}\n\n" - f"**URI:** [http://schema.repronim.org/{klass}](http://schema.repronim.org/{klass})\n" - ) - writer.headers = ["Name", "Description", "URI"] - writer.value_matrix = [] - del props["klass_desc"] - for val, comment in sorted(props.items()): - property = [ - val[2], - comment, - f"[{val[1]}{val[2]}]({val[1]}{val[2]})", - ] - writer.value_matrix.append(property) - writer.margin = 1 # add a whitespace for both sides of each cell - doc.append("\n") - doc.append("##" + writer.dumps()) - with open("docs/30_schema.md", "wt") as fp: - fp.writelines(doc) - - -if __name__ == "__main__": - if len(sys.argv) < 2: - version = "master" - else: - version = sys.argv[1] - print(f"Generating classes and properties for version: {version}") - edit_properties(version) diff --git a/scripts/fix_pydantic.py b/scripts/fix_pydantic.py index b82f5d295a..5fdf4bbac3 100644 --- a/scripts/fix_pydantic.py +++ b/scripts/fix_pydantic.py @@ -1,4 +1,8 @@ -""" Using ast transformer to fix issues with automatic pydantic generation""" +""" Using ast transformer to fix issues with automatic pydantic generation +Currently the main issue is with the LangString representation in the pydantic +model, I'm changing it to Dict[str, str]. +Perhaps in the future this script will not be required. +""" import ast import sys diff --git a/scripts/makeRelease.py b/scripts/makeRelease.py deleted file mode 100644 index f933d973a1..0000000000 --- a/scripts/makeRelease.py +++ /dev/null @@ -1,59 +0,0 @@ -import json -import os -import sys -from shutil import copyfile - -import rdflib as rl -from pyld import jsonld - -from reproschema.jsonldutils import to_newformat - - -def create_release(version): - # read all the terms - terms = [] - for root, _, files in os.walk("terms"): - for name in files: - terms.extend( - json.loads(to_newformat(os.path.join(root, name), "jsonld")) - ) - - kwargs = {"algorithm": "URDNA2015", "format": "application/n-quads"} - data = jsonld.normalize(terms, kwargs) - - os.makedirs(f"releases/{version}", exist_ok=True) - copyfile("contexts/base", f"releases/{version}/base") - copyfile("contexts/generic", f"releases/{version}/generic") - with open(f"releases/{version}/base") as fp: - base_context = json.load(fp) - compacted = jsonld.compact(terms, ctx=base_context) - - g = rl.Graph() - for key, val in base_context["@context"].items(): - if not key.startswith("@"): - g.namespace_manager.bind(key, val) - g.parse(data=data, format="nt") - - # write n-triples and turtle files - with open(f"releases/{version}/reproschema.jsonld", "w") as fp: - json.dump(compacted, fp, indent=2) - with open(f"releases/{version}/reproschema.nt", "w") as fp: - fp.write(data) - with open(f"releases/{version}/reproschema.ttl", "w") as fp: - fp.write(g.serialize(format="turtle")) - - g1 = rl.Graph() - g1.parse("validation/reproschema-shacl.ttl", format="turtle") - with open(f"releases/{version}/reproschema-shacl.ttl", "w") as fp: - fp.write(g1.serialize(format="turtle")) - - return g - - -if __name__ == "__main__": - if len(sys.argv) < 2: - version = "master" - else: - version = sys.argv[1] - print(f"Generating release for version: {version}") - create_release(version)