From 6238fb968a3a8b6709dbe8bfc1fd53e41eec8f15 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 5 Nov 2025 13:40:21 -0800 Subject: [PATCH 01/11] added new tutorial --- .../python/create_json_schema copy.md | 45 +++++++++++++++++++ .../tutorial_scripts/create_json_schema.py | 24 ++++++++++ 2 files changed, 69 insertions(+) create mode 100644 docs/tutorials/python/create_json_schema copy.md create mode 100644 docs/tutorials/python/tutorial_scripts/create_json_schema.py diff --git a/docs/tutorials/python/create_json_schema copy.md b/docs/tutorials/python/create_json_schema copy.md new file mode 100644 index 000000000..3aa986e01 --- /dev/null +++ b/docs/tutorials/python/create_json_schema copy.md @@ -0,0 +1,45 @@ +JSON Schema is a tool used to validate data. In Synapse, JSON Schemas can be used to validate the metadata applied to an entity such as project, file, folder, table, or view, including the [annotations](https://help.synapse.org/docs/Annotating-Data-With-Metadata.2667708522.html) applied to it. To learn more about JSON Schemas, check out [JSON-Schema.org](https://json-schema.org/). + +Synapse supports a subset of features from [json-schema-draft-07](https://json-schema.org/draft-07). To see the list of features currently supported, see the [JSON Schema object definition](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/JsonSchema.html) from Synapse's REST API Documentation + +In this tutorial, you will learn how to create theses JSON Schema using an existing data-model + +## Tutorial Purpose +You will create a JSON schema using your data model + +## Prerequisites +* You have a working [installation](../installation.md) of the Synapse Python Client. + + +## 1. Set up your variables + +```python +{!docs/tutorials/python/tutorial_scripts/create_json_schema.py!lines=6-8} +``` + +To create a JSON Schema you need a data-model, and the data-type you want to create. The data-model must be in either CSV or JSON-LD form. See [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) for instructions on how to crate a data-model.The data-type must exist in your data-model. + +## 2. Create the JSON Schema +Try creating the JSON Schema +```python +{!docs/tutorials/python/tutorial_scripts/json_schema.py!lines=10-16} +``` + + + +## Source Code for this Tutorial + +
+ Click to show me + +```python +{!docs/tutorials/python/tutorial_scripts/create_json_schema.py!} +``` +
+ + +## Reference +- [Data models](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) +- [JSON Schema Object Definition](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/JsonSchema.html) +- [JSON Schema Draft 7](https://json-schema.org/draft-07) +- [JSON-Schema.org](https://json-schema.org./) diff --git a/docs/tutorials/python/tutorial_scripts/create_json_schema.py b/docs/tutorials/python/tutorial_scripts/create_json_schema.py new file mode 100644 index 000000000..2227f8947 --- /dev/null +++ b/docs/tutorials/python/tutorial_scripts/create_json_schema.py @@ -0,0 +1,24 @@ +import logging + +from synapseclient.extensions.curator.schema_generation import ( + DataModelGraph, + DataModelGraphExplorer, + DataModelParser, + create_json_schema, +) + +LOGGER = logging.Logger("create_json_schema") +# Put the path to your data model here, either CSV or JSONLD format +DATA_MODEL_PATH = "tests/unit/synapseclient/extensions/schema_files/example.model.csv" +# Put the name of the datatype in your data-model you want to create here +DATATYPE = "Patient" + +data_model_parser = DataModelParser(path_to_data_model=DATA_MODEL_PATH, logger=LOGGER) +parsed_data_model = data_model_parser.parse_model() +data_model_grapher = DataModelGraph(parsed_data_model) +graph_data_model = data_model_grapher.graph +dmge = DataModelGraphExplorer(graph_data_model, logger=LOGGER) +json_schema = create_json_schema( + dmge, datatype=DATATYPE, schema_name=DATATYPE, logger=LOGGER +) +print(json_schema) From 340eec05f566539b7f26f64ea509a654896a867c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 5 Nov 2025 13:43:20 -0800 Subject: [PATCH 02/11] rename file --- .../python/{create_json_schema copy.md => create_json_schema.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/tutorials/python/{create_json_schema copy.md => create_json_schema.md} (100%) diff --git a/docs/tutorials/python/create_json_schema copy.md b/docs/tutorials/python/create_json_schema.md similarity index 100% rename from docs/tutorials/python/create_json_schema copy.md rename to docs/tutorials/python/create_json_schema.md From 5d6c107f08c710ef3830b2ba5258642b00449e40 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 5 Nov 2025 13:46:14 -0800 Subject: [PATCH 03/11] fixes to tutorial --- docs/tutorials/python/create_json_schema.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorials/python/create_json_schema.md b/docs/tutorials/python/create_json_schema.md index 3aa986e01..271d02401 100644 --- a/docs/tutorials/python/create_json_schema.md +++ b/docs/tutorials/python/create_json_schema.md @@ -2,7 +2,7 @@ JSON Schema is a tool used to validate data. In Synapse, JSON Schemas can be use Synapse supports a subset of features from [json-schema-draft-07](https://json-schema.org/draft-07). To see the list of features currently supported, see the [JSON Schema object definition](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/JsonSchema.html) from Synapse's REST API Documentation -In this tutorial, you will learn how to create theses JSON Schema using an existing data-model +In this tutorial, you will learn how to create these JSON Schema using an existing data-model ## Tutorial Purpose You will create a JSON schema using your data model @@ -14,7 +14,7 @@ You will create a JSON schema using your data model ## 1. Set up your variables ```python -{!docs/tutorials/python/tutorial_scripts/create_json_schema.py!lines=6-8} +{!docs/tutorials/python/tutorial_scripts/create_json_schema.py!lines=10-14} ``` To create a JSON Schema you need a data-model, and the data-type you want to create. The data-model must be in either CSV or JSON-LD form. See [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) for instructions on how to crate a data-model.The data-type must exist in your data-model. @@ -22,9 +22,9 @@ To create a JSON Schema you need a data-model, and the data-type you want to cre ## 2. Create the JSON Schema Try creating the JSON Schema ```python -{!docs/tutorials/python/tutorial_scripts/json_schema.py!lines=10-16} +{!docs/tutorials/python/tutorial_scripts/json_schema.py!lines=16-24} ``` - +You should see the JSON Schema for the datatype you selected printed. ## Source Code for this Tutorial From a27e7ebc0c4ae56a1c36abf1a38486e600add8ff Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 5 Nov 2025 13:46:48 -0800 Subject: [PATCH 04/11] fixes to tutorial --- docs/tutorials/python/create_json_schema.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/python/create_json_schema.md b/docs/tutorials/python/create_json_schema.md index 271d02401..ca0cec1f9 100644 --- a/docs/tutorials/python/create_json_schema.md +++ b/docs/tutorials/python/create_json_schema.md @@ -1,11 +1,11 @@ JSON Schema is a tool used to validate data. In Synapse, JSON Schemas can be used to validate the metadata applied to an entity such as project, file, folder, table, or view, including the [annotations](https://help.synapse.org/docs/Annotating-Data-With-Metadata.2667708522.html) applied to it. To learn more about JSON Schemas, check out [JSON-Schema.org](https://json-schema.org/). -Synapse supports a subset of features from [json-schema-draft-07](https://json-schema.org/draft-07). To see the list of features currently supported, see the [JSON Schema object definition](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/JsonSchema.html) from Synapse's REST API Documentation +Synapse supports a subset of features from [json-schema-draft-07](https://json-schema.org/draft-07). To see the list of features currently supported, see the [JSON Schema object definition](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/JsonSchema.html) from Synapse's REST API Documentation. -In this tutorial, you will learn how to create these JSON Schema using an existing data-model +In this tutorial, you will learn how to create these JSON Schema using an existing data-model. ## Tutorial Purpose -You will create a JSON schema using your data model +You will create a JSON schema using your data model. ## Prerequisites * You have a working [installation](../installation.md) of the Synapse Python Client. From 8d1bea543f07fff99c94d43a35db3c1d4bbe0ad0 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 5 Nov 2025 13:54:45 -0800 Subject: [PATCH 05/11] renamed files --- .../python/{create_json_schema.md => schema_operations.md} | 6 +++--- .../{create_json_schema.py => schema_operations.py} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename docs/tutorials/python/{create_json_schema.md => schema_operations.md} (90%) rename docs/tutorials/python/tutorial_scripts/{create_json_schema.py => schema_operations.py} (100%) diff --git a/docs/tutorials/python/create_json_schema.md b/docs/tutorials/python/schema_operations.md similarity index 90% rename from docs/tutorials/python/create_json_schema.md rename to docs/tutorials/python/schema_operations.md index ca0cec1f9..032f9fc70 100644 --- a/docs/tutorials/python/create_json_schema.md +++ b/docs/tutorials/python/schema_operations.md @@ -14,7 +14,7 @@ You will create a JSON schema using your data model. ## 1. Set up your variables ```python -{!docs/tutorials/python/tutorial_scripts/create_json_schema.py!lines=10-14} +{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=10-14} ``` To create a JSON Schema you need a data-model, and the data-type you want to create. The data-model must be in either CSV or JSON-LD form. See [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) for instructions on how to crate a data-model.The data-type must exist in your data-model. @@ -22,7 +22,7 @@ To create a JSON Schema you need a data-model, and the data-type you want to cre ## 2. Create the JSON Schema Try creating the JSON Schema ```python -{!docs/tutorials/python/tutorial_scripts/json_schema.py!lines=16-24} +{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=16-24} ``` You should see the JSON Schema for the datatype you selected printed. @@ -33,7 +33,7 @@ You should see the JSON Schema for the datatype you selected printed. Click to show me ```python -{!docs/tutorials/python/tutorial_scripts/create_json_schema.py!} +{!docs/tutorials/python/tutorial_scripts/schema_operations.py!} ``` diff --git a/docs/tutorials/python/tutorial_scripts/create_json_schema.py b/docs/tutorials/python/tutorial_scripts/schema_operations.py similarity index 100% rename from docs/tutorials/python/tutorial_scripts/create_json_schema.py rename to docs/tutorials/python/tutorial_scripts/schema_operations.py From 8557276adf11f27df84ae731acfe520b76cd89f6 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 5 Nov 2025 13:58:17 -0800 Subject: [PATCH 06/11] added tutorial to yaml file --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index cca09f73a..63b4d27db 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: # - Team: tutorials/python/team.md - Upload data in bulk: tutorials/python/upload_data_in_bulk.md - Download data in bulk: tutorials/python/download_data_in_bulk.md + - Creating JSON Schema: tutorials/python/schema_operations.md - Working with JSON Schema: tutorials/python/json_schema.md # - Move Files and Folders: tutorials/python/move_files_and_folders.md # - Migrate data to other storage locations: tutorials/python/migrate_data_to_other_storage_locations.md From e338c9f4550cc3b6578977060175976e5a35234c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 6 Nov 2025 08:58:59 -0800 Subject: [PATCH 07/11] add data model assumption --- docs/tutorials/python/schema_operations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tutorials/python/schema_operations.md b/docs/tutorials/python/schema_operations.md index 032f9fc70..4d273a3fb 100644 --- a/docs/tutorials/python/schema_operations.md +++ b/docs/tutorials/python/schema_operations.md @@ -9,6 +9,7 @@ You will create a JSON schema using your data model. ## Prerequisites * You have a working [installation](../installation.md) of the Synapse Python Client. +* You have a data-model, see [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) ## 1. Set up your variables From 745976e62f12d90a548da20be167ded431aa759c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 7 Nov 2025 12:38:30 -0800 Subject: [PATCH 08/11] used different function for tutorial --- docs/tutorials/python/schema_operations.md | 28 +++++++++---- .../tutorial_scripts/schema_operations.py | 39 +++++++++---------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/docs/tutorials/python/schema_operations.md b/docs/tutorials/python/schema_operations.md index 4d273a3fb..6f412b7ce 100644 --- a/docs/tutorials/python/schema_operations.md +++ b/docs/tutorials/python/schema_operations.md @@ -11,21 +11,35 @@ You will create a JSON schema using your data model. * You have a working [installation](../installation.md) of the Synapse Python Client. * You have a data-model, see [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) +## 1. Imports -## 1. Set up your variables +```python +{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=1-2} +``` + +## 2. Set up your variables + +```python +{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=4-10} +``` + +To create a JSON Schema you need a data-model, and the data-types you want to create. +The data-model must be in either CSV or JSON-LD form. See [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) for instructions on how to crate a data-model. The data model maybe a local path or a URL. + +The data-types must exist in your data-model. This cna be a list of data-types, or `None` to create all datatatypes in the data-model. +## 3. Log into Synapse ```python -{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=10-14} +{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=12-13} ``` -To create a JSON Schema you need a data-model, and the data-type you want to create. The data-model must be in either CSV or JSON-LD form. See [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) for instructions on how to crate a data-model.The data-type must exist in your data-model. -## 2. Create the JSON Schema -Try creating the JSON Schema +## 4. Create the JSON Schema +Create the JSON Schema(s) ```python -{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=16-24} +{!docs/tutorials/python/tutorial_scripts/schema_operations.py!lines=15-23} ``` -You should see the JSON Schema for the datatype you selected printed. +You should see the first JSON Schema for the datatype(s) you selected printed. ## Source Code for this Tutorial diff --git a/docs/tutorials/python/tutorial_scripts/schema_operations.py b/docs/tutorials/python/tutorial_scripts/schema_operations.py index 2227f8947..20555bc76 100644 --- a/docs/tutorials/python/tutorial_scripts/schema_operations.py +++ b/docs/tutorials/python/tutorial_scripts/schema_operations.py @@ -1,24 +1,23 @@ -import logging +from synapseclient import Synapse +from synapseclient.extensions.curator.schema_generation import generate_jsonschema -from synapseclient.extensions.curator.schema_generation import ( - DataModelGraph, - DataModelGraphExplorer, - DataModelParser, - create_json_schema, -) +# Put the path/url of your data model here, either CSV or JSONLD format +DATA_MODEL_SOURCE = "tests/unit/synapseclient/extensions/schema_files/example.model.csv" +# Put the names of the datatypes in your data-model you want to create here +# or None to create them all +DATA_TYPE = ["Patient"] +# Put the directory where you want the JSONSchema to generated at here +OUTPUT_DIRECTORY = "./" -LOGGER = logging.Logger("create_json_schema") -# Put the path to your data model here, either CSV or JSONLD format -DATA_MODEL_PATH = "tests/unit/synapseclient/extensions/schema_files/example.model.csv" -# Put the name of the datatype in your data-model you want to create here -DATATYPE = "Patient" +syn = Synapse() +syn.login() -data_model_parser = DataModelParser(path_to_data_model=DATA_MODEL_PATH, logger=LOGGER) -parsed_data_model = data_model_parser.parse_model() -data_model_grapher = DataModelGraph(parsed_data_model) -graph_data_model = data_model_grapher.graph -dmge = DataModelGraphExplorer(graph_data_model, logger=LOGGER) -json_schema = create_json_schema( - dmge, datatype=DATATYPE, schema_name=DATATYPE, logger=LOGGER +schemas, file_paths = generate_jsonschema( + data_model_source=DATA_MODEL_SOURCE, + output_directory=OUTPUT_DIRECTORY, + data_type=DATA_TYPE, + data_model_labels="class_label", + synapse_client=syn, ) -print(json_schema) + +print(schemas[0]) From 9b7bd9d2509ee3bb684ba8328fe7239196d1b10f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 7 Nov 2025 12:50:50 -0800 Subject: [PATCH 09/11] remove links --- docs/tutorials/python/schema_operations.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tutorials/python/schema_operations.md b/docs/tutorials/python/schema_operations.md index 6f412b7ce..20ce5d4c8 100644 --- a/docs/tutorials/python/schema_operations.md +++ b/docs/tutorials/python/schema_operations.md @@ -24,7 +24,7 @@ You will create a JSON schema using your data model. ``` To create a JSON Schema you need a data-model, and the data-types you want to create. -The data-model must be in either CSV or JSON-LD form. See [here](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) for instructions on how to crate a data-model. The data model maybe a local path or a URL. +The data-model must be in either CSV or JSON-LD form. The data model maybe a local path or a URL. The data-types must exist in your data-model. This cna be a list of data-types, or `None` to create all datatatypes in the data-model. @@ -54,7 +54,6 @@ You should see the first JSON Schema for the datatype(s) you selected printed. ## Reference -- [Data models](https://sagebionetworks.jira.com/wiki/spaces/SCHEM/pages/2473623559/The+Data+Model+Schema) - [JSON Schema Object Definition](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/schema/JsonSchema.html) - [JSON Schema Draft 7](https://json-schema.org/draft-07) - [JSON-Schema.org](https://json-schema.org./) From 8bc8ee160728ece93ec42d0f6b825cc6fd6d0414 Mon Sep 17 00:00:00 2001 From: andrewelamb Date: Fri, 7 Nov 2025 13:12:38 -0800 Subject: [PATCH 10/11] Update docs/tutorials/python/tutorial_scripts/schema_operations.py Co-authored-by: BryanFauble <17128019+BryanFauble@users.noreply.github.com> --- docs/tutorials/python/tutorial_scripts/schema_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/python/tutorial_scripts/schema_operations.py b/docs/tutorials/python/tutorial_scripts/schema_operations.py index 20555bc76..c24ae077b 100644 --- a/docs/tutorials/python/tutorial_scripts/schema_operations.py +++ b/docs/tutorials/python/tutorial_scripts/schema_operations.py @@ -1,5 +1,5 @@ from synapseclient import Synapse -from synapseclient.extensions.curator.schema_generation import generate_jsonschema +from synapseclient.extensions.curator import generate_jsonschema # Put the path/url of your data model here, either CSV or JSONLD format DATA_MODEL_SOURCE = "tests/unit/synapseclient/extensions/schema_files/example.model.csv" From d0d9c0de4c7796982f9ec6641e3793fb676cd1e3 Mon Sep 17 00:00:00 2001 From: andrewelamb Date: Fri, 7 Nov 2025 13:12:54 -0800 Subject: [PATCH 11/11] Update docs/tutorials/python/schema_operations.md Co-authored-by: BryanFauble <17128019+BryanFauble@users.noreply.github.com> --- docs/tutorials/python/schema_operations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/python/schema_operations.md b/docs/tutorials/python/schema_operations.md index 20ce5d4c8..03c49209f 100644 --- a/docs/tutorials/python/schema_operations.md +++ b/docs/tutorials/python/schema_operations.md @@ -24,7 +24,7 @@ You will create a JSON schema using your data model. ``` To create a JSON Schema you need a data-model, and the data-types you want to create. -The data-model must be in either CSV or JSON-LD form. The data model maybe a local path or a URL. +The data-model must be in either CSV or JSON-LD form. The data model may be a local path or a URL. The data-types must exist in your data-model. This cna be a list of data-types, or `None` to create all datatatypes in the data-model.