Merge remote-tracking branch 'origin/main' into csv-test

Author: joshmoore

.github/workflows/main.yaml

@@ -1,6 +1,6 @@
 # Built from:
 # https://docs.github.com/en/actions/guides/building-and-testing-python
-# https://github.com/snok/install-poetry#workflows-and-tips
+# https://github.com/actions/setup-python/
 
 name: Build and test linkml-runtime
 
@@ -23,42 +23,24 @@ jobs:
 
     steps:
 
+      #----------------------------------------------
+      #          install poetry
+      #----------------------------------------------
+      - name: Install Poetry
+        # Pin to 1.3.2 to workaround https://github.com/python-poetry/poetry/issues/7611
+        run: pipx install poetry==1.3.2
+        
       #----------------------------------------------
       #       check-out repo and set-up python
       #----------------------------------------------
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      #----------------------------------------------
-      #          install & configure poetry
-      #----------------------------------------------
-      - name: Install Poetry
-        uses: snok/install-poetry@v1.3
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-
-      #----------------------------------------------
-      #       load cached venv if cache exists      
-      #----------------------------------------------
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v2
-        with:
-          path: .venv
-          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
-
-      #----------------------------------------------
-      # install dependencies if cache does not exist 
-      #----------------------------------------------
-      - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
+          cache: 'poetry'
 
       #----------------------------------------------
       #    install your root project, if required 
@@ -85,7 +67,7 @@ jobs:
       #           upload coverage results
       #----------------------------------------------
       - name: Upload coverage report
-        uses: codecov/codecov-action@v1.0.5
+        uses: codecov/codecov-action@v3
         with:
           name: codecov-results-${{ matrix.os }}-${{ matrix.python-version }}
           token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/pypi-publish.yaml

@@ -18,10 +18,7 @@ jobs:
         python-version: 3.8
 
     - name: Install Poetry
-      uses: snok/install-poetry@v1.1.6
-      with:
-        virtualenvs-create: true
-        virtualenvs-in-project: true
+      run: pipx install poetry
 
     - name: Install dependencies
       run: poetry install --no-interaction

CONTRIBUTING.md

@@ -6,14 +6,17 @@ When contributing to this repository, please first discuss the changes you wish
 
 ### Reporting bugs or making feature requests
 
-You can use the [Issues](https://github.com/linkml/linkml-runtime/issues) tab to create bug and feature requests. Providing enough details to the developers to verify and troubleshoot your issue is paramount:
+To report a bug or suggest a new feature, please go to the [linkml/linkml issue tracker](https://github.com/linkml/linkml/issues), as we are
+consolidating issues there.
+
+Please supply enough details to the developers to enable them to verify and troubleshoot your issue:
 
 * Provide a clear and descriptive title as well as a concise summary of the issue to identify the problem.
 * Describe the exact steps which reproduce the problem in as many details as possible.
 * Describe the behavior you observed after following the steps and point out what exactly is the problem with that behavior.
 * Explain which behavior you expected to see instead and why.
 * Provide screenshots of the expected or actual behaviour where applicable.
-* Tag the issue with appropriate label names. For ex., if your issue is about a bug that needs to be fixed in the repo, tag the issue with the `bug` label from the list of [labels](https://github.com/linkml/linkml-runtime/labels). If an appropriate label name doesn't exist, create one, with a name that clearly encapsulates the topic of the issue.
+* Tag the issue with appropriate label names. For example, if your issue is about a bug that needs to be fixed in the repo, tag the issue with the `bug` label from the list of [labels](https://github.com/linkml/linkml-runtime/labels). If an appropriate label doesn't exist, create one, with a name that clearly encapsulates the topic of the issue.
 
 ### The development lifecycle
 
@@ -24,6 +27,8 @@ You can use the [Issues](https://github.com/linkml/linkml-runtime/issues) tab to
 
 Note: All the development must be done on a branch on your fork.
 
+ALSO NOTE: github.com lets you create a pull request from the main branch, automating the steps above.
+
 > A code review (which happens with both the contributor and the reviewer present) is required for contributing.
 
 ## Development environment setup

Makefile

@@ -11,3 +11,7 @@ update_model:
 
 test:
 	poetry run python -m unittest discover
+
+# temporary measure until linkml-model is synced
+linkml_runtime/processing/validation_datamodel.py: linkml_runtime/processing/validation_datamodel.yaml
+	gen-python $< > $@.tmp && mv $@.tmp $@

README.md

@@ -7,7 +7,7 @@
 [![PyPIDownloadsMonth](https://img.shields.io/pypi/dm/linkml-runtime?logo=PyPI&color=blue)](https://pypi.org/project/linkml-runtime)
 [![codecov](https://codecov.io/gh/linkml/linkml-runtime/branch/main/graph/badge.svg?token=FOBHNSK5WG)](https://codecov.io/gh/linkml/linkml-runtime)
 
-Runtime support for linkml generated models
+Runtime support for linkml generated data classes
 
 ## About
 

linkml_runtime/__init__.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from linkml_runtime.utils.curienamespace import CurieNamespace
 from linkml_runtime.utils.schemaview import SchemaView
 from rdflib import RDF, RDFS, SKOS, XSD, OWL
@@ -21,6 +22,13 @@
 __version__ = importlib_metadata.version(__name__)
 
 
+THIS_PATH = Path(__file__).parent
+
+SCHEMA_DIRECTORY = THIS_PATH / "linkml_model" / "model" / "schema"
+
+MAIN_SCHEMA_PATH = SCHEMA_DIRECTORY / "meta.yaml"
+
+
 class MappingError(ValueError):
     """
     An error when mapping elements of a LinkML model to runtime objects

linkml_runtime/dumpers/__init__.py

@@ -1,6 +1,7 @@
 from linkml_runtime.dumpers.json_dumper import JSONDumper
 from linkml_runtime.dumpers.rdf_dumper import RDFDumper
 from linkml_runtime.dumpers.rdflib_dumper import RDFLibDumper
+from linkml_runtime.dumpers.tsv_dumper import TSVDumper
 from linkml_runtime.dumpers.yaml_dumper import YAMLDumper
 from linkml_runtime.dumpers.csv_dumper import CSVDumper
 
@@ -9,3 +10,4 @@
 rdflib_dumper = RDFLibDumper()
 yaml_dumper = YAMLDumper()
 csv_dumper = CSVDumper()
+tsv_dumper = TSVDumper()

linkml_runtime/dumpers/csv_dumper.py

@@ -1,33 +1,8 @@
-import io
-import yaml
-import json
-from typing import Dict, List, Any
+from linkml_runtime.dumpers.delimited_file_dumper import DelimitedFileDumper
 
-from linkml_runtime.dumpers.dumper_root import Dumper
-from linkml_runtime.dumpers.json_dumper import JSONDumper
-from linkml_runtime.utils.yamlutils import YAMLRoot
-from linkml_runtime.linkml_model.meta import SlotDefinitionName, SchemaDefinition
-from linkml_runtime.utils.schemaview import SchemaView
 
-from linkml_runtime.utils.csvutils import GlobalConfig, get_configmap
-from json_flattener import flatten_to_csv
+class CSVDumper(DelimitedFileDumper):
 
-
-class CSVDumper(Dumper):
-
-    def dumps(self, element: YAMLRoot,
-              index_slot: SlotDefinitionName = None,
-              schema: SchemaDefinition = None,
-              schemaview: SchemaView = None,
-              **kwargs) -> str:
-        """ Return element formatted as CSV lines """
-        json_dumper = JSONDumper()
-        element_j = json.loads(json_dumper.dumps(element))
-        objs = element_j[index_slot]
-        if schemaview is None:
-            schemaview = SchemaView(schema)
-        configmap = get_configmap(schemaview, index_slot)
-        config = GlobalConfig(key_configs=configmap)
-        output = io.StringIO()
-        flatten_to_csv(objs, output, config=config, **kwargs)
-        return output.getvalue()
+    @property
+    def delimiter(self):
+        return ","

linkml_runtime/dumpers/delimited_file_dumper.py

@@ -0,0 +1,40 @@
+import io
+import yaml
+import json
+from abc import ABC, abstractmethod
+from typing import Union
+from pydantic import BaseModel
+
+from linkml_runtime.dumpers.dumper_root import Dumper
+from linkml_runtime.dumpers.json_dumper import JSONDumper
+from linkml_runtime.utils.yamlutils import YAMLRoot
+from linkml_runtime.linkml_model.meta import SlotDefinitionName, SchemaDefinition
+from linkml_runtime.utils.schemaview import SchemaView
+
+from linkml_runtime.utils.csvutils import GlobalConfig, get_configmap
+from json_flattener import flatten_to_csv
+
+
+class DelimitedFileDumper(Dumper, ABC):
+
+    @property
+    @abstractmethod
+    def delimiter(self):
+        pass
+
+    def dumps(self, element: Union[BaseModel, YAMLRoot],
+              index_slot: SlotDefinitionName = None,
+              schema: SchemaDefinition = None,
+              schemaview: SchemaView = None,
+              **kwargs) -> str:
+        """ Return element formatted as CSV lines """
+        json_dumper = JSONDumper()
+        element_j = json.loads(json_dumper.dumps(element))
+        objs = element_j[index_slot]
+        if schemaview is None:
+            schemaview = SchemaView(schema)
+        configmap = get_configmap(schemaview, index_slot)
+        config = GlobalConfig(key_configs=configmap, csv_delimiter=self.delimiter)
+        output = io.StringIO()
+        flatten_to_csv(objs, output, config=config, **kwargs)
+        return output.getvalue()

linkml_runtime/dumpers/dumper_root.py

@@ -1,12 +1,14 @@
 from abc import ABC, abstractmethod
+from typing import Union
 
 from linkml_runtime.utils.yamlutils import YAMLRoot
+from pydantic import BaseModel
 
 
 class Dumper(ABC):
     """ Abstract base class for all dumpers """
 
-    def dump(self, element: YAMLRoot, to_file: str, **_) -> None:
+    def dump(self, element: Union[BaseModel, YAMLRoot], to_file: str, **_) -> None:
         """
         Write element to to_file
         :param element: LinkML object to be dumped
@@ -17,10 +19,10 @@ def dump(self, element: YAMLRoot, to_file: str, **_) -> None:
             output_file.write(self.dumps(element, **_))
 
     @abstractmethod
-    def dumps(self, element: YAMLRoot, **_) -> str:
+    def dumps(self, element: Union[BaseModel, YAMLRoot], **_) -> str:
         """
         Convert element to a string
-        @param element: YAMLRoot object to be rendered
+        @param element: Union[BaseModel, YAMLRoot] object to be rendered
         @param _: method specific arguments
         @return: stringified representation of element
         """