Merge branch 'main' into ODSC-34935/add_flex_shape_opctl_config

Author: lu-ohai

.github/workflows/run-unittests-default_setup.yml

@@ -1,6 +1,7 @@
 name: tests/unitary/default_setup/**
 
 on:
+  workflow_dispatch:
   pull_request:
     branches:
       - main

.github/workflows/run-unittests.yml

@@ -1,6 +1,7 @@
 name: tests/unitary/**
 
 on:
+  workflow_dispatch:
   pull_request:
     branches:
       - main
@@ -41,7 +42,7 @@ jobs:
         test-path: ["tests/unitary", "tests/unitary/with_extras/model"]
         include:
           - test-path: "tests/unitary"
-            ignore-path: "tests/unitary/with_extras/model"
+            ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
             name: "unitary"
           - test-path: "tests/unitary/with_extras/model"
             name: "model"
@@ -115,8 +116,7 @@ jobs:
           # Run tests
           python -m pytest -v -p no:warnings --durations=5 \
           -n auto --dist loadfile ${{ matrix.cov-reports }} \
-          ${{ matrix.test-path }} \
-          --ignore "${{ matrix.ignore-path }}"
+          ${{ matrix.test-path }} ${{ matrix.ignore-path }}
 
       - name: "Save coverage files"
         uses: actions/upload-artifact@v3

ads/pipeline/ads_pipeline.py

@@ -1904,7 +1904,7 @@ def __override_configurations(
     #     ) as ml_step_schema_file:
     #         ml_step_schema = json.load(ml_step_schema_file)
 
-    #     yaml_dict = yaml.load(Pipeline._read_from_file(uri=uri))
+    #     yaml_dict = yaml.load(Pipeline._read_from_file(uri=uri), Loader=yaml.FullLoader)
 
     #     pipeline_validator = Validator(pipeline_schema)
     #     if not pipeline_validator.validate(yaml_dict):
@@ -1996,7 +1996,6 @@ def init(self) -> "Pipeline":
         )
 
 
-
 class DataSciencePipeline(OCIDataScienceMixin, oci.data_science.models.Pipeline):
     @classmethod
     def from_ocid(cls, ocid: str) -> "DataSciencePipeline":
@@ -2277,4 +2276,4 @@ def delete(
             operation_kwargs=operation_kwargs,
             waiter_kwargs=waiter_kwargs,
         )
-        return self.sync()
+        return self.sync()

ads/secrets/secrets.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import ads
@@ -273,7 +273,7 @@ def load_secret(
                 if format.lower() == "json":
                     vault_info = json.load(vf)
                 elif format.lower() in ["yaml", "yml"]:
-                    vault_info = yaml.load(vf)
+                    vault_info = yaml.load(vf, Loader=yaml.FullLoader)
                 if not cls._validate_required_vault_attributes(vault_info):
                     logger.error(
                         f"Missing required Attributes in file {uri}: {cls.required_keys}"

docs/source/ads.explanations.rst

@@ -0,0 +1,2 @@
+ads.explanations package
+========================

docs/source/ads.jobs.rst

@@ -1,12 +1,6 @@
 ads.jobs package
 ================
 
-.. toctree::
-   :maxdepth: 3
-
-   ads.jobs
-
-
 Subpackages
 -----------
 

docs/source/conf.py

@@ -173,3 +173,11 @@
 todo_include_todos = True
 
 mathjax_path = "math_jax_3_2_0.js"
+
+# This css will be included in htlm pages to process where we add .. raw:: html for nb cell nice outputs with
+# <div class="nboutput nblast docutils container"> and
+# <div class="output_area rendered_html docutils container">
+# See ads_tuner.rst dataframe output in .. raw:: html sections
+html_css_files = [
+    "nbsphinx-code-cells.css"
+]

docs/source/index.rst

@@ -97,6 +97,7 @@ Oracle Accelerated Data Science (ADS)
    `https://github.com/oracle/accelerated-data-science <https://github.com/oracle/accelerated-data-science>`_
 
 .. code-block:: python3
+
    >>> import ads
    >>> ads.hello()
 

docs/source/user_guide/big_data_service/file_management.rst

@@ -71,7 +71,7 @@ Upload
 ------
 
 The `.put() <https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.put>`_ method is used to upload files from local storage to HDFS. The first parameter is the local path of the files to upload. The second parameter is the HDFS path where the files are to be stored. 
-`.upload() <https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.upload>`_ is an alias of `.put()`.
+`.upload() <https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.upload>`_ is an alias of ``.put()``.
 .. code-block:: python3
 
     fs.put(
@@ -82,7 +82,7 @@ The `.put() <https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.sp
 Ibis
 ====
 
-`Ibis <https://github.com/ibis-project/ibis>`_ is an open-source library by `Cloudera <https://www.cloudera.com/>`_ that provides a Python framework to access data and perform analytical computations from different sources. Ibis allows access to the data ising HDFS. You use the ``ibis.impala.hdfs_connect()`` method to make a connection to HDFS, and it returns a handler. This handler has methods such as ``.ls()`` to list, ``.get()`` to download, ``.put()`` to upload, and ``.rm()`` to delete files. These operations support globbing. Ibis' HDFS connector supports a variety of `additional operations <https://ibis-project.org/docs/dev/backends/Impala/#hdfs-interaction>`_.
+`Ibis <https://github.com/ibis-project/ibis>`_ is an open-source library by `Cloudera <https://www.cloudera.com/>`_ that provides a Python framework to access data and perform analytical computations from different sources. Ibis allows access to the data ising HDFS. You use the ``ibis.impala.hdfs_connect()`` method to make a connection to HDFS, and it returns a handler. This handler has methods such as ``.ls()`` to list, ``.get()`` to download, ``.put()`` to upload, and ``.rm()`` to delete files. These operations support globbing. Ibis' HDFS connector supports a variety of `additional operations <https://ibis-project.org/backends/impala/#hdfs-interaction>`_.
 
 Connect
 -------
@@ -159,7 +159,7 @@ Use the `.put() <https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspe
 Pandas
 ======
 
-Pandas allows access to BDS' HDFS system through :ref: `FSSpec`. This section demonstrates some common operations.
+Pandas allows access to BDS' HDFS system through :ref:`FSSpec`. This section demonstrates some common operations.
 
 Connect
 -------
@@ -259,4 +259,3 @@ The following sample code shows several different PyArrow methods for working wi
     table = pa.Table.from_pandas(df)
     pq.write_to_dataset(table, root_path="/path/on/BDS/HDFS", partition_cols=["dt"], 
                         flavor="spark", filesystem=fs)
-

docs/source/user_guide/big_data_service/sql_data_management.rst

@@ -13,7 +13,7 @@ Ibis
 Connect
 -------
 
-Obtaining a Kerberos ticket, depending on your system configuration, you may need to define the ``ibis.options.impala.temp_db`` and ``ibis.options.impala.temp_hdfs_path`` options. The ``ibis.impala.connect()`` method makes a connection to the `Impala execution backend <https://ibis-project.org/docs/dev/backends/Impala/>`_. The ``.sql()`` allows you to run SQL commands on the data.
+Obtaining a Kerberos ticket, depending on your system configuration, you may need to define the ``ibis.options.impala.temp_db`` and ``ibis.options.impala.temp_hdfs_path`` options. The ``ibis.impala.connect()`` method makes a connection to the `Impala execution backend <https://ibis-project.org/backends/impala/>`_. The ``.sql()`` allows you to run SQL commands on the data.
 
 .. code-block:: python3
 
@@ -167,5 +167,3 @@ It is important to close sessions when you don't need them anymore. This frees u
 .. code-block:: python3
 
     cursor.close()
-
-