Merge branch 'feature/forecast_v2' into feature/anomaly_detect

Author: ahosler

MANIFEST.in

@@ -1,30 +1,9 @@
-include LICENSE.txt
-include THIRD_PARTY_LICENSES.txt
-include ads/common/*.json
-include ads/feature_engineering/*.json
-include ads/templates/*.jinja2
-include ads/common/function/*.yaml
-include ads/common/artifact/*.*
-include ads/model/runtime/schemas/*.yaml
-include ads/jobs/schema/**
-include ads/opctl/index.yaml
-include ads/opctl/operators/**/*.py
-include ads/opctl/docker/**
-include ads/opctl/conda/manifest_template.yaml
-include ads/opctl/conda/config.yaml
-include ads/opctl/config/diagnostics/**/*.yaml
-include ads/opctl/templates/*.jinja2
-include pyproject.toml
-global-exclude tests/**
-global-exclude notebooks/**
-exclude tests/**
-exclude notebooks/**
-exclude build/lib/tests/**
-exclude build/lib/notebooks/**
-exclude benchmark/**
-include ads/ads
-include ads/model/common/*.*
-include ads/operator/**/*.md
-include ads/operator/**/*.yaml
-include ads/operator/**/*.whl
-include ads/operator/**/MLoperator
+# Manifest.in file used by setuptools, which we are not using anymore.
+# Do not add anything to this file, unless revert back to setuptools, which is not recommended. See why here:
+# https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html.
+
+# Instead flit-core used as backend to build ADS - see section [build-system] in pyproject.toml.
+# For flit-core we can specify files and folders, which needs to be included or excluded in build using
+# [tool.flit.sdist] section of pyproject.toml. See: https://flit.pypa.io/en/latest/pyproject_toml.html#sdist-section
+# If build-system library will change, make changed to list of included/excluded to build files accordingly
+# as it will be suggested by new build-system library.

Makefile

@@ -1,36 +1,3 @@
-RELEASE_BRANCH := release/ads
-DOCS_RELEASE_BRANCH := release
-CLONE_DIR := /tmp/advanced-ds
-DOCS_CLONE_DIR := /tmp/ads-docs
-COPY_INVENTORY := CONTRIBUTING.md LICENSE.txt MANIFEST.in README-development.md README.md SECURITY.md THIRD_PARTY_LICENSES.txt
-
-prepare-release-branch: clean
-	@git checkout master
-	@git clean -xdf
-	@git pull
-	git checkout -b release/$(RELEASE_VERSION)
-
-prepare-ads:
-	@echo "Started advanced-ds clone at $$(date)"
-	@git clone ssh://git@bitbucket.oci.oraclecorp.com:7999/odsc/advanced-ds.git --branch $(RELEASE_BRANCH)  --depth 1 $(CLONE_DIR)
-	@echo "Finished cloning at $$(date)"
-	cp -r $(CLONE_DIR)/ads .
-	$(foreach var,$(COPY_INVENTORY),cp $(CLONE_DIR)/$(var) .;)
-
-prepare-docs:
-	@echo "Started ads_docs clone at $$(date)"
-	@git clone ssh://git@bitbucket.oci.oraclecorp.com:7999/odsc/ads_docs.git --branch $(DOCS_RELEASE_BRANCH)  --depth 1 $(DOCS_CLONE_DIR)
-	@echo "Finished cloning at $$(date)"
-	cp -r $(DOCS_CLONE_DIR)/source docs/ && cp $(DOCS_CLONE_DIR)/requirements.txt docs
-
-prepare: prepare-release-branch prepare-ads prepare-docs
-
-push: clean
-	@bash -c 'if [[ $$(git branch | grep \*) == "* release/$(RELEASE_VERSION)" ]];then echo "Version matching current branch"; else echo "Set proper value to RELEASE_VERSION";exit 1 ; fi'
-	@git add .
-	@git commit -m "Release version: $(RELEASE_VERSION)"
-	@git push --set-upstream origin release/$(RELEASE_VERSION)
-
 dist: clean
 	@python3 -m build
 
@@ -43,5 +10,4 @@ clean:
 	@find ./ -name '*.pyc' -exec rm -f {} \;
 	@find ./ -name 'Thumbs.db' -exec rm -f {} \;
 	@find ./ -name '*~' -exec rm -f {} \;
-	@rm -rf $(CLONE_DIR)
-	@rm -rf $(DOCS_CLONE_DIR)
+	@find ./ -name '.DS_Store' -exec rm -f {} \;

THIRD_PARTY_LICENSES.txt

@@ -54,6 +54,12 @@ docker
 * Source code: https://github.com/docker
 * Project home: https://www.docker.com/
 
+evaluate
+* Copyright 2023 HuggingFace Inc.
+* License: Apache-2.0 license
+* Source code: https://github.com/huggingface/evaluate
+* Project home: https://huggingface.co/docs/evaluate/index
+
 fastavro
 * Copyright (c) 2011 Miki Tebeka
 * License: MIT License
@@ -133,6 +139,12 @@ jinja2
 * Source code: https://github.com/pallets/jinja/
 * Project home: https://palletsprojects.com/p/jinja/
 
+langchain
+* Copyright (c) 2023 LangChain, Inc.
+* License: MIT license
+* Source code: https://github.com/langchain-ai/langchain
+* Project home: https://www.langchain.com/
+
 lightgbm
 * Copyright (c) 2023 Microsoft Corporation
 * License: MIT license

ads/common/oci_mixin.py

@@ -33,7 +33,7 @@
 WORK_REQUEST_STOP_STATE = ("SUCCEEDED", "FAILED", "CANCELED")
 DEFAULT_WAIT_TIME = 1200
 DEFAULT_POLL_INTERVAL = 10
-DEFAULT_WORKFLOW_STEPS = 2
+WORK_REQUEST_PERCENTAGE = 100
 
 
 class MergeStrategy(Enum):
@@ -939,7 +939,6 @@ def get_work_request_response(
     def wait_for_progress(
         self,
         work_request_id: str,
-        num_steps: int = DEFAULT_WORKFLOW_STEPS,
         max_wait_time: int = DEFAULT_WAIT_TIME,
         poll_interval: int = DEFAULT_POLL_INTERVAL,
     ):
@@ -949,8 +948,6 @@ def wait_for_progress(
         ----------
         work_request_id: str
             Work Request OCID.
-        num_steps: (int, optional). Defaults to 2.
-            Number of steps for the progress indicator.
         max_wait_time: int
             Maximum amount of time to wait in seconds (Defaults to 1200).
             Negative implies infinite wait time.
@@ -965,13 +962,14 @@ def wait_for_progress(
 
         i = 0
         start_time = time.time()
-        with get_progress_bar(num_steps) as progress:
+        with get_progress_bar(WORK_REQUEST_PERCENTAGE) as progress:
             seconds_since = time.time() - start_time
             exceed_max_time = max_wait_time > 0 and seconds_since >= max_wait_time
             if exceed_max_time:
                 logger.error(f"Max wait time ({max_wait_time} seconds) exceeded.")
+            previous_percent_complete = 0
             while not exceed_max_time and (
-                not work_request_logs or len(work_request_logs) < num_steps
+                not work_request_logs or previous_percent_complete <= WORK_REQUEST_PERCENTAGE
             ):
                 time.sleep(poll_interval)
                 new_work_request_logs = []
@@ -988,9 +986,23 @@ def wait_for_progress(
                     work_request_logs[i:] if work_request_logs else []
                 )
 
-                for wr_item in new_work_request_logs:
-                    progress.update(wr_item.message)
-                    i += 1
+                percent_change = work_request.percent_complete - previous_percent_complete
+                previous_percent_complete = work_request.percent_complete
+
+                if len(new_work_request_logs) > 0:
+                    start_index = True
+                    for wr_item in new_work_request_logs:
+                        if start_index:
+                            progress.update(wr_item.message, percent_change)
+                            start_index = False
+                        else:
+                            progress.update(wr_item.message, 0)
+                        i += 1
+                else:
+                    # if there is new percent change but the new work request logs is empty
+                    # needs to add this percent change to the bar to ensure the final percentage is 100
+                    if percent_change != 0:
+                        progress.update(n=percent_change)
 
                 if work_request and work_request.status in WORK_REQUEST_STOP_STATE:
                     if work_request.status != "SUCCEEDED":

ads/llm/__init__.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+try:
+    import langchain
+    from ads.llm.langchain.plugins.llm_gen_ai import GenerativeAI
+    from ads.llm.langchain.plugins.llm_md import ModelDeploymentTGI
+    from ads.llm.langchain.plugins.llm_md import ModelDeploymentVLLM
+    from ads.llm.langchain.plugins.embeddings import GenerativeAIEmbeddings
+except ImportError as ex:
+    if ex.name == "langchain":
+        raise ImportError(
+            f"{ex.msg}\nPlease install/update langchain with `pip install langchain -U`"
+        ) from ex
+    raise ex