diff --git a/notebooks/06_03b.ipynb b/notebooks/06_03b.ipynb index 15c680e..27cb1fc 100644 --- a/notebooks/06_03b.ipynb +++ b/notebooks/06_03b.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -21,10 +21,103 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "6b20b117", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0Sepal.LengthSepal.WidthPetal.LengthPetal.WidthSpecies
015.13.51.40.2setosa
124.93.01.40.2setosa
234.73.21.30.2setosa
344.63.11.50.2setosa
455.03.61.40.2setosa
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 Sepal.Length Sepal.Width Petal.Length Petal.Width Species\n", + "0 1 5.1 3.5 1.4 0.2 setosa\n", + "1 2 4.9 3.0 1.4 0.2 setosa\n", + "2 3 4.7 3.2 1.3 0.2 setosa\n", + "3 4 4.6 3.1 1.5 0.2 setosa\n", + "4 5 5.0 3.6 1.4 0.2 setosa" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "address = '/workspaces/python-for-data-science-and-machine-learning-essential-training-part-1-3006708/data/iris.csv'\n", "\n", @@ -32,6 +125,27 @@ "dataset.head()" ] }, + { + "cell_type": "code", + "execution_count": 3, + "id": "35196d06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['setosa', 'versicolor', 'virginica'], dtype=object)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.Species.unique()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -39,6 +153,182 @@ "### Separating features and labels" ] }, + { + "cell_type": "code", + "execution_count": 4, + "id": "05dbc1c5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Sepal.LengthSepal.WidthPetal.LengthPetal.Width
05.13.51.40.2
14.93.01.40.2
24.73.21.30.2
34.63.11.50.2
45.03.61.40.2
...............
1456.73.05.22.3
1466.32.55.01.9
1476.53.05.22.0
1486.23.45.42.3
1495.93.05.11.8
\n", + "

150 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " Sepal.Length Sepal.Width Petal.Length Petal.Width\n", + "0 5.1 3.5 1.4 0.2\n", + "1 4.9 3.0 1.4 0.2\n", + "2 4.7 3.2 1.3 0.2\n", + "3 4.6 3.1 1.5 0.2\n", + "4 5.0 3.6 1.4 0.2\n", + ".. ... ... ... ...\n", + "145 6.7 3.0 5.2 2.3\n", + "146 6.3 2.5 5.0 1.9\n", + "147 6.5 3.0 5.2 2.0\n", + "148 6.2 3.4 5.4 2.3\n", + "149 5.9 3.0 5.1 1.8\n", + "\n", + "[150 rows x 4 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = dataset.iloc[:,1:5]\n", + "X" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "6d89d25a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 setosa\n", + "1 setosa\n", + "2 setosa\n", + "3 setosa\n", + "4 setosa\n", + " ... \n", + "145 virginica\n", + "146 virginica\n", + "147 virginica\n", + "148 virginica\n", + "149 virginica\n", + "Name: Species, Length: 150, dtype: object" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y = dataset.iloc[:,5]\n", + "y" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -46,6 +336,16 @@ "### Train Test Split" ] }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1c957a2d", + "metadata": {}, + "outputs": [], + "source": [ + "X_train, X_test , y_train, y_test = train_test_split(X,y,test_size=0.3, random_state=0)\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -53,12 +353,499 @@ "### Training Decision Tree Classifier" ] }, + { + "cell_type": "code", + "execution_count": 7, + "id": "be82bdbb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
DecisionTreeClassifier()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "DecisionTreeClassifier()" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clf = DecisionTreeClassifier()\n", + "clf.fit(X_train,y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "0c0123a1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['virginica', 'versicolor', 'setosa', 'virginica', 'setosa',\n", + " 'virginica', 'setosa', 'versicolor', 'versicolor', 'versicolor',\n", + " 'virginica', 'versicolor', 'versicolor', 'versicolor',\n", + " 'versicolor', 'setosa', 'versicolor', 'versicolor', 'setosa',\n", + " 'setosa', 'virginica', 'versicolor', 'setosa', 'setosa',\n", + " 'virginica', 'setosa', 'setosa', 'versicolor', 'versicolor',\n", + " 'setosa', 'virginica', 'versicolor', 'setosa', 'virginica',\n", + " 'virginica', 'versicolor', 'setosa', 'virginica', 'versicolor',\n", + " 'versicolor', 'virginica', 'setosa', 'virginica', 'setosa',\n", + " 'setosa'], dtype=object)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y_predict = clf.predict(X_test)\n", + "y_predict" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Evaluation metric" ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "be34b7b6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy: 0.9777777777777777\n" + ] + } + ], + "source": [ + "accuracy = metrics.accuracy_score(y_test,y_predict)\n", + "print(\"Accuracy:\",accuracy)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2def4f2e", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -77,7 +864,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.1" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/07_02b.ipynb b/notebooks/07_02b.ipynb index 7c51ef0..4364801 100644 --- a/notebooks/07_02b.ipynb +++ b/notebooks/07_02b.ipynb @@ -14,7 +14,31 @@ "id": "3731af81-600b-4560-8e71-92693df00f7f", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import requests" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d58f028d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get('https://www.python.org')\n", + "response" + ] }, { "cell_type": "markdown", @@ -24,6 +48,27 @@ "### Headers" ] }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0f51de7b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Connection': 'keep-alive', 'Content-Length': '51160', 'x-frame-options': 'SAMEORIGIN', 'via': '1.1 varnish, 1.1 varnish, 1.1 varnish', 'content-type': 'text/html; charset=utf-8', 'Accept-Ranges': 'bytes', 'Date': 'Sat, 12 Oct 2024 12:24:22 GMT', 'Age': '3241', 'X-Served-By': 'cache-iad-kiad7000048-IAD, cache-iad-kiad7000114-IAD, cache-qpg1244-QPG', 'X-Cache': 'MISS, HIT, HIT', 'X-Cache-Hits': '0, 50, 28', 'X-Timer': 'S1728735862.362453,VS0,VE0', 'Vary': 'Cookie', 'Strict-Transport-Security': 'max-age=63072000; includeSubDomains; preload'}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response.headers" + ] + }, { "cell_type": "markdown", "id": "799bc8a5-5b9f-4c93-ad74-2d56a4de8bc5", @@ -32,6 +77,27 @@ "### Content Types" ] }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7f30eed3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'text/html; charset=utf-8'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response.headers['Content-Type']" + ] + }, { "cell_type": "markdown", "id": "cd63c2df-ecca-48e6-a84d-1275d5df3d55", @@ -39,6 +105,114 @@ "source": [ "### Body/Content" ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "47e5520c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'\\n\\n\\n\\n \\n\\n\\n \\n \\n \\n \\n \\n\\n \\n \\n\\n \\n \\n\\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n\\n \\n \\n\\n \\n \\n \\n \\n\\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n\\n Welcome to Python.org\\n\\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n\\n \\n\\n \\n \\n \\n \\n \\n\\n \\n\\n \\n \\n\\n \\n \\n \\n\\n\\n\\n\\n
\\n\\n
\\n

Notice: While JavaScript is not essential for this website, your interaction with the content will be limited. Please turn JavaScript on for the full experience.

\\n
\\n\\n \\n\\n \\n
\\n\\n \\n\\n
\\n\\n \\n
\\n
\\n\\n

\\n \"python™\"\\n

\\n\\n
\\n Donate\\n
\\n \\n Menu
\\n
\\n\\n \\n\\n \\n \\n\\n \\n\\n \\n \\n\\n
\\n
\\n \\n
\\n \\n
\\n \\n
\\n
\\n\\n \\n\\n
\\n \\n
\\n\\n \\n\\n
    \\n \\n
  • \\n
    # Python 3: Fibonacci series up to n\\r\\n>>> def fib(n):\\r\\n>>>     a, b = 0, 1\\r\\n>>>     while a < n:\\r\\n>>>         print(a, end=\\' \\')\\r\\n>>>         a, b = b, a+b\\r\\n>>>     print()\\r\\n>>> fib(1000)\\r\\n0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987
    \\n

    Functions Defined

    \\r\\n

    The core of extensible programming is defining functions. Python allows mandatory and optional arguments, keyword arguments, and even arbitrary argument lists. More about defining functions in Python 3

    \\n
  • \\n \\n
  • \\n
    # Python 3: List comprehensions\\r\\n>>> fruits = [\\'Banana\\', \\'Apple\\', \\'Lime\\']\\r\\n>>> loud_fruits = [fruit.upper() for fruit in fruits]\\r\\n>>> print(loud_fruits)\\r\\n[\\'BANANA\\', \\'APPLE\\', \\'LIME\\']\\r\\n\\r\\n# List and the enumerate function\\r\\n>>> list(enumerate(fruits))\\r\\n[(0, \\'Banana\\'), (1, \\'Apple\\'), (2, \\'Lime\\')]
    \\n

    Compound Data Types

    \\r\\n

    Lists (known as arrays in other languages) are one of the compound data types that Python understands. Lists can be indexed, sliced and manipulated with other built-in functions. More about lists in Python 3

    \\n
  • \\n \\n
  • \\n
    # Python 3: Simple arithmetic\\r\\n>>> 1 / 2\\r\\n0.5\\r\\n>>> 2 ** 3\\r\\n8\\r\\n>>> 17 / 3  # classic division returns a float\\r\\n5.666666666666667\\r\\n>>> 17 // 3  # floor division\\r\\n5
    \\n

    Intuitive Interpretation

    \\r\\n

    Calculations are simple with Python, and expression syntax is straightforward: the operators +, -, * and / work as expected; parentheses () can be used for grouping. More about simple math functions in Python 3.

    \\n
  • \\n \\n
  • \\n
    # For loop on a list\\r\\n>>> numbers = [2, 4, 6, 8]\\r\\n>>> product = 1\\r\\n>>> for number in numbers:\\r\\n...    product = product * number\\r\\n... \\r\\n>>> print(\\'The product is:\\', product)\\r\\nThe product is: 384
    \\n

    All the Flow You’d Expect

    \\r\\n

    Python knows the usual control flow statements that other languages speak — if, for, while and range — with some of its own twists, of course. More control flow tools in Python 3

    \\n
  • \\n \\n
  • \\n
    # Simple output (with Unicode)\\r\\n>>> print(\"Hello, I\\'m Python!\")\\r\\nHello, I\\'m Python!\\r\\n# Input, assignment\\r\\n>>> name = input(\\'What is your name?\\\\n\\')\\r\\nWhat is your name?\\r\\nPython\\r\\n>>> print(f\\'Hi, {name}.\\')\\r\\nHi, Python.\\r\\n
    \\n

    Quick & Easy to Learn

    \\r\\n

    Experienced programmers in any other language can pick up Python very quickly, and beginners find the clean syntax and indentation structure easy to learn. Whet your appetite with our Python 3 overview.

    \\n
  • \\n \\n
\\n
\\n\\n\\n
\\n\\n \\n
\\n

Python is a programming language that lets you work quickly and integrate systems more effectively. Learn More

\\n
\\n\\n\\n
\\n
\\n\\n
\\n \\n
\\n\\n
\\n\\n \\n \\n\\n \\n\\n \\n\\n
\\n \\n \\n \\n Join the official Python Developers Survey 2024 and have a chance to win a prize   Take the 2024 survey!\\n \\n
\\n\\n\\n
\\n\\n
\\n

Get Started

\\r\\n

Whether you\\'re new to programming or an experienced developer, it\\'s easy to learn and use Python.

\\r\\n

Start with our Beginner’s Guide

\\n
\\n\\n
\\n

Download

\\r\\n

Python source code and installers are available for download for all versions!

\\r\\n

Latest: Python 3.13.0

\\n
\\n\\n
\\n

Docs

\\r\\n

Documentation for Python\\'s standard library, along with tutorials and guides, are available online.

\\r\\n

docs.python.org

\\n
\\n\\n
\\n

Jobs

\\r\\n

Looking for work or have a Python related position that you\\'re trying to hire for? Our relaunched community-run job board is the place to go.

\\r\\n

jobs.python.org

\\n
\\n\\n
\\n\\n
\\n\\n
\\n \\n \\n\\n
\\n\\n
\\n \\n
\\n \\n

Upcoming Events

\\n

More

\\n \\n \\n
\\n\\n
\\n\\n
\\n\\n
\\n\\n \\n\\n
\\n
\\n

Use Python for…

\\r\\n

More

\\r\\n\\r\\n\\n
\\n
\\n\\n
\\n\\n
\\n\\n
\\n \\n

\\r\\n >>> Python Software Foundation\\r\\n

\\r\\n

The mission of the Python Software Foundation is to promote, protect, and advance the Python programming language, and to support and facilitate the growth of a diverse and international community of Python programmers. Learn more

\\r\\n

\\r\\n Become a Member\\r\\n Donate to the PSF\\r\\n

\\n
\\n\\n\\n\\n\\n
\\n\\n \\n \\n\\n \\n \\n\\n\\n
\\n
\\n\\n \\n \\n \\n \\n\\n
\\n\\n \\n \\n \\n \\n \\n\\n \\n \\n\\n \\n \\n\\n \\n\\n \\n\\n \\n\\n \\n \\n\\n\\n\\n'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response.content" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ae815859", + "metadata": {}, + "outputs": [], + "source": [ + "import difflib" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c092937d", + "metadata": {}, + "outputs": [], + "source": [ + "fline = 'Hello. How are you? I am Fine'\n", + "glines = 'How are you, Lillian? I am doing well.'" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "dcab8040", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "- H\n", + "- e\n", + "- l\n", + "- l\n", + "- o\n", + "- .\n", + "- \n", + " H\n", + " o\n", + " w\n", + " \n", + " a\n", + " r\n", + " e\n", + " \n", + " y\n", + " o\n", + " u\n", + "+ ,\n", + "+ \n", + "+ L\n", + "+ i\n", + "+ l\n", + "+ l\n", + "+ i\n", + "+ a\n", + "+ n\n", + " ?\n", + " \n", + " I\n", + " \n", + " a\n", + " m\n", + " \n", + "- F\n", + "+ d\n", + "+ o\n", + " i\n", + " n\n", + "+ g\n", + "+ \n", + "+ w\n", + " e\n", + "+ l\n", + "+ l\n", + "+ .\n" + ] + } + ], + "source": [ + "d = difflib.Differ()\n", + "diff = d.compare(fline,glines)\n", + "for line in diff:\n", + " print(line)" + ] } ], "metadata": { @@ -57,7 +231,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/07_03b.ipynb b/notebooks/07_03b.ipynb index 8f25a7a..b1f33ab 100644 --- a/notebooks/07_03b.ipynb +++ b/notebooks/07_03b.ipynb @@ -7,6 +7,33 @@ "## Part 1 - Objects in BeautifulSoup" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3.12.1 (main, Sep 30 2024, 17:05:21) [GCC 9.4.0]\n" + ] + } + ], + "source": [ + "import sys\n", + "print(sys.version)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup" + ] + }, { "cell_type": "markdown", "metadata": { @@ -18,7 +45,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -94,6 +121,122 @@ "'''" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "IoT Articles\n", + "\n", + "

2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers

\n", + "

It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use...\n", + "
\n", + "
\n", + "It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use to get practice in designing your first IoT applications.\n", + "

Looking Back at My Coolest IoT Find in 2017

\n", + "Before going into detail about best new IoT device ideas, here’s the backstory. Last month Ericsson Digital invited me to tour the Ericsson Studio in Kista, Sweden. Up until that visit, IoT had been largely theoretical to me. Of course, I know the usual mumbo-jumbo about wearables and IoT-connected fitness trackers. That stuff is all well and good, but it’s somewhat old hat – plus I am not sure we are really benefiting so much from those, so I’m not that impressed.\n", + "\n", + "It wasn’t until I got to the Ericsson Studio that I became extremely impressed by how far IoT has really come. Relying on the promise of the 5g network expansion, IoT-powered smart devices are on the cusp of an explosive growth in adoption. It was Ericsson’s Smart Car that sent me reeling:\"Get\n", + "\n", + "This car is connected to Ericsson’s Connected Vehicle Cloud, an IoT platform that manages services for the Smart Cars to which it’s connected. The Volvo pictured above acts as a drop-off location for groceries that have been ordered by its owner.\n", + "\n", + "To understand how it works, imagine you’re pulling your normal 9-to-5 and you know you need to grab some groceries on your way home. Well, since you’re smart you’ve used Ericsson IoT platform to connect your car to the local grocery delivery service (Mat.se), so all you need to do is open the Mat.se app and make your usual order. Mat.se automatically handles the payment, grocery selection, delivery, and delivery scheduling. Since your car is IoT-enabled, Mat.se issues its trusted delivery agent a 1-time token to use for opening your car in order to place your groceries in your car for you at 4:40 pm (just before you get off from work).\n", + "\n", + "To watch some of the amazing IoT device demos I witnessed at Ericsson Studio, make sure to go watch the videos on this page.\n", + "

Future Trends for IoT in 2018

\n", + "New IoT device ideas won’t do you much good unless you at least know the basic technology trends that are set to impact IoT over the next year(s). These include:\n", + "
    \n", + "
  1. Big Data & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.
  2. \n", + "
  3. Machine Learning Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.
  4. \n", + "
  5. Blockchain-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.
  6. \n", + "
\n", + "

Best New IoT Device Ideas

\n", + "This listing of new IoT device ideas has been sub-divided according to the main technology upon which the IoT devices are built. Below I’m providing a list of new IoT device ideas, but for detailed instructions on how to build these IoT applications, I recommend the IoT courses on Udemy (ß Please note: if you purchase a Udemy course through this link, I may receive a small commission), or courses that are available at SkyFi and Coursera.\n", + "

Raspberry Pi IoT Ideas

\n", + "Using Raspberry Pi as open-source hardware, you can build IoT applications that offer any one of the following benefits:\n", + "
    \n", + "
  1. Enable built-in sensing to build a weather station that measures ambient temperature and humidity
  2. \n", + "
  3. Build a system that detects discrepancies in electrical readings to identify electricity theft
  4. \n", + "
  5. Use IoT to build a Servo that is controlled by motion detection readings
  6. \n", + "
  7. Build a smart control switch that operates devices based on external stimuli. Use this for home automation.
  8. \n", + "
  9. Build a music playing application that enables music for each room in your house
  10. \n", + "
  11. Implement biometrics on IoT-connected devices
  12. \n", + "
\n", + "

Arduino IoT Ideas

\n", + "There are a number of new IoT device ideas that deploy Arduino as a microcontroller. These include:\n", + "
    \n", + "
  1. Integrate Arduino with Android to build a remote-control RGB LED device.
  2. \n", + "
  3. Connect PIR sensors across the IoT to implement a smart building.
  4. \n", + "
  5. Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.
  6. \n", + "
  7. Deploy Arduino and IoT to automate your neighborhood streetlights.
  8. \n", + "
  9. Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.
  10. \n", + "
\n", + "[caption id=\"attachment_3807\" align=\"aligncenter\" width=\"300\"]\"\" An IoT Chatbot Tree at the Ericsson Studio[/caption]\n", + "

Wireless (GSM) IoT Ideas

\n", + "Several new IoT device ideas are developed around the GSM wireless network. Those are:\n", + "
    \n", + "
  1. Monitor soil moisture to automate agricultural irrigation cycles.
  2. \n", + "
  3. Automate and control the conditions of a greenhouse.
  4. \n", + "
  5. Enable bio-metrics to build a smart security system for your home or office building
  6. \n", + "
  7. Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.
  8. \n", + "
  9. Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).
  10. \n", + "
\n", + "

IoT Automation Ideas

\n", + "Almost all new IoT device ideas offer automation benefits, but to outline a few more ideas:\n", + "
    \n", + "
  1. Build an IoT device that automatically locates and reports the closest nearby parking spot.
  2. \n", + "
  3. Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.
  4. \n", + "
  5. Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.
  6. \n", + "
  7. Use bio-metric sensors to build a smart system that automate security for your home or office building
  8. \n", + "
\n", + "To learn more about IoT and what’s happening on the leading edge, be sure to pop over to Ericsson’s Studio Tour recap and watch these videos.\n", + "\n", + "(I captured some of this content on behalf of DevMode Strategies during an invite-only tour of the Ericsson Studio in Kista. Rest assure, the text and opinions are my own)\n", + "

...

\n", + "

\n" + ] + } + ], + "source": [ + "our_html_document = BeautifulSoup(our_html_document, 'html.parser')\n", + "print(our_html_document)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " \n", + " \n", + " IoT Articles\n", + " \n", + " \n", + " \n", + "

\n", + " \n", + " 2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers\n", + " \n", + "

\n", + "

\n", + " It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article,\n" + ] + } + ], + "source": [ + "print(our_html_document.prettify()[0:300])" + ] + }, { "cell_type": "markdown", "metadata": { @@ -112,6 +255,106 @@ "#### Tag names" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "bs4.element.Tag" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup_object = BeautifulSoup('

Future Trends for IoT in 2018

','html.parser')\n", + "tag = soup_object.h1\n", + "type(tag)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "

Future Trends for IoT in 2018

\n" + ] + } + ], + "source": [ + "print(tag)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'h1'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag.name" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Future Trends for IoT in 2018" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag.name = 'heading 1'\n", + "tag" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'heading 1'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag.name" + ] + }, { "cell_type": "markdown", "metadata": { @@ -121,12 +364,368 @@ "#### Tag attributes" ] }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "

Future Trends for IoT in 2018

" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup_object = BeautifulSoup('

Future Trends for IoT in 2018

','html.parser')\n", + "tag = soup_object.h1\n", + "tag" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Heading Level 1'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag['attribute_1']" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'attribute_1': 'Heading Level 1', 'attribute_2': 'Heading Level 1*'}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag['attribute_2'] = 'Heading Level 1*'\n", + "tag.attrs" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "

Future Trends for IoT in 2018

" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "

Future Trends for IoT in 2018

" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "del tag['attribute_2']\n", + "tag" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'attribute_1': 'Heading Level 1'}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag.attrs" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Navigating a parse tree using tags" ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "our_html_document = '''\n", + "IoT Articles\n", + "\n", + "

2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers

\n", + "\n", + "

It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use...\n", + "
\n", + "
\n", + "It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use to get practice in designing your first IoT applications.\n", + "

Looking Back at My Coolest IoT Find in 2017

\n", + "Before going into detail about best new IoT device ideas, here’s the backstory. Last month Ericsson Digital invited me to tour the Ericsson Studio in Kista, Sweden. Up until that visit, IoT had been largely theoretical to me. Of course, I know the usual mumbo-jumbo about wearables and IoT-connected fitness trackers. That stuff is all well and good, but it’s somewhat old hat – plus I am not sure we are really benefiting so much from those, so I’m not that impressed.\n", + "\n", + "It wasn’t until I got to the Ericsson Studio that I became extremely impressed by how far IoT has really come. Relying on the promise of the 5g network expansion, IoT-powered smart devices are on the cusp of an explosive growth in adoption. It was Ericsson’s Smart Car that sent me reeling:\"Get\n", + "\n", + "This car is connected to Ericsson’s Connected Vehicle Cloud, an IoT platform that manages services for the Smart Cars to which it’s connected. The Volvo pictured above acts as a drop-off location for groceries that have been ordered by its owner.\n", + "\n", + "To understand how it works, imagine you’re pulling your normal 9-to-5 and you know you need to grab some groceries on your way home. Well, since you’re smart you’ve used Ericsson IoT platform to connect your car to the local grocery delivery service (Mat.se), so all you need to do is open the Mat.se app and make your usual order. Mat.se automatically handles the payment, grocery selection, delivery, and delivery scheduling. Since your car is IoT-enabled, Mat.se issues its trusted delivery agent a 1-time token to use for opening your car in order to place your groceries in your car for you at 4:40 pm (just before you get off from work).\n", + "\n", + "To watch some of the amazing IoT device demos I witnessed at Ericsson Studio, make sure to go watch the videos on this page.\n", + "

Future Trends for IoT in 2018

\n", + "New IoT device ideas won’t do you much good unless you at least know the basic technology trends that are set to impact IoT over the next year(s). These include:\n", + "
    \n", + " \t
  1. Big Data & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.
  2. \n", + " \t
  3. Machine Learning Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.
  4. \n", + " \t
  5. Blockchain-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.
  6. \n", + "
\n", + "

Best New IoT Device Ideas

\n", + "This listing of new IoT device ideas has been sub-divided according to the main technology upon which the IoT devices are built. Below I’m providing a list of new IoT device ideas, but for detailed instructions on how to build these IoT applications, I recommend the IoT courses on Udemy (ß Please note: if you purchase a Udemy course through this link, I may receive a small commission), or courses that are available at SkyFi and Coursera.\n", + "

Raspberry Pi IoT Ideas

\n", + "Using Raspberry Pi as open-source hardware, you can build IoT applications that offer any one of the following benefits:\n", + "
    \n", + " \t
  1. Enable built-in sensing to build a weather station that measures ambient temperature and humidity
  2. \n", + " \t
  3. Build a system that detects discrepancies in electrical readings to identify electricity theft
  4. \n", + " \t
  5. Use IoT to build a Servo that is controlled by motion detection readings
  6. \n", + " \t
  7. Build a smart control switch that operates devices based on external stimuli. Use this for home automation.
  8. \n", + " \t
  9. Build a music playing application that enables music for each room in your house
  10. \n", + " \t
  11. Implement biometrics on IoT-connected devices
  12. \n", + "
\n", + "

Arduino IoT Ideas

\n", + "There are a number of new IoT device ideas that deploy Arduino as a microcontroller. These include:\n", + "
    \n", + " \t
  1. Integrate Arduino with Android to build a remote-control RGB LED device.
  2. \n", + " \t
  3. Connect PIR sensors across the IoT to implement a smart building.
  4. \n", + " \t
  5. Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.
  6. \n", + " \t
  7. Deploy Arduino and IoT to automate your neighborhood streetlights.
  8. \n", + " \t
  9. Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.
  10. \n", + "
\n", + "[caption id=\"attachment_3807\" align=\"aligncenter\" width=\"300\"]\"\" An IoT Chatbot Tree at the Ericsson Studio[/caption]\n", + "

Wireless (GSM) IoT Ideas

\n", + "Several new IoT device ideas are developed around the GSM wireless network. Those are:\n", + "
    \n", + " \t
  1. Monitor soil moisture to automate agricultural irrigation cycles.
  2. \n", + " \t
  3. Automate and control the conditions of a greenhouse.
  4. \n", + " \t
  5. Enable bio-metrics to build a smart security system for your home or office building
  6. \n", + " \t
  7. Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.
  8. \n", + " \t
  9. Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).
  10. \n", + "
\n", + "

IoT Automation Ideas

\n", + "Almost all new IoT device ideas offer automation benefits, but to outline a few more ideas:\n", + "
    \n", + " \t
  1. Build an IoT device that automatically locates and reports the closest nearby parking spot.
  2. \n", + " \t
  3. Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.
  4. \n", + " \t
  5. Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.
  6. \n", + " \t
  7. Use bio-metric sensors to build a smart system that automate security for your home or office building
  8. \n", + "
\n", + "To learn more about IoT and what’s happening on the leading edge, be sure to pop over to Ericsson’s Studio Tour recap and watch these videos.\n", + "\n", + "(I captured some of this content on behalf of DevMode Strategies during an invite-only tour of the Ericsson Studio in Kista. Rest assure, the text and opinions are my own)\n", + "

...

\n", + "'''\n", + "our_html_document = BeautifulSoup(our_html_document, 'html.parser')" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "IoT Articles" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "our_html_document.head" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "IoT Articles" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "our_html_document.title" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "our_html_document.body.b" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\n", + "

2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers

\n", + "

It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use...\n", + "
\n", + "
\n", + "It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use to get practice in designing your first IoT applications.\n", + "

Looking Back at My Coolest IoT Find in 2017

\n", + "Before going into detail about best new IoT device ideas, here’s the backstory. Last month Ericsson Digital invited me to tour the Ericsson Studio in Kista, Sweden. Up until that visit, IoT had been largely theoretical to me. Of course, I know the usual mumbo-jumbo about wearables and IoT-connected fitness trackers. That stuff is all well and good, but it’s somewhat old hat – plus I am not sure we are really benefiting so much from those, so I’m not that impressed.\n", + "\n", + "It wasn’t until I got to the Ericsson Studio that I became extremely impressed by how far IoT has really come. Relying on the promise of the 5g network expansion, IoT-powered smart devices are on the cusp of an explosive growth in adoption. It was Ericsson’s Smart Car that sent me reeling:\"Get\n", + "\n", + "This car is connected to Ericsson’s Connected Vehicle Cloud, an IoT platform that manages services for the Smart Cars to which it’s connected. The Volvo pictured above acts as a drop-off location for groceries that have been ordered by its owner.\n", + "\n", + "To understand how it works, imagine you’re pulling your normal 9-to-5 and you know you need to grab some groceries on your way home. Well, since you’re smart you’ve used Ericsson IoT platform to connect your car to the local grocery delivery service (Mat.se), so all you need to do is open the Mat.se app and make your usual order. Mat.se automatically handles the payment, grocery selection, delivery, and delivery scheduling. Since your car is IoT-enabled, Mat.se issues its trusted delivery agent a 1-time token to use for opening your car in order to place your groceries in your car for you at 4:40 pm (just before you get off from work).\n", + "\n", + "To watch some of the amazing IoT device demos I witnessed at Ericsson Studio, make sure to go watch the videos on this page.\n", + "

Future Trends for IoT in 2018

\n", + "New IoT device ideas won’t do you much good unless you at least know the basic technology trends that are set to impact IoT over the next year(s). These include:\n", + "
    \n", + "
  1. Big Data & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.
  2. \n", + "
  3. Machine Learning Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.
  4. \n", + "
  5. Blockchain-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.
  6. \n", + "
\n", + "

Best New IoT Device Ideas

\n", + "This listing of new IoT device ideas has been sub-divided according to the main technology upon which the IoT devices are built. Below I’m providing a list of new IoT device ideas, but for detailed instructions on how to build these IoT applications, I recommend the IoT courses on Udemy (ß Please note: if you purchase a Udemy course through this link, I may receive a small commission), or courses that are available at SkyFi and Coursera.\n", + "

Raspberry Pi IoT Ideas

\n", + "Using Raspberry Pi as open-source hardware, you can build IoT applications that offer any one of the following benefits:\n", + "
    \n", + "
  1. Enable built-in sensing to build a weather station that measures ambient temperature and humidity
  2. \n", + "
  3. Build a system that detects discrepancies in electrical readings to identify electricity theft
  4. \n", + "
  5. Use IoT to build a Servo that is controlled by motion detection readings
  6. \n", + "
  7. Build a smart control switch that operates devices based on external stimuli. Use this for home automation.
  8. \n", + "
  9. Build a music playing application that enables music for each room in your house
  10. \n", + "
  11. Implement biometrics on IoT-connected devices
  12. \n", + "
\n", + "

Arduino IoT Ideas

\n", + "There are a number of new IoT device ideas that deploy Arduino as a microcontroller. These include:\n", + "
    \n", + "
  1. Integrate Arduino with Android to build a remote-control RGB LED device.
  2. \n", + "
  3. Connect PIR sensors across the IoT to implement a smart building.
  4. \n", + "
  5. Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.
  6. \n", + "
  7. Deploy Arduino and IoT to automate your neighborhood streetlights.
  8. \n", + "
  9. Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.
  10. \n", + "
\n", + "[caption id=\"attachment_3807\" align=\"aligncenter\" width=\"300\"]\"\" An IoT Chatbot Tree at the Ericsson Studio[/caption]\n", + "

Wireless (GSM) IoT Ideas

\n", + "Several new IoT device ideas are developed around the GSM wireless network. Those are:\n", + "
    \n", + "
  1. Monitor soil moisture to automate agricultural irrigation cycles.
  2. \n", + "
  3. Automate and control the conditions of a greenhouse.
  4. \n", + "
  5. Enable bio-metrics to build a smart security system for your home or office building
  6. \n", + "
  7. Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.
  8. \n", + "
  9. Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).
  10. \n", + "
\n", + "

IoT Automation Ideas

\n", + "Almost all new IoT device ideas offer automation benefits, but to outline a few more ideas:\n", + "
    \n", + "
  1. Build an IoT device that automatically locates and reports the closest nearby parking spot.
  2. \n", + "
  3. Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.
  4. \n", + "
  5. Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.
  6. \n", + "
  7. Use bio-metric sensors to build a smart system that automate security for your home or office building
  8. \n", + "
\n", + "To learn more about IoT and what’s happening on the leading edge, be sure to pop over to Ericsson’s Studio Tour recap and watch these videos.\n", + "\n", + "(I captured some of this content on behalf of DevMode Strategies during an invite-only tour of the Ericsson Studio in Kista. Rest assure, the text and opinions are my own)\n", + "

...

\n", + "

" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "our_html_document.body" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -145,7 +744,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/07_04b.ipynb b/notebooks/07_04b.ipynb index f049e4e..d1e37cd 100644 --- a/notebooks/07_04b.ipynb +++ b/notebooks/07_04b.ipynb @@ -7,6 +7,33 @@ "## NavigatableString Objects" ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3.12.1 (main, Sep 30 2024, 17:05:21) [GCC 9.4.0]\n" + ] + } + ], + "source": [ + "import sys\n", + "print(sys.version)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -14,6 +41,130 @@ "### NavigatableString objects" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "bs4.element.Tag" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup_object = BeautifulSoup('

Future trends for IoT in 2018

','html.parser')\n", + "tag = soup_object.h1\n", + "type(tag)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'h1'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag.name" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Future trends for IoT in 2018 '" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tag.string" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "bs4.element.NavigableString" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(tag.string)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Future trends for IoT in 2018 '" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "our_navigatable_string = tag.string\n", + "our_navigatable_string" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'NaN'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "our_navigatable_string.replace_with('NaN')\n", + "tag.string" + ] + }, { "cell_type": "markdown", "metadata": { @@ -25,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -98,7 +249,170 @@ "\n", "(I captured some of this content on behalf of DevMode Strategies during an invite-only tour of the Ericsson Studio in Kista. Rest assure, the text and opinions are my own)\n", "

...

\n", - "'''" + "'''\n", + "our_soup_object = BeautifulSoup(our_html_document, 'html.parser')" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'IoT Articles'\n", + "'2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers'\n", + "'It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use...'\n", + "'It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use to get practice in designing your first IoT applications.'\n", + "'Looking Back at My Coolest IoT Find in 2017'\n", + "'Before going into detail about best new IoT device ideas, here’s the backstory.'\n", + "'Last month Ericsson Digital invited me'\n", + "'to tour the Ericsson Studio in Kista, Sweden. Up until that visit,'\n", + "'IoT'\n", + "'had been largely theoretical to me. Of course, I know the usual mumbo-jumbo about wearables and IoT-connected fitness trackers. That stuff is all well and good, but it’s somewhat old hat – plus I am not sure we are really benefiting so much from those, so I’m not that impressed.\\n\\nIt wasn’t until I got to the Ericsson Studio that I became extremely impressed by how far IoT has really come. Relying on the promise of the 5g network expansion, IoT-powered smart devices are on the cusp of an explosive growth in adoption. It was Ericsson’s Smart Car that sent me reeling:'\n", + "'This car is connected to Ericsson’s Connected Vehicle Cloud, an IoT platform that manages services for the Smart Cars to which it’s connected. The Volvo pictured above acts as a drop-off location for groceries that have been ordered by its owner.\\n\\nTo understand how it works, imagine you’re pulling your normal 9-to-5 and you know you need to grab some groceries on your way home. Well, since you’re smart you’ve used Ericsson IoT platform to connect your car to the local grocery delivery service ('\n", + "'Mat.se'\n", + "'), so all you need to do is open the Mat.se app and make your usual order. Mat.se automatically handles the payment, grocery selection, delivery, and delivery scheduling. Since your car is IoT-enabled, Mat.se issues its trusted delivery agent a 1-time token to use for opening your car in order to place your groceries in your car for you at 4:40 pm (just before you get off from work).\\n\\nTo watch some of the amazing IoT device demos I witnessed at Ericsson Studio, make sure to go'\n", + "'watch the videos on this page'\n", + "'.'\n", + "'Future Trends for IoT in 2018'\n", + "'New IoT device ideas won’t do you much good unless you at least know the basic technology trends that are set to impact IoT over the next year(s). These include:'\n", + "'Big Data'\n", + "'& Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.'\n", + "'Machine Learning'\n", + "'Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.'\n", + "'Blockchain'\n", + "'-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.'\n", + "'Best New IoT Device Ideas'\n", + "'This listing of new IoT device ideas has been sub-divided according to the main technology upon which the IoT devices are built. Below I’m providing a list of new IoT device ideas, but for detailed instructions on how to build these IoT applications, I recommend the'\n", + "'IoT courses on Udemy'\n", + "'(ß Please note: if you purchase a Udemy course through this link, I may receive a small commission), or courses that are available at'\n", + "'SkyFi'\n", + "'and'\n", + "'Coursera'\n", + "'.'\n", + "'Raspberry Pi IoT Ideas'\n", + "'Using Raspberry Pi as open-source hardware, you can build IoT applications that offer any one of the following benefits:'\n", + "'Enable built-in sensing to build a weather station that measures ambient temperature and humidity'\n", + "'Build a system that detects discrepancies in electrical readings to identify electricity theft'\n", + "'Use IoT to build a Servo that is controlled by motion detection readings'\n", + "'Build a smart control switch that operates devices based on external stimuli. Use this for home automation.'\n", + "'Build a music playing application that enables music for each room in your house'\n", + "'Implement biometrics on IoT-connected devices'\n", + "'Arduino IoT Ideas'\n", + "'There are a number of new IoT device ideas that deploy Arduino as a microcontroller. These include:'\n", + "'Integrate Arduino with Android to build a remote-control RGB LED device.'\n", + "'Connect PIR sensors across the IoT to implement a smart building.'\n", + "'Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.'\n", + "'Deploy Arduino and IoT to automate your neighborhood streetlights.'\n", + "'Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.'\n", + "'[caption id=\"attachment_3807\" align=\"aligncenter\" width=\"300\"]'\n", + "'An IoT Chatbot Tree at the Ericsson Studio[/caption]'\n", + "'Wireless (GSM) IoT Ideas'\n", + "'Several new IoT device ideas are developed around the GSM wireless network. Those are:'\n", + "'Monitor soil moisture to automate agricultural irrigation cycles.'\n", + "'Automate and control the conditions of a greenhouse.'\n", + "'Enable bio-metrics to build a smart security system for your home or office building'\n", + "'Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.'\n", + "'Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).'\n", + "'IoT Automation Ideas'\n", + "'Almost all new IoT device ideas offer automation benefits, but to outline a few more ideas:'\n", + "'Build an IoT device that automatically locates and reports the closest nearby parking spot.'\n", + "'Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.'\n", + "'Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.'\n", + "'Use bio-metric sensors to build a smart system that automate security for your home or office building'\n", + "'To learn more about IoT and what’s happening on the leading edge, be sure to pop over to Ericsson’s Studio Tour recap and'\n", + "'watch these videos'\n", + "'.'\n", + "'(I captured some of this content on behalf of DevMode Strategies during an invite-only tour of the Ericsson Studio in Kista. Rest assure, the text and\\xa0opinions are my own'\n", + "')'\n", + "'...'\n" + ] + } + ], + "source": [ + "for string in our_soup_object.stripped_strings:\n", + " print(repr(string))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Last month Ericsson Digital invited me\n" + ] + } + ], + "source": [ + "first_link = our_soup_object.a\n", + "print(first_link)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Last month Ericsson Digital invited me" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "first_link.parent" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Last month Ericsson Digital invited me'" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "first_link.string" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Last month Ericsson Digital invited me" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "first_link.parent" ] }, { @@ -125,7 +439,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/07_05b.ipynb b/notebooks/07_05b.ipynb index 1172a04..64e8ced 100644 --- a/notebooks/07_05b.ipynb +++ b/notebooks/07_05b.ipynb @@ -15,11 +15,43 @@ "source": [ "from bs4 import BeautifulSoup\n", "\n", - "import urllib\n", + "import urllib \n", "import urllib.request\n", "import re" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "with urllib.request.urlopen('https://raw.githubusercontent.com/BigDataGal/Data-Mania-Demos/master/IoT-2018.html') as response:\n", + " html = response.read()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "bs4.BeautifulSoup" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup = BeautifulSoup(html, 'html.parser')\n", + "type(soup)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -27,6 +59,32 @@ "### Parsing your data" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " \n", + " \n", + " IoT Articles\n", + " \n", + " \n", + " \n", + "

\n", + " \n", + " \n" + ] + } + ], + "source": [ + "print(soup.prettify()[0:100])" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -34,6 +92,90 @@ "### Getting data from a parse tree" ] }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "IoT Articles\n", + "\n", + "2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers\n", + "It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use...\n", + "\n", + "\n", + "It’s almost 2018 and IoT is on the cusp of an explosive expansion. In this article, I offer you a listing of new IoT device ideas that you can use to get practice in designing your first IoT applications.\n", + "Looking Back at My Coolest IoT Find in 2017\n", + "Before going into detail about best new IoT device ideas, here’s the backstory. Last month Ericsson Digital invited me to tour the Ericsson Studio in Kista, Sweden. Up until that visit, IoT had been largely theoretical to me. Of course, I know the usual mumbo-jumbo about wearables and IoT-connected fitness trackers. That stuff is all well and good, but it’s somewhat old hat – plus I am not sure we are really benefiting so much from those, so I’m not that impressed.\n", + "\n", + "It wasn’t until I got to the Ericsson Studio that I became extremely impressed by how far IoT has really come. Relying on the promise of the 5g network expansion, IoT-powered smart devices are on the cusp of an explosive growth in adoption. It was Ericsson’s Smart Car that sent me reeling:\n", + "\n", + "This car is connected to Ericsson’s Connected Vehicle Cloud, an IoT platform that manages services for the Smart Cars to which it’s connected. The Volvo pictured above acts as a drop-off location for groceries that have been ordered by its owner.\n", + "\n", + "To understand how it works, imagine you’re pulling your normal 9-to-5 and you know you need to grab some groceries on your way home. Well, since you’re smart you’ve used Ericsson IoT platform to connect your car to the local grocery delivery service (Mat.se), so all you need to do is open the Mat.se app and make your usual order. Mat.se automatically handles the payment, grocery selection, delivery, and delivery scheduling. Since your car is IoT-enabled, Mat.se issues its trusted delivery agent a 1-time token to use for opening your car in order to place your groceries in your car for you at 4:40 pm (just before you get off from work).\n", + "\n", + "To watch some of the amazing IoT device demos I witnessed at Ericsson Studio, make sure to go watch the videos on this page.\n", + "Future Trends for IoT in 2018\n", + "New IoT device ideas won’t do you much good unless you at least know the basic technology trends that are set to impact IoT over the next year(s). These include:\n", + "\n", + "Big Data & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.\n", + "Machine Learning Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.\n", + "Blockchain-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.\n", + "\n", + "Best New IoT Device Ideas\n", + "This listing of new IoT device ideas has been sub-divided according to the main technology upon which the IoT devices are built. Below I’m providing a list of new IoT device ideas, but for detailed instructions on how to build these IoT applications, I recommend the IoT courses on Udemy (ß Please note: if you purchase a Udemy course through this link, I may receive a small commission), or courses that are available at SkyFi and Coursera.\n", + "Raspberry Pi IoT Ideas\n", + "Using Raspberry Pi as open-source hardware, you can build IoT applications that offer any one of the following benefits:\n", + "\n", + "Enable built-in sensing to build a weather station that measures ambient temperature and humidity\n", + "Build a system that detects discrepancies in electrical readings to identify electricity theft\n", + "Use IoT to build a Servo that is controlled by motion detection readings\n", + "Build a smart control switch that operates devices based on external stimuli. Use this for home automation.\n", + "Build a music playing application that enables music for each room in your house\n", + "Implement biometrics on IoT-connected devices\n", + "\n", + "Arduino IoT Ideas\n", + "There are a number of new IoT device ideas that deploy Arduino as a microcontroller. These include:\n", + "\n", + "Integrate Arduino with Android to build a remote-control RGB LED device.\n", + "Connect PIR sensors across the IoT to implement a smart building.\n", + "Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.\n", + "Deploy Arduino and IoT to automate your neighborhood streetlights.\n", + "Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.\n", + "\n", + "[caption id=\"attachment_3807\" align=\"aligncenter\" width=\"300\"] An IoT Chatbot Tree at the Ericsson Studio[/caption]\n", + "Wireless (GSM) IoT Ideas\n", + "Several new IoT device ideas are developed around the GSM wireless network. Those are:\n", + "\n", + "Monitor soil moisture to automate agricultural irrigation cycles.\n", + "Automate and control the conditions of a greenhouse.\n", + "Enable bio-metrics to build a smart security system for your home or office building\n", + "Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.\n", + "Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).\n", + "\n", + "IoT Automation Ideas\n", + "Almost all new IoT device ideas offer automation benefits, but to outline a few more ideas:\n", + "\n", + "Build an IoT device that automatically locates and reports the closest nearby parking spot.\n", + "Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.\n", + "Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.\n", + "Use bio-metric sensors to build a smart system that automate security for your home or office building\n", + "\n", + "To learn more about IoT and what’s happening on the leading edge, be sure to pop over to Ericsson’s Studio Tour recap and watch these videos.\n", + "\n", + "(I captured some of this content on behalf of DevMode Strategies during an invite-only tour of the Ericsson Studio in Kista. Rest assure, the text and opinions are my own)\n", + "...\n" + ] + } + ], + "source": [ + "text_only = soup.get_text()\n", + "print(text_only)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -59,6 +201,48 @@ "- Name argument - Search for tags by filtering based on tag name" ] }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[

  • Big Data & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.
  • ,\n", + "
  • Machine Learning Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.
  • ,\n", + "
  • Blockchain-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.
  • ,\n", + "
  • Enable built-in sensing to build a weather station that measures ambient temperature and humidity
  • ,\n", + "
  • Build a system that detects discrepancies in electrical readings to identify electricity theft
  • ,\n", + "
  • Use IoT to build a Servo that is controlled by motion detection readings
  • ,\n", + "
  • Build a smart control switch that operates devices based on external stimuli. Use this for home automation.
  • ,\n", + "
  • Build a music playing application that enables music for each room in your house
  • ,\n", + "
  • Implement biometrics on IoT-connected devices
  • ,\n", + "
  • Integrate Arduino with Android to build a remote-control RGB LED device.
  • ,\n", + "
  • Connect PIR sensors across the IoT to implement a smart building.
  • ,\n", + "
  • Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.
  • ,\n", + "
  • Deploy Arduino and IoT to automate your neighborhood streetlights.
  • ,\n", + "
  • Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.
  • ,\n", + "
  • Monitor soil moisture to automate agricultural irrigation cycles.
  • ,\n", + "
  • Automate and control the conditions of a greenhouse.
  • ,\n", + "
  • Enable bio-metrics to build a smart security system for your home or office building
  • ,\n", + "
  • Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.
  • ,\n", + "
  • Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).
  • ,\n", + "
  • Build an IoT device that automatically locates and reports the closest nearby parking spot.
  • ,\n", + "
  • Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.
  • ,\n", + "
  • Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.
  • ,\n", + "
  • Use bio-metric sensors to build a smart system that automate security for your home or office building
  • ]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup.find_all('li')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -67,6 +251,26 @@ "- Keyword argument - Search for tags by filtering based on tag attribute" ] }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[SkyFi]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup.find_all(id='link 7')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -75,6 +279,58 @@ "- String argument - Search for tags by filtering based on an exact string" ] }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[
      \n", + "
    1. Big Data & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.
    2. \n", + "
    3. Machine Learning Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.
    4. \n", + "
    5. Blockchain-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.
    6. \n", + "
    ,\n", + "
      \n", + "
    1. Enable built-in sensing to build a weather station that measures ambient temperature and humidity
    2. \n", + "
    3. Build a system that detects discrepancies in electrical readings to identify electricity theft
    4. \n", + "
    5. Use IoT to build a Servo that is controlled by motion detection readings
    6. \n", + "
    7. Build a smart control switch that operates devices based on external stimuli. Use this for home automation.
    8. \n", + "
    9. Build a music playing application that enables music for each room in your house
    10. \n", + "
    11. Implement biometrics on IoT-connected devices
    12. \n", + "
    ,\n", + "
      \n", + "
    1. Integrate Arduino with Android to build a remote-control RGB LED device.
    2. \n", + "
    3. Connect PIR sensors across the IoT to implement a smart building.
    4. \n", + "
    5. Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.
    6. \n", + "
    7. Deploy Arduino and IoT to automate your neighborhood streetlights.
    8. \n", + "
    9. Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.
    10. \n", + "
    ,\n", + "
      \n", + "
    1. Monitor soil moisture to automate agricultural irrigation cycles.
    2. \n", + "
    3. Automate and control the conditions of a greenhouse.
    4. \n", + "
    5. Enable bio-metrics to build a smart security system for your home or office building
    6. \n", + "
    7. Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.
    8. \n", + "
    9. Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).
    10. \n", + "
    ,\n", + "
      \n", + "
    1. Build an IoT device that automatically locates and reports the closest nearby parking spot.
    2. \n", + "
    3. Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.
    4. \n", + "
    5. Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.
    6. \n", + "
    7. Use bio-metric sensors to build a smart system that automate security for your home or office building
    8. \n", + "
    ]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup.find_all('ol')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -83,6 +339,59 @@ "- Lists - Search for tags by filtering based on lists" ] }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[2018 Trends: Best New IoT Device Ideas for Data Scientists and Engineers,\n", + "
      \n", + "
    1. Big Data & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.
    2. \n", + "
    3. Machine Learning Data Science: While a lot of IoT devices are still operated according to rules-based decision criteria, the age of artificial intelligence is upon us. IoT will increasingly depend on machine learning algorithms to control device operations so that devices are able to autonomously respond to a complex set of overlapping stimuli.
    4. \n", + "
    5. Blockchain-Enabled Security: Above all else, IoT networks must be secure. Blockchain technology is primed to meet the security demands that come along with building and expanding the IoT.
    6. \n", + "
    ,\n", + "
      \n", + "
    1. Enable built-in sensing to build a weather station that measures ambient temperature and humidity
    2. \n", + "
    3. Build a system that detects discrepancies in electrical readings to identify electricity theft
    4. \n", + "
    5. Use IoT to build a Servo that is controlled by motion detection readings
    6. \n", + "
    7. Build a smart control switch that operates devices based on external stimuli. Use this for home automation.
    8. \n", + "
    9. Build a music playing application that enables music for each room in your house
    10. \n", + "
    11. Implement biometrics on IoT-connected devices
    12. \n", + "
    ,\n", + "
      \n", + "
    1. Integrate Arduino with Android to build a remote-control RGB LED device.
    2. \n", + "
    3. Connect PIR sensors across the IoT to implement a smart building.
    4. \n", + "
    5. Build a temperature and sunlight sensor system to remotely monitor and control the conditions of your garden.
    6. \n", + "
    7. Deploy Arduino and IoT to automate your neighborhood streetlights.
    8. \n", + "
    9. Build a smart irrigation system based on IoT-connected temperature and moisture sensors built-in to your agricultural plants.
    10. \n", + "
    ,\n", + "
      \n", + "
    1. Monitor soil moisture to automate agricultural irrigation cycles.
    2. \n", + "
    3. Automate and control the conditions of a greenhouse.
    4. \n", + "
    5. Enable bio-metrics to build a smart security system for your home or office building
    6. \n", + "
    7. Build an autonomously operating fitness application that automatically makes recommendations based on motion detection and heart rate sensors that are embedded on wearable fitness trackers.
    8. \n", + "
    9. Build a healthcare monitoring system that tracks, informs, and automatically alerts healthcare providers based on sensor readings that describe a patients vital statistics (like temperature, pulse, blood pressure, etc).
    10. \n", + "
    ,\n", + "
      \n", + "
    1. Build an IoT device that automatically locates and reports the closest nearby parking spot.
    2. \n", + "
    3. Build a motion detection system that automatically issues emails or sms messages to alert home owners of a likely home invasion.
    4. \n", + "
    5. Use temperature sensors connected across the IoT to automatically alert you if your home windows or doors have been left open.
    6. \n", + "
    7. Use bio-metric sensors to build a smart system that automate security for your home or office building
    8. \n", + "
    ]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup.find_all(['ol','b'])" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -91,6 +400,32 @@ "- Regular expressions - Search for tags and strings by filtering based on regular expression" ] }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "html\n", + "title\n", + "strong\n", + "strong\n", + "strong\n", + "strong\n", + "strong\n", + "strong\n" + ] + } + ], + "source": [ + "t = re.compile('t')\n", + "for tag in soup.find_all(t):\n", + " print(tag.name)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -99,6 +434,90 @@ "- Boolean values- Search for tags by filtering based on a Boolean value" ] }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "html\n", + "head\n", + "title\n", + "body\n", + "p\n", + "b\n", + "p\n", + "br\n", + "br\n", + "h1\n", + "span\n", + "strong\n", + "a\n", + "a\n", + "a\n", + "img\n", + "a\n", + "span\n", + "strong\n", + "a\n", + "h1\n", + "ol\n", + "li\n", + "strong\n", + "li\n", + "strong\n", + "li\n", + "strong\n", + "h1\n", + "a\n", + "a\n", + "a\n", + "h2\n", + "ol\n", + "li\n", + "li\n", + "li\n", + "li\n", + "li\n", + "li\n", + "h2\n", + "ol\n", + "li\n", + "li\n", + "li\n", + "li\n", + "li\n", + "a\n", + "img\n", + "h2\n", + "ol\n", + "li\n", + "li\n", + "li\n", + "li\n", + "li\n", + "h2\n", + "ol\n", + "li\n", + "li\n", + "li\n", + "li\n", + "span\n", + "strong\n", + "a\n", + "em\n", + "p\n" + ] + } + ], + "source": [ + "for tag in soup.find_all(True):\n", + " print(tag.name)\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -107,6 +526,33 @@ "- Strings - Search for weblinks by filtering based on string objects" ] }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "http://bit.ly/LPlNDJj\n", + "http://www.data-mania.com/blog/m2m-vs-iot/\n", + "bit.ly/LPlNDJj\n", + "http://mat.se/\n", + "http://bit.ly/LPlNDJj\n", + "https://click.linksynergy.com/deeplink?id=*JDLXjeE*wk&mid=39197&murl=https%3A%2F%2Fwww.udemy.com%2Ftopic%2Finternet-of-things%2F%3Fsort%3Dhighest-rated\n", + "http://www.skyfilabs.com/iot-online-courses\n", + "https://www.coursera.org/specializations/iot\n", + "bit.ly/LPlNDJj\n", + "http://bit.ly/LPlNDJj\n" + ] + } + ], + "source": [ + "for link in soup.find_all('a'):\n", + " print(link.get('href'))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -114,6 +560,33 @@ "##### Retrieving strings by filtering with regular expressions\n", "- Regular expressions - Search for tags and strings by filtering based on regular expression" ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[' & Data Engineering: Sensors that are embedded within IoT devices spin off machine-generated data like it’s going out of style. For IoT to function, the platform must be solidly engineered to handle big data. Be assured, that requires some serious data engineering.']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "soup.find_all(string=re.compile('data'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -132,7 +605,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/07_06b.ipynb b/notebooks/07_06b.ipynb index 558b1bf..ded15a8 100644 --- a/notebooks/07_06b.ipynb +++ b/notebooks/07_06b.ipynb @@ -19,12 +19,345 @@ "import re" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "bs4.BeautifulSoup" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r = urllib.request.urlopen('https://analytics.usa.gov').read()\n", + "soup = BeautifulSoup(r, 'html.parser')\n", + "type(soup)" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Scraping a webpage and saving your results\n" ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " API\n" + ] + } + ], + "source": [ + "for link in soup.find_all('a',attrs={'href': re.compile('^http')}):\n", + " print(link)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "bs4.element.Tag" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(link)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "API\n" + ] + } + ], + "source": [ + "file = open('parsed_data.txt', 'w')\n", + "for link in soup.find_all('a',attrs={'href': re.compile('^http')}):\n", + " soup_link = str(link)\n", + " print(soup_link)\n", + " file.write(soup_link)\n", + "file.flush()\n", + "file.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'/workspaces/python-for-data-science-and-machine-learning-essential-training-part-1-3006708/notebooks'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -43,7 +376,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/07_07b.ipynb b/notebooks/07_07b.ipynb index f055771..6dcf5df 100644 --- a/notebooks/07_07b.ipynb +++ b/notebooks/07_07b.ipynb @@ -10,7 +10,34 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, + "id": "b7b4b66d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: aiohttp in /home/codespace/.python/current/lib/python3.12/site-packages (3.10.10)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /home/codespace/.python/current/lib/python3.12/site-packages (from aiohttp) (2.4.3)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /home/codespace/.python/current/lib/python3.12/site-packages (from aiohttp) (1.3.1)\n", + "Requirement already satisfied: attrs>=17.3.0 in /home/codespace/.local/lib/python3.12/site-packages (from aiohttp) (24.2.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /home/codespace/.python/current/lib/python3.12/site-packages (from aiohttp) (1.4.1)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /home/codespace/.python/current/lib/python3.12/site-packages (from aiohttp) (6.1.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.12.0 in /home/codespace/.python/current/lib/python3.12/site-packages (from aiohttp) (1.15.0)\n", + "Requirement already satisfied: idna>=2.0 in /home/codespace/.local/lib/python3.12/site-packages (from yarl<2.0,>=1.12.0->aiohttp) (3.10)\n", + "Requirement already satisfied: propcache>=0.2.0 in /home/codespace/.python/current/lib/python3.12/site-packages (from yarl<2.0,>=1.12.0->aiohttp) (0.2.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install aiohttp" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "id": "cb191576-cfe0-4823-aeb8-2ce0ed7a1cb3", "metadata": {}, "outputs": [], @@ -21,6 +48,104 @@ "import csv\n", "import re" ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "109d2655", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: nest-asyncio in /home/codespace/.local/lib/python3.12/site-packages (1.6.0)\n" + ] + } + ], + "source": [ + "!pip install nest-asyncio\n", + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a3976c2d", + "metadata": {}, + "outputs": [], + "source": [ + "async def scrap_and_save_links(text):\n", + " soup = BeautifulSoup(text, 'html.parser')\n", + " file = open('csv_file','a',newline='')\n", + " writer = csv.writer(file, delimiter=',')\n", + " for link in soup.findall('a',attrs={'href':re.compile('^http')}):\n", + " link = link.get('href')\n", + " writer.writerow([link])\n", + " file.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "4f45ce0b", + "metadata": {}, + "outputs": [], + "source": [ + "async def fetch(session, url):\n", + " try:\n", + " async with session.get(url) as response:\n", + " text = await response.text()\n", + " task = asyncio.create_task(scrap_and_save_links(text))\n", + " await task\n", + " except Exception as e:\n", + " print(str(e))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "15267f89", + "metadata": {}, + "outputs": [], + "source": [ + "async def scrap(urls):\n", + " tasks = []\n", + " async with aiohttp.ClientSession() as session:\n", + " for url in urls:\n", + " tasks.append(fetch(session,url))\n", + " await asyncio.gather(*tasks)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4115300f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'NoneType' object is not callable\n", + "'NoneType' object is not callable\n", + "'NoneType' object is not callable\n" + ] + } + ], + "source": [ + "urls = ['https://analytics.usa.gov/','https://www.python.org/','https://www.linkedin.com/']\n", + "asyncio.run(scrap(urls=urls))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd3c6f6d", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -39,7 +164,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/parsed_data.txt b/notebooks/parsed_data.txt index 0c6bc0c..d5115df 100644 --- a/notebooks/parsed_data.txt +++ b/notebooks/parsed_data.txt @@ -1,10 +1 @@ -APIDownload the full dataset.Download the full dataset.Digital Analytics Programdoes not track individualsanonymizes the IP addresses520 executive branch government second level domainsabout 6,800 total hostnamesAPI -Github Icon - Suggest a feature or report an issue - -Github Icon - View application code on GitHub -Github Icon - View code for the data on GitHub -GSA -Digital Analytics Programcloud.gov \ No newline at end of file +API \ No newline at end of file diff --git a/streamlit/08_02b.py b/streamlit/08_02b.py index e69de29..6a1fc35 100644 --- a/streamlit/08_02b.py +++ b/streamlit/08_02b.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.write('Hello World!') \ No newline at end of file diff --git a/streamlit/08_03b.py b/streamlit/08_03b.py index e69de29..e763676 100644 --- a/streamlit/08_03b.py +++ b/streamlit/08_03b.py @@ -0,0 +1,20 @@ +import pandas as pd +import numpy as np +import streamlit as st +import matplotlib.pyplot as plt + +col_names = ['column1','column2','column3'] +data = pd.DataFrame(np.random.randint(30,size=(30,3)), columns=col_names) +'line graph:' +st.line_chart(data) + +'bar graph:' +st.bar_chart(data) + +animals = ['cat','cow','dog'] +heights = [30,150,80] + +'pie chart:' +fig, ax = plt.subplots() +ax.pie(heights,labels=animals) +st.pyplot(fig) \ No newline at end of file diff --git a/streamlit/08_04b.py b/streamlit/08_04b.py index e69de29..0ee2c7b 100644 --- a/streamlit/08_04b.py +++ b/streamlit/08_04b.py @@ -0,0 +1,22 @@ +import time +import numpy as np +import pandas as pd +import streamlit as st +import matplotlib.pyplot as plt + +rows = np.random.randn(1,1) + +'Growing Line Chart:' +chart = st.line_chart(rows) + +for i in range(1,100): + new_rows = rows[0] + np.random.randn(1,1) + chart.add_rows(new_rows) + rows = new_rows + time.sleep(0.05) + + values = np.random.rand(10) + 'matplotlibs Line Chart:' + fig, ax = plt.subplots() + ax.plot(values) + st.pyplot(fig) \ No newline at end of file diff --git a/streamlit/08_05b.py b/streamlit/08_05b.py index 8b13789..2307445 100644 --- a/streamlit/08_05b.py +++ b/streamlit/08_05b.py @@ -1 +1,29 @@ +import pandas as pd +import numpy as np +import streamlit as st +import matplotlib.pyplot as plt +animals = ['cat','cow','dog','goat'] +heights = [30,150,80,60] +weights = [5,400,40,50] + +fig, ax = plt.subplots() + +x = np.arange(len(heights)) +width = 0.40 + +ax.bar(x-0.2, heights,width, color='red') +ax.bar(x+0.2, weights,width, color='orange') + + +ax.legend(['height', 'weight']) +ax.set_xticks(x) +ax.set_xticklabels(animals) + +st.pyplot(fig) + +explode = [0.2,0.1,0.1,0.1] +plot_pie, ax = plt.subplots() +ax.pie(heights, explode = explode, labels=animals, autopct='%1.1f%%',shadow=True) +ax.axis('equal') +st.pyplot(plot_pie) \ No newline at end of file diff --git a/streamlit/08_06b.py b/streamlit/08_06b.py index e69de29..e5015bd 100644 --- a/streamlit/08_06b.py +++ b/streamlit/08_06b.py @@ -0,0 +1,17 @@ +import streamlit as st +import seaborn as sns +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.datasets import load_iris + +iris_data = load_iris() +data = pd.DataFrame(iris_data, columns = iris_data.feature_names) +fig = plt.figure() +sns.histplot(data=data, bins=20) +st.pyplot(fig) +fig = plt.figure() +sns.boxplot(data=data) +st.pyplot(fig) +fig = plt.figure() +sns.scatterplot(data=data) +st.pyplot(fig) \ No newline at end of file