Added web crawling example in LWP, Mechanize and Mojo

Author: rai-gaurav

README.md

@@ -129,11 +129,11 @@ Hopefully will work on older version too (as old as 5.10) and across diferent OS
 | Problem 7     | Completed |
 | Problem 8     | Completed |
 
-| Web Programming    | Status    |
-| ------------------ | --------- |
-| Catalyst           |
-| Dancer2            |
-| Mojolicious        |
-| Web Crawling <ul><li>Stock Price LWP</li></ul><ul><li>Stock Price Mechanize</li></ul>        |
-| Currency Converter | Completed |
-| Weather            | Completed |
+| Web Programming                                                                                                         | Status    |
+| ----------------------------------------------------------------------------------------------------------------------- | --------- |
+| Catalyst                                                                                                                |
+| Dancer2                                                                                                                 |
+| Mojolicious                                                                                                             |
+| Web Crawling <ul><li>Stock Price LWP</li></ul><ul><li>Stock Price Mechanize</li></ul><ul><li>Stock Price Mojo</li></ul> | Completed |
+| Currency Converter <ul><li>LWP::UserAgent</li></ul><ul><li>Mojo::UserAgent</li></ul>                                    | Completed |
+| Weather <ul><li>LWP::UserAgent</li></ul><ul><li>Mojo::UserAgent</li></ul>                                               | Completed |

web_programming/web_crawling.pl

@@ -0,0 +1,42 @@
+#!/usr/bin/env perl
+# https://metacpan.org/pod/WWW::Mechanize
+# https://metacpan.org/pod/XML::LibXML
+
+# We can use 'HTML::TreeBuilder' or 'XML::Twig' also here for parsing HTML.
+# We are using 'XML::LibXML' just as an example as it can also parse HTML
+
+use strict;
+use warnings;
+use Carp qw( croak );
+use LWP::UserAgent ();
+use XML::LibXML;
+
+sub get_stock_price {
+    my ($ua, $symbol) = @_;
+    my $url      = "https://in.finance.yahoo.com/quote/" . $symbol;
+    my $response = $ua->get($url);
+
+    if ($response->is_success) {
+        my $dom = XML::LibXML->load_html(
+            string          => $response->decoded_content,
+            recover         => 1,
+            suppress_errors => 1
+        );
+
+        my $xpath = '//span[contains(@class, "Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)")]';
+        return $dom->findvalue($xpath);
+    }
+    else {
+        croak $response->status_line;
+    }
+}
+
+sub main {
+    my $ua = LWP::UserAgent->new();
+    $ua->agent('Mozilla/5.0');
+    my $symbol      = "AMZN";
+    my $stock_price = get_stock_price($ua, $symbol);
+    print "\n Current stock price for $symbol is $stock_price";
+}
+
+main();

web_programming/web_crawling/stock_price_lwp.pl

@@ -0,0 +1,44 @@
+#!/usr/bin/env perl
+# https://metacpan.org/pod/WWW::Mechanize
+# https://metacpan.org/pod/HTML::TreeBuilder
+
+use strict;
+use warnings;
+use Carp qw( croak );
+use WWW::Mechanize;
+use HTML::TreeBuilder;
+
+sub get_stock_price {
+    my ($mech, $symbol) = @_;
+    my $url      = "https://in.finance.yahoo.com/quote/" . $symbol;
+    my $response = $mech->get($url);
+
+    if ($response->is_success) {
+        if ($mech->is_html()) {
+            my $tree = HTML::TreeBuilder->new();
+            $tree->parse($mech->content);
+            my $current_stock_price = $tree->look_down(
+                '_tag'  => 'span',
+                'class' => 'Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)'
+            )->as_text;
+            return $current_stock_price;
+        }
+        else {
+            print "\n Not a HTML response";
+            return 0;
+        }
+    }
+    else {
+        croak $response->status_line;
+    }
+}
+
+sub main {
+    my $mech = WWW::Mechanize->new();
+    $mech->agent_alias("Windows Mozilla");
+    my $symbol      = "AMZN";
+    my $stock_price = get_stock_price($mech, $symbol);
+    print "\n Current stock price for $symbol is $stock_price";
+}
+
+main();

web_programming/web_crawling/stock_price_mechanize.pl

@@ -0,0 +1,38 @@
+#!/usr/bin/env perl
+# https://metacpan.org/pod/Mojo::Useragent
+# https://metacpan.org/pod/Mojo::DOM
+
+use strict;
+use warnings;
+use Carp qw( croak );
+use Mojo::UserAgent;
+
+sub get_stock_price {
+    my ($ua, $symbol) = @_;
+    my $url      = "https://in.finance.yahoo.com/quote/" . $symbol;
+    my $response = $ua->get($url)->result;
+
+    if ($response->is_success) {
+
+        # https://docs.mojolicious.org/Mojo/DOM#find
+        my $current_stock_price
+            = $response->dom->find('span.Trsdu(0\.3s).Fw(b).Fz(36px).Mb(-4px).D(ib)')->map('text')
+            ->join("\n");
+
+        return $current_stock_price;
+    }
+    else {
+        croak $response->message;
+    }
+}
+
+sub main {
+    my $ua = Mojo::UserAgent->new;
+    $ua->transactor->name('Mozilla/5.0');
+    $ua = $ua->connect_timeout(25);
+    my $symbol      = "AMZN";
+    my $stock_price = get_stock_price($ua, $symbol);
+    print "\n Current stock price for $symbol is $stock_price";
+}
+
+main();