Speed up auctions fetching when skipping details

Author: Galarzaa90

CHANGELOG.rst

@@ -6,6 +6,14 @@ Changelog
     Due to this library relying on external content, older versions are not guaranteed to work.
     Try to always use the latest version.
 
+.. v3.5.4:
+
+3.5.4 (2020-09-24)
+==================
+
+- Fetching auctions while skipping details is now faster.
+- Fixed bug in tournaments parsing.
+
 .. v3.5.3:
 
 3.5.3 (2020-09-24)

tests/tests_bazaar.py

@@ -231,6 +231,31 @@ def test_auction_details_from_content_finished(self):
         self.assertEqual(509, len(auction.bestiary_progress))
         self.assertEqual(205, len(auction.completed_bestiary_entries))
 
+    def test_auction_details_from_content_finished_skip_details(self):
+        auction = AuctionDetails.from_content(self.load_resource(FILE_AUCTION_FINISHED), skip_details=True)
+
+        self.assertIsNotNone(auction)
+
+        # Listing box
+        self.assertEqual("Vireloz", auction.name)
+        self.assertIn(auction.name, auction.character_url)
+        self.assertIn(str(auction.auction_id), auction.url)
+        self.assertEqual(1161, auction.level)
+        self.assertEqual(Vocation.ROYAL_PALADIN, auction.vocation)
+        self.assertEqual(Sex.MALE, auction.sex)
+        self.assertEqual("Wintera", auction.world)
+        self.assertIsNotNone(auction.outfit)
+        self.assertEqual(1322, auction.outfit.outfit_id)
+        self.assertEqual(4, len(auction.displayed_items))
+        self.assertEqual("gnome armor", auction.displayed_items[0].name)
+        self.assertEqual("falcon coif", auction.displayed_items[1].name)
+        self.assertEqual("pair of soulstalkers", auction.displayed_items[2].name)
+        self.assertEqual("lion spangenhelm", auction.displayed_items[3].name)
+
+        self.assertEqual(330000, auction.bid)
+        self.assertEqual(BidType.MINIMUM, auction.bid_type)
+        self.assertEqual(AuctionStatus.FINISHED, auction.status)
+
     def test_auction_details_from_content_not_found(self):
         auction = AuctionDetails.from_content(self.load_resource(FILE_AUCTION_NOT_FOUND))
 

tibiapy/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '3.5.3'
+__version__ = '3.5.4'
 __author__ = 'Allan Galarza'
 
 import logging

tibiapy/bazaar.py

@@ -1,4 +1,5 @@
 import datetime
+import logging
 import re
 import urllib.parse
 import warnings
@@ -46,6 +47,8 @@
 description_regex = re.compile(r'"(?:an?\s)?([^"]+)"')
 quotes = re.compile(r'"([^"]+)"')
 
+log = logging.getLogger("tibiapy")
+
 
 class AchievementEntry(abc.Serializable):
     """An unlocked achievement by the character.
@@ -1003,7 +1006,7 @@ def from_content(cls, content, auction_id=0, skip_details=False):
         InvalidContent
             If the content does not belong to a auction detail's page.
         """
-        parsed_content = parse_tibiacom_content(content, builder='html5lib')
+        parsed_content = parse_tibiacom_content(content, builder='html5lib' if not skip_details else 'lxml')
         auction_row = parsed_content.find("div", attrs={"class": "Auction"})
         if not auction_row:
             if "internal error" in content:
@@ -1218,7 +1221,7 @@ def parse_page_items(cls, content, entry_class):
         -
             The entries contained in the page.
         """
-        parsed_content = parse_tibiacom_content(content, builder='html5lib')
+        parsed_content = parse_tibiacom_content(content, builder='lxml')
         item_boxes = parsed_content.find_all("div", attrs={"class": "CVIcon"})
         entries = []
         for item_box in item_boxes:

tibiapy/client.py

@@ -183,9 +183,9 @@ async def _request(self, method, url, data=None, headers=None):
         await self._session_ready.wait()
         try:
             init_time = time.perf_counter()
-            log.info(f"{url} | {method} | Fetching...")
+            log.info(f"%s | %s | Fetching...", url, method)
             async with self.session.request(method, url, data=data, headers=headers) as resp:
-                log.info(f"{url} | {method} | {resp.status} {resp.reason}")
+                log.info(f"%s | %s | %s %s", url, method, resp.status, resp.reason)
                 if "maintenance.tibia.com" in str(resp.url):
                     raise SiteMaintenanceError("Tibia.com is down for maintenance.")
                 self._handle_status(resp.status)

tibiapy/tournament.py

@@ -482,7 +482,7 @@ def _parse_tournament_rules(self, table):
         for row in rows[1:]:
             cols_raw = row.find_all('td')
             cols = [ele.text.strip() for ele in cols_raw]
-            field, value = cols
+            field, value, *_ = cols
             field = field.replace("\xa0", "_").replace(" ", "_").replace(":", "").lower()
             value = value.replace("\xa0", " ")
             if field in bool_fields:

tibiapy/utils.py

@@ -333,7 +333,7 @@ def parse_tibiacom_content(content, *, html_class="BoxContent", tag="div", build
         The parsed content.
     """
     strainer = bs4.SoupStrainer(tag, class_=html_class) if builder != "html5lib" else None
-    return bs4.BeautifulSoup(content.replace('ISO-8859-1', 'utf-8'), builder, parse_only=strainer)
+    return bs4.BeautifulSoup(content.replace('ISO-8859-1', 'utf-8', 1), builder, parse_only=strainer)
 
 
 T = TypeVar('T')