fix [crawler bug](RayeRen/acad-homepage.github.io#125)

Author: anyeZHY

google_scholar_crawler/main.py

@@ -1,11 +1,31 @@
-from scholarly import scholarly
+from scholarly import scholarly, ProxyGenerator
 import jsonpickle
 import json
 from datetime import datetime
 import os
+import time
+
+max_attempts = 100
+wait_seconds = 600  # 10 minutes
+
+for attempt in range(1, max_attempts + 1):
+    try:
+        print(f"Attempt {attempt}:")
+        # Setup proxy
+        pg = ProxyGenerator()
+        pg.FreeProxies()  # Use free rotating proxies
+        scholarly.use_proxy(pg)
+        
+        author: dict = scholarly.search_author_id(os.environ['GOOGLE_SCHOLAR_ID'])
+        scholarly.fill(author, sections=['basics', 'indices', 'counts', 'publications'])
+        print(f"Attempt {attempt} success")
+        break  # Exit loop on first success
+    except Exception as e:
+        print(f"Attempt {attempt} failed with error: {e}")
+        time.sleep(wait_seconds)
+else:
+    print("All 100 attempts failed.")
 
-author: dict = scholarly.search_author_id(os.environ['GOOGLE_SCHOLAR_ID'])
-scholarly.fill(author, sections=['basics', 'indices', 'counts', 'publications'])
 name = author['name']
 author['updated'] = str(datetime.now())
 author['publications'] = {v['author_pub_id']:v for v in author['publications']}
@@ -20,4 +40,4 @@
   "message": f"{author['citedby']}",
 }
 with open(f'results/gs_data_shieldsio.json', 'w') as outfile:
-    json.dump(shieldio_data, outfile, ensure_ascii=False)
+    json.dump(shieldio_data, outfile, ensure_ascii=False)

google_scholar_crawler/requirements.txt

@@ -1,2 +1,3 @@
-jsonpickle==1.4.2
-scholarly==1.5.1
+jsonpickle==4.0.5
+scholarly==1.7.11
+httpx==0.23.3