fix get citation

Author: Valeyards

google_scholar_crawler/main.py

@@ -1,23 +1,28 @@
-from scholarly import scholarly
-import jsonpickle
-import json
-from datetime import datetime
-import os
-
-author: dict = scholarly.search_author_id(os.environ['GOOGLE_SCHOLAR_ID'])
-scholarly.fill(author, sections=['basics', 'indices', 'counts', 'publications'])
-name = author['name']
-author['updated'] = str(datetime.now())
-author['publications'] = {v['author_pub_id']:v for v in author['publications']}
-print(json.dumps(author, indent=2))
-os.makedirs('results', exist_ok=True)
-with open(f'results/gs_data.json', 'w') as outfile:
-    json.dump(author, outfile, ensure_ascii=False)
-
-shieldio_data = {
-  "schemaVersion": 1,
-  "label": "citations",
-  "message": f"{author['citedby']}",
-}
-with open(f'results/gs_data_shieldsio.json', 'w') as outfile:
-    json.dump(shieldio_data, outfile, ensure_ascii=False)
+from scholarly import scholarly, ProxyGenerator
+import jsonpickle
+import json
+from datetime import datetime
+import os
+
+# Setup proxy
+pg = ProxyGenerator()
+pg.FreeProxies()  # Use free rotating proxies
+scholarly.use_proxy(pg)
+
+author: dict = scholarly.search_author_id(os.environ['GOOGLE_SCHOLAR_ID'])
+scholarly.fill(author, sections=['basics', 'indices', 'counts', 'publications'])
+name = author['name']
+author['updated'] = str(datetime.now())
+author['publications'] = {v['author_pub_id']:v for v in author['publications']}
+print(json.dumps(author, indent=2))
+os.makedirs('results', exist_ok=True)
+with open(f'results/gs_data.json', 'w') as outfile:
+    json.dump(author, outfile, ensure_ascii=False)
+
+shieldio_data = {
+  "schemaVersion": 1,
+  "label": "citations",
+  "message": f"{author['citedby']}",
+}
+with open(f'results/gs_data_shieldsio.json', 'w') as outfile:
+    json.dump(shieldio_data, outfile, ensure_ascii=False)

google_scholar_crawler/requirements.txt

@@ -1,2 +1,3 @@
-jsonpickle==1.4.2
-scholarly==1.5.1
+jsonpickle==4.0.5
+scholarly==1.7.11
+httpx==0.23.3