Merge branch 'hotfix/2.6.2'

Author: chrisseto

share/harvesters/edu_ageconsearch.py

@@ -53,8 +53,8 @@ def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum)
 
     # Fetch the list of work urls on a single result page and return results within date range
     def fetch_records(self, start_date, end_date):
-        logger.info('Harvesting % - %s', start_date, end_date)
-        logger.debug('Fetching page %s', self.base_url)
+        logger.info('Harvesting %s - %s', start_date, end_date)
+        logger.debug('Fetching page %s', self.config.base_url)
 
         url = furl(self.config.base_url)
         url.args['starts_with'] = start_date

share/harvesters/edu_gwu.py

@@ -17,7 +17,7 @@ def do_harvest(self, start_date, end_date):
         start_date = start_date.date()
 
         # There is no apparent way to filter by date range, just sort by date.
-        url = furl(self.config.base_url)
+        url = furl(self.config.base_url + '/catalog')
         url.args['per_page'] = 10  # If it gets more active, consider upping to 50 or 100
         url.args['sort'] = 'system_modified_dtsi+desc'
 
@@ -30,7 +30,7 @@ def fetch_records(self, url, start_date, end_date):
         resp = self.requests.get(furl(url).set(query_params={'page': page}))
         soup = BeautifulSoup(resp.content, 'lxml')
         try:
-            total = int(soup.select('#sortAndPerPage .page_entries strong')[-1].text)
+            total = int(soup.select('#sortAndPerPage .page_entries strong')[-1].text.replace(',', ''))
         except IndexError:
             total = 0
 
@@ -43,9 +43,8 @@ def fetch_records(self, url, start_date, end_date):
                 break
 
             logger.info('On document %d of %d (%d%%)', count, total, (count / total) * 100)
-
             for link in links:
-                item_response = self.requests.get(self.config.home_page + link)
+                item_response = self.requests.get(self.config.base_url + link)
                 if item_response.status_code // 100 != 2:
                     logger.warning('Got non-200 status %s from %s', item_response, link)
                     continue

share/models/logs.py

@@ -215,7 +215,7 @@ class HarvestLog(AbstractBaseLog):
     class Meta:
         unique_together = ('source_config', 'start_date', 'end_date', 'harvester_version', 'source_config_version', )
 
-    def spawn_task(self, ingest=False, force=False, limit=None, superfluous=False, ignore_disabled=False, async=True):
+    def spawn_task(self, ingest=True, force=False, limit=None, superfluous=False, ignore_disabled=False, async=True):
         from share.tasks import HarvesterTask
         # TODO Move most if not all of the logic for task argument massaging here.
         # It's bad to have two places already but this is required to backharvest a source without timing out on uwsgi

share/sources/au.uow/source.yaml

@@ -5,8 +5,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: oai_dc}
   label: au.uow
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: oai_dc
   transformer_kwargs:
     approved_sets: null

share/sources/be.ghent/source.yaml

@@ -5,8 +5,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: oai_dc}
   label: be.ghent
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: oai_dc
   transformer_kwargs:
     approved_sets: null
@@ -19,8 +19,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: mods}
   label: be.ghent.mods
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: mods
   transformer_kwargs:
     approved_sets: null

share/sources/br.pcurio/source.yaml

@@ -5,8 +5,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: oai_dc, time_granularity: false}
   label: br.pcurio
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: oai_dc
   transformer_kwargs:
     approved_sets: null

share/sources/ca.lwbin/source.yaml

@@ -5,8 +5,8 @@ configs:
   harvester: ca.lwbin
   harvester_kwargs: {}
   label: ca.lwbin
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: ca.lwbin
   transformer_kwargs: {}
 home_page: http://130.179.67.140

share/sources/ca.umontreal/source.yaml

@@ -5,8 +5,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: mods}
   label: ca.umontreal.mods
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: mods
   transformer_kwargs:
     approved_sets: null
@@ -19,8 +19,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: oai_dc}
   label: ca.umontreal
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: oai_dc
   transformer_kwargs:
     approved_sets: null

share/sources/ca.uwo/source.yaml

@@ -5,8 +5,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: oai_dc}
   label: ca.uwo
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: oai_dc
   transformer_kwargs:
     approved_sets: null

share/sources/ch.cern/source.yaml

@@ -5,8 +5,8 @@ configs:
   harvester: oai
   harvester_kwargs: {metadata_prefix: oai_dc}
   label: ch.cern
-  rate_limit_allowance: 5
-  rate_limit_period: 1
+  rate_limit_allowance: 1
+  rate_limit_period: 2
   transformer: oai_dc
   transformer_kwargs:
     approved_sets: null