Merge branch 'hotfix/2.6.7' into develop

Author: chrisseto

api/serializers.py

@@ -64,7 +64,7 @@ def not_none(value):
 class RawDatumSerializer(ShareModelSerializer):
     class Meta:
         model = models.RawDatum
-        fields = ('id', 'source', 'app_label', 'provider_doc_id', 'data', 'sha256', 'date_seen', 'date_harvested')
+        fields = ('id', 'suid', 'datum', 'sha256', 'date_modified', 'date_created', 'logs')
 
 
 class ProviderRegistrationSerializer(ShareModelSerializer):

api/views/workflow.py

@@ -115,9 +115,9 @@ def get_queryset(self):
             return RawDatum.objects.filter(
                 normalizeddata__changeset__changes__target_id=object_id,
                 normalizeddata__changeset__changes__target_type__model=object_type
-            ).distinct('id').select_related('source')
+            ).distinct('id').select_related('suid')
         else:
-            return RawDatum.objects.all().select_related('source')
+            return RawDatum.objects.all().select_related('suid')
 
 
 class V1DataView(views.APIView):

share/management/commands/loadsources.py

@@ -20,6 +20,7 @@
 class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument('sources', nargs='*', type=str, help='Names of the sources to load (if omitted, load all)')
+        parser.add_argument('--overwrite', action='store_true', help='Overwrite existing sources and source configs')
 
     def handle(self, *args, **options):
         sources = options.get('sources')
@@ -32,7 +33,7 @@ def handle(self, *args, **options):
         with transaction.atomic():
             self.known_harvesters = self.sync_drivers('share.harvesters', apps.get_model('share.Harvester'))
             self.known_transformers = self.sync_drivers('share.transformers', apps.get_model('share.Transformer'))
-            self.update_sources(source_dirs)
+            self.update_sources(source_dirs, overwrite=options.get('overwrite'))
 
     def sync_drivers(self, namespace, model):
         names = set(extension.ExtensionManager(namespace).entry_points_names())
@@ -43,7 +44,7 @@ def sync_drivers(self, namespace, model):
             print('Warning: Missing {} drivers: {}'.format(model._meta.model_name, missing))
         return names
 
-    def update_sources(self, source_dirs):
+    def update_sources(self, source_dirs, overwrite):
         loaded_sources = set()
         loaded_configs = set()
         for source_dir in source_dirs:
@@ -55,36 +56,42 @@ def update_sources(self, source_dirs):
             loaded_sources.add(name)
 
             user = self.get_or_create_user(serialized.pop('user'))
-            source, _ = Source.objects.update_or_create(
-                name=name,
-                defaults={
-                    'user': user,
-                    **self.process_defaults(Source, serialized)
-                }
-            )
+            source_defaults = {
+                'user': user,
+                **self.process_defaults(Source, serialized)
+            }
+            if overwrite:
+                source, _ = Source.objects.update_or_create(name=name, defaults=source_defaults)
+            else:
+                source, _ = Source.objects.get_or_create(name=name, defaults=source_defaults)
+
             with open(os.path.join(source_dir, 'icon.ico'), 'rb') as fobj:
                 source.icon.save(name, File(fobj))
             for config in configs:
                 assert config['label'] not in loaded_configs
                 loaded_configs.add(config['label'])
-                self.update_source_config(source, config)
+                self.update_source_config(source, config, overwrite)
 
-    def update_source_config(self, source, serialized):
+    def update_source_config(self, source, serialized, overwrite):
         label = serialized.pop('label')
         if serialized['harvester'] and serialized['harvester'] not in self.known_harvesters:
             print('Unknown harvester {}! Skipping source config {}'.format(serialized['harvester'], label))
             return
         if serialized['transformer'] and serialized['transformer'] not in self.known_transformers:
             print('Unknown transformer {}! Skipping source config {}'.format(serialized['transformer'], label))
             return
-        source_config, _ = apps.get_model('share.SourceConfig').objects.update_or_create(
-            label=label,
-            defaults={
-                'source': source,
-                **self.process_defaults(apps.get_model('share.SourceConfig'), serialized)
-            }
-        )
-        self.schedule_harvest_task(source_config.label, source_config.disabled)
+
+        SourceConfig = apps.get_model('share.SourceConfig')
+        config_defaults = {
+            'source': source,
+            **self.process_defaults(SourceConfig, serialized)
+        }
+        if overwrite:
+            source_config, created = SourceConfig.objects.update_or_create(label=label, defaults=config_defaults)
+        else:
+            source_config, created = SourceConfig.objects.get_or_create(label=label, defaults=config_defaults)
+        if overwrite or created:
+            self.schedule_harvest_task(source_config.label, source_config.disabled)
 
     def get_or_create_user(self, username):
         try:

share/sources/edu.udc/source.yaml

@@ -9,7 +9,7 @@ configs:
   rate_limit_period: 2
   transformer: mods
   transformer_kwargs:
-    approved_sets: [com_11299_45272, com_11299_169792, com_11299_166578]
+    approved_sets: [com_11299_45272, col_11299_169792, col_11299_166578]
     emitted_type: CreativeWork
     property_list: []
     type_map: {}
@@ -23,7 +23,7 @@ configs:
   rate_limit_period: 2
   transformer: oai_dc
   transformer_kwargs:
-    approved_sets: [com_11299_45272, com_11299_169792, com_11299_166578]
+    approved_sets: [com_11299_45272, col_11299_169792, col_11299_166578]
     emitted_type: CreativeWork
     property_list: []
     type_map: {}

share/sources/org.biorxiv/source.yaml

@@ -24,5 +24,5 @@ configs:
           'http://purl.org/dc/elements/1.1/': dc
 home_page: http://biorxiv.org/
 long_title: bioRxiv
-name: org.biorxiv.rss
-user: providers.org.biorxiv.rss
+name: org.biorxiv
+user: providers.org.biorxiv

tests/api/test_readonly_endpoints.py

@@ -0,0 +1,67 @@
+import json
+import pytest
+
+
+def get_test_data(endpoint_type):
+    test_data = {
+        'data': {
+            'type': endpoint_type,
+            'attributes': {
+                'data': {
+                    '@graph': [{
+                        '@type': 'Person',
+                        'given_name': 'Jim',
+                    }]
+                }
+            }
+        }
+    }
+    return test_data
+
+
+@pytest.mark.django_db
+class TestRawDataEndpoint:
+    endpoint = '/api/v2/rawdata/'
+
+    def test_status(self, client):
+        assert client.get(self.endpoint).status_code == 200
+
+    def test_post(self, client, trusted_user):
+        assert client.post(
+            self.endpoint,
+            json.dumps(get_test_data('RawData')),
+            content_type='application/vnd.api+json',
+            HTTP_AUTHORIZATION='Bearer ' + trusted_user.accesstoken_set.first().token,
+        ).status_code == 405
+
+
+@pytest.mark.django_db
+class TestSourcesEndpoint:
+    endpoint = '/api/v2/sources/'
+
+    def test_status(self, client):
+        assert client.get(self.endpoint).status_code == 200
+
+    def test_post(self, client, trusted_user):
+        assert client.post(
+            self.endpoint,
+            json.dumps(get_test_data('Source')),
+            content_type='application/vnd.api+json',
+            HTTP_AUTHORIZATION='Bearer ' + trusted_user.accesstoken_set.first().token,
+        ).status_code == 405
+
+
+@pytest.mark.django_db
+class TestSiteBannersEndpoint:
+    endpoint = '/api/v2/site_banners/'
+
+    def test_status(self, client):
+        assert client.get(self.endpoint).status_code == 200
+
+    def test_post(self, client, trusted_user):
+        assert client.post(
+            self.endpoint,
+            json.dumps(get_test_data('SiteBanner')),
+            content_type='application/vnd.api+json',
+            HTTP_AUTHORIZATION='Bearer ' + trusted_user.accesstoken_set.first().token,
+        ).status_code == 405