Update main branch with develop branch changes (#98)

* Ft locations endpoint (#75)

* Adds Location Serializer

* Allows users to list and create locations

* Adds a basename for sensors_location_router

* Adds Pagination

* Adds Node creation API endpoint (#76)

* Adds Node creation API endpoint

* Remove un used serializer import

* Add Sensors + Sensor_Type Endpoint (#77)

* sensors endpoint

* fix serializers

* authentication on post

* sensors type endpoint

* Update sensorsafrica/api/v2/views.py

Co-authored-by: _ Kilemensi <kilemensi@users.noreply.github.com>

* Update sensorsafrica/api/v2/views.py

Co-authored-by: _ Kilemensi <kilemensi@users.noreply.github.com>

* spelling

Co-authored-by: _ Kilemensi <kilemensi@users.noreply.github.com>

* [Hotfix] Creation api (#79)

* Standardize naming around views, models and routes
* Add tests

* [Feature] Public / Private Data (#81)

* Add SensorDataView using CursorPagination to show all available data

* Make /v2/data return all data, keept /data/<sensortype> for air/water,
etc.

* Switch computation to only work with public data (sensorsAFRICA data for
now)

* Upgrade backward compatible deps

* Add default AUTHENTICATION_CLASSES (#82)

Feinstaub framework doesn't set per view authentication classes and
hence we need default ones

* [Fix] serializer (#83)

* Introduce SensorTypeSerializer that will handle uid

* Use SensorTypeSerializer

* Ensure city and data exists before trying to upload (#85)

* Upload data from public sensors only (#87)

* [Feature] Make v1/now return public data only (#86)

* Recreate feinstaub NowView but with public sensors filter

* Switch to custom NowView

* Add owner as part of node (#89)

* [Fix] v1/node should check groups (#88)

* Add NodeView that checks group ownership + pagination

* Update router to use new NodeView

* Use user id instead of user object (#90)

* Adds endpoint for providing metadata (#91)

* Adds endpoint for providing metadata

* Use dynamic database name

* Returns sensors locations and last time database was updated.

* Adds sensor networks to returned metadata

* Adds authentication to the meta endpoint

* Remove a default NETWORKS_OWNER

* Use latest created SensorDataValue to know when the database was last updated.

* Ensure we only return SensorLocations with a country

* Ft increase gurnicorn timeout (#92)

* Increase Gunicorn timeout to 3 minutes

* Return sorted sensors locations

* Includes country in the sensor data serializer (#93)

* Includes country in the sensor data serializer

* Only return location id and country

* Ft add location filter (#95)

* Adds a nodes filter using location

* Adds nodes filter for v1 of the API

* Remove filterling on v2 since it's only used for the F.E map

* Use default django filter backend

* Makes country location matching case insensitive.

* [Ft] Nodes last notify (#94)

* update node last_notify

* update nodeserializer

* import from feinstaub

* class naming

* code refactor

* remove repeating line

* comment line

* Update sensorsafrica/api/v1/serializers.py

Co-authored-by: _ Kilemensi <kilemensi@users.noreply.github.com>

* rename verbose to lastnotify

* update only when current notify is less than sensordata timestamp

Co-authored-by: _ Kilemensi <kilemensi@users.noreply.github.com>

* Ft add location filter for /v2/data (#97)

* Adds location filter for /data endpoint

* Temporaliry Remove iexact

* [FT] Time filter  (#96)

* add last_notify filter on nodes, sensorfilter

* view class override

* filter ovverides

* add exact to look up exp

* Includes missing import

Co-authored-by: esir <esirkings@gmail.com>

Co-authored-by: Khadija Mahanga <khadijamahanga@y7mail.com>
Co-authored-by: _ Kilemensi <kilemensi@users.noreply.github.com>

Author: 3 people

Makefile

@@ -25,9 +25,14 @@ migrate:
 test:
 	$(COMPOSE) exec api pytest --pylama
 
+testexpr:
+	$(COMPOSE) exec api pytest --pylama -k '$(expr)'
+
 createsuperuser:
 	$(COMPOSE) exec api python manage.py createsuperuser
 
+down:
+	$(COMPOSE) down
 
 clean:
 	@find . -name "*.pyc" -exec rm -rf {} \;

contrib/start.sh

@@ -16,7 +16,8 @@ celery -A sensorsafrica flower --basic_auth=$SENSORSAFRICA_FLOWER_ADMIN_USERNAME
 echo Starting Gunicorn.
 exec gunicorn \
     --bind 0.0.0.0:8000 \
-    --workers 3 \
+    --timeout 180 \
+    --workers 5 \
     --worker-class gevent \
     --log-level=info \
     --log-file=/src/logs/gunicorn.log \

requirements.txt

@@ -1,11 +1,11 @@
-django==1.11.27 #LTS
+django==1.11.29 #LTS
 coreapi==2.3.3
 dj-database-url==0.5.0
 timeago==1.0.10
 
-flower==0.9.2
+flower==0.9.5
 tornado<6
-sentry-sdk==0.7.3
+sentry-sdk==0.19.5
 celery==4.2.1
 gevent==1.2.2
 greenlet==0.4.12

sensorsafrica/api/v1/filters.py

@@ -0,0 +1,30 @@
+import django_filters
+from django.db import models
+
+from feinstaub.sensors.models import Node, SensorData
+
+class NodeFilter(django_filters.FilterSet):
+    class Meta:
+        model = Node
+        fields = {
+            "location__country": ["exact"],
+            "last_notify": ["exact", "gte", "lte"]}
+        filter_overrides = {
+            models.DateTimeField: {
+                'filter_class': django_filters.IsoDateTimeFilter,
+            },
+        }
+
+
+class SensorFilter(django_filters.FilterSet):
+    class Meta:
+        model = SensorData
+        fields = {
+            "sensor": ["exact"],
+            "timestamp": ["exact", "gte", "lte"]
+            }
+        filter_overrides = {
+            models.DateTimeField: {
+                'filter_class': django_filters.IsoDateTimeFilter,
+            },
+        }

sensorsafrica/api/v1/router.py

@@ -1,28 +1,33 @@
 # The base version is entirely based on feinstaub
 from feinstaub.main.views import UsersView
 from feinstaub.sensors.views import (
-    NodeView,
-    NowView,
-    PostSensorDataView,
     SensorView,
     StatisticsView,
     SensorDataView,
 )
 
-from .views import SensorDataView as SensorsAfricaSensorDataView, FilterView
+from .views import (
+    FilterView,
+    NodeView,
+    NowView,
+    PostSensorDataView,
+    SensorsAfricaSensorDataView,
+    VerboseSensorDataView,
+)
 
 from rest_framework import routers
 
 router = routers.DefaultRouter()
 router.register(r"push-sensor-data", PostSensorDataView)
 router.register(r"node", NodeView)
 router.register(r"sensor", SensorView)
-router.register(r"data", SensorDataView)
+router.register(r"data", VerboseSensorDataView)
 router.register(r"statistics", StatisticsView, basename="statistics")
 router.register(r"now", NowView)
 router.register(r"user", UsersView)
-router.register(r"sensors/(?P<sensor_id>\d+)",
-                SensorsAfricaSensorDataView, basename="sensors")
+router.register(
+    r"sensors/(?P<sensor_id>\d+)", SensorsAfricaSensorDataView, basename="sensors"
+)
 router.register(r"filter", FilterView, basename="filter")
 
 api_urls = router.urls

sensorsafrica/api/v1/serializers.py

@@ -1,10 +1,23 @@
 from rest_framework import serializers
 from feinstaub.sensors.models import (
+    Node,
     SensorData,
     SensorDataValue,
     SensorLocation
 )
+from feinstaub.sensors.serializers import (
+    NestedSensorLocationSerializer,
+    NestedSensorSerializer,
+    SensorDataSerializer as PostSensorDataSerializer
+)
+
+class NodeSerializer(serializers.ModelSerializer):
+    sensors = NestedSensorSerializer(many=True)
+    location = NestedSensorLocationSerializer()
 
+    class Meta:
+        model = Node
+        fields = ('id', 'sensors', 'uid', 'owner', 'location', 'last_notify')
 
 class SensorLocationSerializer(serializers.ModelSerializer):
     class Meta:
@@ -25,3 +38,19 @@ class SensorDataSerializer(serializers.ModelSerializer):
     class Meta:
         model = SensorData
         fields = ['location', 'timestamp', 'sensordatavalues']
+
+class LastNotifySensorDataSerializer(PostSensorDataSerializer):
+
+    def create(self, validated_data):
+        sd = super().create(validated_data)
+         # use node from authenticator
+        successful_authenticator = self.context['request'].successful_authenticator
+        node, pin = successful_authenticator.authenticate(self.context['request'])
+
+        #sometimes we post historical data (eg: from other network)
+        #this means we have to update last_notify only if current timestamp is greater than what's there
+        if node.last_notify is None or node.last_notify < sd.timestamp: 
+            node.last_notify = sd.timestamp
+            node.save()
+
+        return sd

sensorsafrica/api/v1/views.py

@@ -1,50 +1,112 @@
 import datetime
 import pytz
 import json
+import django_filters
 
-from rest_framework.exceptions import ValidationError
 
 from django.conf import settings
-from django.utils import timezone
-from dateutil.relativedelta import relativedelta
 from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q
 from django.db.models.functions import Cast, TruncDate
+from dateutil.relativedelta import relativedelta
+from django.utils import timezone
 from rest_framework import mixins, pagination, viewsets
+from rest_framework.authentication import SessionAuthentication, TokenAuthentication
+from rest_framework.exceptions import ValidationError
+from rest_framework.permissions import IsAuthenticatedOrReadOnly
 
-from .serializers import SensorDataSerializer
-from feinstaub.sensors.models import SensorData
+from feinstaub.sensors.models import Node, SensorData
+from feinstaub.sensors.serializers import NowSerializer
+from feinstaub.sensors.views import SensorDataView, StandardResultsSetPagination
+from feinstaub.sensors.authentication import NodeUidAuthentication
 
+from .filters import NodeFilter, SensorFilter
+from .serializers import LastNotifySensorDataSerializer, NodeSerializer, SensorDataSerializer
 
-class SensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet):
+class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet):
     serializer_class = SensorDataSerializer
 
     def get_queryset(self):
+        sensor_type = self.request.GET.get("type", r"\w+")
+        country = self.request.GET.get("country", r"\w+")
+        city = self.request.GET.get("city", r"\w+")
         return (
-            SensorData.objects
-            .filter(
+            SensorData.objects.filter(
                 timestamp__gte=timezone.now() - datetime.timedelta(minutes=5),
-                sensor=self.kwargs["sensor_id"]
+                sensor__sensor_type__uid__iregex=sensor_type,
+                location__country__iregex=country,
+                location__city__iregex=city,
             )
-            .only('sensor', 'timestamp')
-            .prefetch_related('sensordatavalues')
+            .only("sensor", "timestamp")
+            .prefetch_related("sensordatavalues")
         )
 
 
-class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet):
+class NodeView(
+    mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    """Show all nodes belonging to authenticated user"""
+
+    authentication_classes = [SessionAuthentication, TokenAuthentication]
+    pagination_class = StandardResultsSetPagination
+    permission_classes = [IsAuthenticatedOrReadOnly]
+    queryset = SensorData.objects.none()
+    serializer_class = NodeSerializer
+    filter_class = NodeFilter
+
+    def get_queryset(self):
+        if self.request.user.is_authenticated():
+            if self.request.user.groups.filter(name="show_me_everything").exists():
+                return Node.objects.all()
+
+            return Node.objects.filter(
+                Q(owner=self.request.user)
+                | Q(
+                    owner__groups__name__in=[
+                        g.name for g in self.request.user.groups.all()
+                    ]
+                )
+            )
+
+        return Node.objects.none()
+
+
+class NowView(mixins.ListModelMixin, viewsets.GenericViewSet):
+    """Show all public sensors active in the last 5 minutes with newest value"""
+
+    permission_classes = []
+    serializer_class = NowSerializer
+    queryset = SensorData.objects.none()
+
+    def get_queryset(self):
+        now = timezone.now()
+        startdate = now - datetime.timedelta(minutes=5)
+        return SensorData.objects.filter(
+            sensor__public=True, modified__range=[startdate, now]
+        )
+
+class PostSensorDataView(mixins.CreateModelMixin,
+                         viewsets.GenericViewSet):
+    """ This endpoint is to POST data from the sensor to the api.
+    """
+    authentication_classes = (NodeUidAuthentication,)
+    permission_classes = tuple()
+    serializer_class = LastNotifySensorDataSerializer
+    queryset = SensorData.objects.all()
+    
+
+class VerboseSensorDataView(SensorDataView):
+    filter_class = SensorFilter
+
+class SensorsAfricaSensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet):
     serializer_class = SensorDataSerializer
 
     def get_queryset(self):
-        sensor_type = self.request.GET.get('type', r'\w+')
-        country = self.request.GET.get('country', r'\w+')
-        city = self.request.GET.get('city', r'\w+')
         return (
-            SensorData.objects
-            .filter(
+            SensorData.objects.filter(
                 timestamp__gte=timezone.now() - datetime.timedelta(minutes=5),
-                sensor__sensor_type__uid__iregex=sensor_type,
-                location__country__iregex=country,
-                location__city__iregex=city
+                sensor=self.kwargs["sensor_id"],
             )
-            .only('sensor', 'timestamp')
-            .prefetch_related('sensordatavalues')
+            .only("sensor", "timestamp")
+            .prefetch_related("sensordatavalues")
         )
+

sensorsafrica/api/v2/filters.py

@@ -0,0 +1,6 @@
+from feinstaub.sensors.views import SensorFilter
+
+class CustomSensorFilter(SensorFilter):
+    class Meta(SensorFilter.Meta):
+        # Pick the fields already defined and add the location__country field
+        fields = {**SensorFilter.Meta.fields, **{'location__country': ['exact']}}

sensorsafrica/api/v2/router.py

@@ -1,22 +1,45 @@
 from rest_framework import routers
 from django.conf.urls import url, include
 
-from .views import SensorDataStatView, CityView, NodesView
+from .views import (
+    CitiesView,
+    NodesView,
+    SensorDataStatsView,
+    SensorDataView,
+    SensorLocationsView,
+    SensorTypesView,
+    SensorsView,
+    meta_data,
+)
+
+stat_data_router = routers.DefaultRouter()
+stat_data_router.register(r"", SensorDataStatsView)
 
 data_router = routers.DefaultRouter()
+data_router.register(r"", SensorDataView)
 
-data_router.register(r"", SensorDataStatView)
+cities_router = routers.DefaultRouter()
+cities_router.register(r"", CitiesView, basename="cities")
 
-city_router = routers.DefaultRouter()
+nodes_router = routers.DefaultRouter()
+nodes_router.register(r"", NodesView, basename="map")
 
-city_router.register(r"", CityView)
+sensors_router = routers.DefaultRouter()
+sensors_router.register(r"", SensorsView, basename="sensors")
 
-nodes_router = routers.DefaultRouter()
+sensor_locations_router = routers.DefaultRouter()
+sensor_locations_router.register(r"", SensorLocationsView, basename="locations")
 
-nodes_router.register(r"", NodesView, basename="map")
+sensor_types_router = routers.DefaultRouter()
+sensor_types_router.register(r"", SensorTypesView, basename="sensor_types")
 
 api_urls = [
-    url(r"data/(?P<sensor_type>[air]+)/", include(data_router.urls)),
-    url(r"cities/", include(city_router.urls)),
-    url(r"nodes/", include(nodes_router.urls))
+    url(r"data/(?P<sensor_type>[air]+)/", include(stat_data_router.urls)),
+    url(r"data/", include(data_router.urls)),
+    url(r"cities/", include(cities_router.urls)),
+    url(r"nodes/", include(nodes_router.urls)),
+    url(r"locations/", include(sensor_locations_router.urls)),
+    url(r"sensors/", include(sensors_router.urls)),
+    url(r"sensor-types/", include(sensor_types_router.urls)),
+    url(r"meta/", meta_data),
 ]