Merge pull request #1442 from normanjaeckel/TornadoAutoupdate

Rework of tornado autoupdate functionality

openslides/agenda/serializers.py

@@ -1,11 +1,11 @@
-from rest_framework.reverse import reverse
+from django.core.urlresolvers import reverse
 
-from openslides.utils.rest_api import serializers
+from openslides.utils.rest_api import get_collection_and_id_from_url, serializers
 
 from .models import Item, Speaker
 
 
-class SpeakerSerializer(serializers.HyperlinkedModelSerializer):
+class SpeakerSerializer(serializers.ModelSerializer):
     """
     Serializer for agenda.models.Speaker objects.
     """
@@ -21,22 +21,20 @@ class SpeakerSerializer(serializers.HyperlinkedModelSerializer):
 
 class RelatedItemRelatedField(serializers.RelatedField):
     """
-    A custom field to use for the `content_object` generic relationship.
+    A custom field to use for the content_object generic relationship.
     """
     def to_representation(self, value):
         """
-        Returns the url to the related object.
+        Returns info concerning the related object extracted from the api URL
+        of this object.
         """
-        request = self.context.get('request', None)
-        assert request is not None, (
-            "`%s` requires the request in the serializer"
-            " context. Add `context={'request': request}` when instantiating "
-            "the serializer." % self.__class__.__name__)
         view_name = '%s-detail' % type(value)._meta.object_name.lower()
-        return reverse(view_name, kwargs={'pk': value.pk}, request=request)
+        url = reverse(view_name, kwargs={'pk': value.pk})
+        collection, obj_id = get_collection_and_id_from_url(url)
+        return {'collection': collection, 'id': obj_id}
 
 
-class ItemSerializer(serializers.HyperlinkedModelSerializer):
+class ItemSerializer(serializers.ModelSerializer):
     """
     Serializer for agenda.models.Item objects.
     """
@@ -49,7 +47,7 @@ class ItemSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Item
         fields = (
-            'url',
+            'id',
             'item_number',
             'item_no',
             'title',

openslides/assignment/serializers.py

@@ -9,7 +9,7 @@ from .models import (
     AssignmentVote)
 
 
-class AssignmentCandidateSerializer(serializers.HyperlinkedModelSerializer):
+class AssignmentCandidateSerializer(serializers.ModelSerializer):
     """
     Serializer for assignment.models.AssignmentCandidate objects.
     """
@@ -19,21 +19,19 @@ class AssignmentCandidateSerializer(serializers.HyperlinkedModelSerializer):
             'id',
             'person',
             'elected',
-            'blocked')
+            'blocked',)
 
 
-class AssignmentVoteSerializer(serializers.HyperlinkedModelSerializer):
+class AssignmentVoteSerializer(serializers.ModelSerializer):
     """
     Serializer for assignment.models.AssignmentVote objects.
     """
     class Meta:
         model = AssignmentVote
-        fields = (
+        fields = ('weight', 'value',)
-            'weight',
-            'value')
 
 
-class AssignmentOptionSerializer(serializers.HyperlinkedModelSerializer):
+class AssignmentOptionSerializer(serializers.ModelSerializer):
     """
     Serializer for assignment.models.AssignmentOption objects.
     """
@@ -41,9 +39,7 @@ class AssignmentOptionSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = AssignmentOption
-        fields = (
+        fields = ('candidate', 'assignmentvote_set',)
-            'candidate',
-            'assignmentvote_set')
 
 
 class FilterPollListSerializer(serializers.ListSerializer):
@@ -62,7 +58,7 @@ class FilterPollListSerializer(serializers.ListSerializer):
         return [self.child.to_representation(item) for item in iterable]
 
 
-class AssignmentAllPollSerializer(serializers.HyperlinkedModelSerializer):
+class AssignmentAllPollSerializer(serializers.ModelSerializer):
     """
     Serializer for assignment.models.AssignmentPoll objects.
 
@@ -80,7 +76,7 @@ class AssignmentAllPollSerializer(serializers.HyperlinkedModelSerializer):
             'assignmentoption_set',
             'votesvalid',
             'votesinvalid',
-            'votescast')
+            'votescast',)
 
 
 class AssignmentShortPollSerializer(AssignmentAllPollSerializer):
@@ -100,10 +96,10 @@ class AssignmentShortPollSerializer(AssignmentAllPollSerializer):
             'assignmentoption_set',
             'votesvalid',
             'votesinvalid',
-            'votescast')
+            'votescast',)
 
 
-class AssignmentFullSerializer(serializers.HyperlinkedModelSerializer):
+class AssignmentFullSerializer(serializers.ModelSerializer):
     """
     Serializer for assignment.models.Assignment objects. With all polls.
     """
@@ -113,7 +109,7 @@ class AssignmentFullSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Assignment
         fields = (
-            'url',
+            'id',
             'name',
             'description',
             'posts',
@@ -133,7 +129,7 @@ class AssignmentShortSerializer(AssignmentFullSerializer):
     class Meta:
         model = Assignment
         fields = (
-            'url',
+            'id',
             'name',
             'description',
             'posts',

openslides/core/serializers.py

@@ -3,19 +3,19 @@ from openslides.utils.rest_api import serializers
 from .models import CustomSlide, Tag
 
 
-class CustomSlideSerializer(serializers.HyperlinkedModelSerializer):
+class CustomSlideSerializer(serializers.ModelSerializer):
     """
     Serializer for core.models.CustomSlide objects.
     """
     class Meta:
         model = CustomSlide
-        fields = ('url', 'title', 'text', 'weight',)
+        fields = ('id', 'title', 'text', 'weight',)
 
 
-class TagSerializer(serializers.HyperlinkedModelSerializer):
+class TagSerializer(serializers.ModelSerializer):
     """
     Serializer for core.models.Tag objects.
     """
     class Meta:
         model = Tag
-        fields = ('url', 'name',)
+        fields = ('id', 'name',)

openslides/global_settings.py

@@ -170,10 +170,13 @@ CKEDITOR_CONFIGS = {
 }
 
 
-# Use small alternative with tornado as frontend or big alternative with a
+# Set this True to use tornado as single wsgi server. Set this False to use
-# webserver as wsgi server.
+# other webserver like Apache or Nginx as wsgi server.
 USE_TORNADO_AS_WSGI_SERVER = True
 
+OPENSLIDES_WSGI_NETWORK_LOCATION = ''
+
+
 TEST_RUNNER = 'openslides.utils.test.OpenSlidesDiscoverRunner'
 
 # Config for the REST Framework

openslides/mediafile/serializers.py

@@ -3,7 +3,7 @@ from openslides.utils.rest_api import serializers
 from .models import Mediafile
 
 
-class MediafileSerializer(serializers.HyperlinkedModelSerializer):
+class MediafileSerializer(serializers.ModelSerializer):
     """
     Serializer for mediafile.models.Mediafile objects.
     """
@@ -11,6 +11,15 @@ class MediafileSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Mediafile
+        fields = (
+            'id',
+            'title',
+            'mediafile',
+            'uploader',
+            'filesize',
+            'filetype',
+            'timestamp',
+            'is_presentable',)
 
     def get_filesize(self, mediafile):
         return mediafile.get_filesize()

openslides/motion/serializers.py

@@ -1,5 +1,3 @@
-from rest_framework.reverse import reverse
-
 from openslides.utils.rest_api import serializers
 
 from .models import (
@@ -16,13 +14,13 @@ from .models import (
     Workflow,)
 
 
-class CategorySerializer(serializers.HyperlinkedModelSerializer):
+class CategorySerializer(serializers.ModelSerializer):
     """
     Serializer for motion.models.Category objects.
     """
     class Meta:
         model = Category
-        fields = ('url', 'name', 'prefix',)
+        fields = ('id', 'name', 'prefix',)
 
 
 class StateSerializer(serializers.ModelSerializer):
@@ -46,7 +44,7 @@ class StateSerializer(serializers.ModelSerializer):
             'next_states',)
 
 
-class WorkflowSerializer(serializers.HyperlinkedModelSerializer):
+class WorkflowSerializer(serializers.ModelSerializer):
     """
     Serializer for motion.models.Workflow objects.
     """
@@ -55,10 +53,10 @@ class WorkflowSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Workflow
-        fields = ('url', 'name', 'state_set', 'first_state',)
+        fields = ('id', 'name', 'state_set', 'first_state',)
 
 
-class MotionSubmitterSerializer(serializers.HyperlinkedModelSerializer):
+class MotionSubmitterSerializer(serializers.ModelSerializer):
     """
     Serializer for motion.models.MotionSubmitter objects.
     """
@@ -67,7 +65,7 @@ class MotionSubmitterSerializer(serializers.HyperlinkedModelSerializer):
         fields = ('person',)  # TODO: Rename this to 'user', see #1348
 
 
-class MotionSupporterSerializer(serializers.HyperlinkedModelSerializer):
+class MotionSupporterSerializer(serializers.ModelSerializer):
     """
     Serializer for motion.models.MotionSupporter objects.
     """
@@ -76,7 +74,7 @@ class MotionSupporterSerializer(serializers.HyperlinkedModelSerializer):
         fields = ('person',)  # TODO: Rename this to 'user', see #1348
 
 
-class MotionLogSerializer(serializers.HyperlinkedModelSerializer):
+class MotionLogSerializer(serializers.ModelSerializer):
     """
     Serializer for motion.models.MotionLog objects.
     """
@@ -136,7 +134,7 @@ class MotionVersionSerializer(serializers.ModelSerializer):
             'reason',)
 
 
-class MotionSerializer(serializers.HyperlinkedModelSerializer):
+class MotionSerializer(serializers.ModelSerializer):
     """
     Serializer for motion.models.Motion objects.
     """
@@ -152,7 +150,7 @@ class MotionSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Motion
         fields = (
-            'url',
+            'id',
             'identifier',
             'identifier_number',
             'parent',
@@ -170,11 +168,6 @@ class MotionSerializer(serializers.HyperlinkedModelSerializer):
 
     def get_workflow(self, motion):
         """
-        Returns the hyperlink to the workflow of the motion.
+        Returns the id of the workflow of the motion.
         """
-        request = self.context.get('request', None)
+        return motion.state.workflow.pk
-        assert request is not None, (
-            "`%s` requires the request in the serializer"
-            " context. Add `context={'request': request}` when instantiating "
-            "the serializer." % self.__class__.__name__)
-        return reverse('workflow-detail', kwargs={'pk': motion.state.workflow.pk}, request=request)

openslides/users/apps.py

@@ -16,7 +16,7 @@ class UsersAppConfig(AppConfig):
         from openslides.projector.api import register_slide_model
         from openslides.utils.rest_api import router
         from .signals import setup_users_config, user_post_save
-        from .views import UserViewSet
+        from .views import GroupViewSet, UserViewSet
 
         # Load User model.
         User = self.get_model('User')
@@ -30,3 +30,4 @@ class UsersAppConfig(AppConfig):
 
         # Register viewsets.
         router.register('users/user', UserViewSet)
+        router.register('users/group', GroupViewSet)

openslides/users/serializers.py

@@ -1,6 +1,6 @@
 from openslides.utils.rest_api import serializers
 
-from .models import User
+from .models import Group, User  # TODO: Don't import Group from models but from core.models.
 
 
 class UserShortSerializer(serializers.ModelSerializer):
@@ -12,12 +12,13 @@ class UserShortSerializer(serializers.ModelSerializer):
     class Meta:
         model = User
         fields = (
-            'url',
+            'id',
             'username',
             'title',
             'first_name',
             'last_name',
-            'structure_level')
+            'structure_level',
+            'groups',)
 
 
 class UserFullSerializer(serializers.ModelSerializer):
@@ -29,7 +30,7 @@ class UserFullSerializer(serializers.ModelSerializer):
     class Meta:
         model = User
         fields = (
-            'url',
+            'id',
             'is_present',
             'username',
             'title',
@@ -38,5 +39,32 @@ class UserFullSerializer(serializers.ModelSerializer):
             'structure_level',
             'about_me',
             'comment',
+            'groups',
             'default_password',
-            'is_active')
+            'last_login',
+            'is_active',)
+
+
+class PermissionRelatedField(serializers.RelatedField):
+    """
+    A custom field to use for the permission relationship.
+    """
+    def to_representation(self, value):
+        """
+        Returns the permission name (app_label.codename).
+        """
+        return '.'.join((value.content_type.app_label, value.codename,))
+
+
+class GroupSerializer(serializers.ModelSerializer):
+    """
+    Serializer for django.contrib.auth.models.Group objects.
+    """
+    permissions = PermissionRelatedField(many=True, read_only=True)
+
+    class Meta:
+        model = Group
+        fields = (
+            'id',
+            'name',
+            'permissions',)

openslides/users/views.py

@@ -20,7 +20,7 @@ from .forms import (GroupForm, UserCreateForm, UserMultipleCreateForm,
                     UsersettingsForm, UserUpdateForm)
 from .models import Group, User
 from .pdf import users_to_pdf, users_passwords_to_pdf
-from .serializers import UserFullSerializer, UserShortSerializer
+from .serializers import GroupSerializer, UserFullSerializer, UserShortSerializer
 
 
 class UserListView(ListView):
@@ -263,7 +263,7 @@ class ResetPasswordView(SingleObjectMixin, QuestionView):
 
 class UserViewSet(viewsets.ModelViewSet):
     """
-    API endpoint to list, retrive, create, update and delete users.
+    API endpoint to list, retrieve, create, update and delete users.
     """
     model = User
     queryset = User.objects.all()
@@ -291,6 +291,27 @@ class UserViewSet(viewsets.ModelViewSet):
         return serializer_class
 
 
+class GroupViewSet(viewsets.ModelViewSet):
+    """
+    API endpoint to list, retrieve, create, update and delete groups.
+    """
+    model = Group
+    queryset = Group.objects.all()
+    serializer_class = GroupSerializer
+
+    def check_permissions(self, request):
+        """
+        Calls self.permission_denied() if the requesting user has not the
+        permission to see users and in case of create, update or destroy
+        requests the permission to see extra user data and to manage users.
+        """
+        if (not request.user.has_perm('users.can_see_name') or
+                (self.action in ('create', 'update', 'destroy') and not
+                 (request.user.has_perm('users.can_manage') and
+                  request.user.has_perm('users.can_see_extra_data')))):
+            self.permission_denied(request)
+
+
 class GroupListView(ListView):
     """
     Overview over all groups.

openslides/utils/autoupdate.py

@@ -1,3 +1,4 @@
+import json
 import os
 import posixpath
 from urllib.parse import unquote
@@ -18,8 +19,7 @@ from tornado.web import (
 )
 from tornado.wsgi import WSGIContainer
 
-REST_URL = 'http://localhost:8000'
+from .rest_api import get_collection_and_id_from_url
-# TODO: this is propably in the config
 
 
 class DjangoStaticFileHandler(StaticFileHandler):
@@ -58,56 +58,62 @@ class DjangoStaticFileHandler(StaticFileHandler):
 
 class OpenSlidesSockJSConnection(SockJSConnection):
     """
-    Sockjs connections for OpenSlides.
+    SockJS connection for OpenSlides.
     """
     waiters = set()
 
-    def on_open(self, request_info):
+    def on_open(self, info):
-        OpenSlidesSockJSConnection.waiters.add(self)
+        self.waiters.add(self)
-        self.request_info = request_info
+        self.connection_info = info
 
     def on_close(self):
         OpenSlidesSockJSConnection.waiters.remove(self)
 
-    def handle_rest_request(self, response):
+    def forward_rest_response(self, response):
         """
-        Handler that is called when the rest api responds.
+        Sends data to the client of the connection instance.
 
-        Sends the response.body to the client.
+        This method is called after succesful response of AsyncHTTPClient().
+        See send_object().
         """
-        # TODO: update cookies
+        collection, obj_id = get_collection_and_id_from_url(response.request.url)
-        if response.code == 200:
+        data = {
-            self.send(response.body)
+            'url': response.request.url,
-
+            'status_code': response.code,
-    @classmethod
+            'collection': collection,
-    def send_updates(cls, data):
+            'id': obj_id,
-        # TODO: use a bluk send
+            'data': json.loads(response.body.decode())}
-        for waiter in cls.waiters:
+        self.send(data)
-            waiter.send(data)
 
     @classmethod
     def send_object(cls, object_url):
         """
-        Send OpenSlides objects to all connected clients.
+        Sends an OpenSlides object to all connected clients (waiters).
 
-        First, receive the object from the OpenSlides ReST API.
+        First, retrieve the object from the OpenSlides REST api using the given
+        object_url.
         """
-        for waiter in cls.waiters:
+        # Join network location with object URL.
-            # Get the object from the ReST API
+        # TODO: Use host and port as given in the start script
-            http_client = AsyncHTTPClient()
+        wsgi_network_location = settings.OPENSLIDES_WSGI_NETWORK_LOCATION or 'http://localhost:8000'
-            headers = HTTPHeaders()
+        url = ''.join((wsgi_network_location, object_url))
-            # TODO: read to python Morselcookies and why "set-Cookie" does not work
-            request_cookies = waiter.request_info.cookies.values()
-            cookie_value = ';'.join("%s=%s" % (cookie.key, cookie.value)
-                                    for cookie in request_cookies)
-            headers.parse_line("Cookie: %s" % cookie_value)
 
+        # Send out internal HTTP request to get data from the REST api.
+        for waiter in cls.waiters:
+            # Read waiter's former cookies and parse session cookie to new header object.
+            session_cookie = waiter.connection_info.cookies[settings.SESSION_COOKIE_NAME]
+            headers = HTTPHeaders()
+            headers.add('Cookie', '%s=%s' % (settings.SESSION_COOKIE_NAME, session_cookie.value))
+            # Setup uncompressed request.
             request = HTTPRequest(
-                url=''.join((REST_URL, object_url)),
+                url=url,
                 headers=headers,
                 decompress_response=False)
-            # TODO: use proxy_host as header from waiter.request_info
+            # Setup non-blocking HTTP client
-            http_client.fetch(request, waiter.handle_rest_request)
+            http_client = AsyncHTTPClient()
+            # Executes the request, asynchronously returning an HTTPResponse
+            # and calling waiter's forward_rest_response() method.
+            http_client.fetch(request, waiter.forward_rest_response)
 
 
 def run_tornado(addr, port, *args, **kwargs):
@@ -150,7 +156,7 @@ def inform_changed_data(*args):
         try:
             rest_urls.add(instance.get_root_rest_url())
         except AttributeError:
-            # instance has no method get_root_rest_url
+            # Instance has no method get_root_rest_url. Just skip it.
             pass
 
     if settings.USE_TORNADO_AS_WSGI_SERVER:
@@ -158,7 +164,7 @@ def inform_changed_data(*args):
             OpenSlidesSockJSConnection.send_object(url)
     else:
         pass
-        # TODO: fix me
+        # TODO: Implement big varainte with Apache or Nginx as wsgi webserver.
 
 
 def inform_changed_data_receiver(sender, instance, **kwargs):

openslides/utils/rest_api.py

@@ -1,6 +1,12 @@
+import re
+
+from urllib.parse import urlparse
+
 from django.core.urlresolvers import reverse
 from rest_framework import response, routers, serializers, viewsets  # noqa
 
+from .exceptions import OpenSlidesError
+
 router = routers.DefaultRouter()
 
 
@@ -26,3 +32,20 @@ class RESTModelMixin:
         root_instance = self.get_root_rest_element()
         rest_url = '%s-detail' % type(root_instance)._meta.object_name.lower()
         return reverse(rest_url, args=[str(root_instance.pk)])
+
+
+def get_collection_and_id_from_url(url):
+    """
+    Helper function. Returns a tuple containing the collection name and the id
+    extracted out of the given REST api URL.
+
+    For example get_collection_and_id_from_url('http://localhost/api/users/user/3/')
+    returns ('users/user', '3').
+
+    Raises OpenSlidesError if the URL is invalid.
+    """
+    path = urlparse(url).path
+    match = re.match(r'^/api/(?P<name>[-\w]+/[-\w]+)/(?P<id>[-\w]+)/$', path)
+    if not match:
+        raise OpenSlidesError('Invalid REST api URL: %s' % url)
+    return match.group('name'), match.group('id')