drop python 3.5

This commit is contained in:
Oskar Hahn 2018-08-22 22:00:08 +02:00
parent cbf8a33b8d
commit aac9dcabf5
43 changed files with 123 additions and 125 deletions

View File

@ -5,7 +5,6 @@ cache:
pip: true pip: true
yarn: true yarn: true
python: python:
- "3.5"
- "3.6" - "3.6"
- "3.7" - "3.7"
env: env:

View File

@ -9,7 +9,8 @@ Version 3.0 (unreleased)
Core: Core:
- Change URL schema [#3798]. - Change URL schema [#3798].
- Update to channels2 - Update to channels2 [#3796].
- Drop Python 3.5 support[#3805].
Version 2.3 (unreleased) Version 2.3 (unreleased)

View File

@ -15,7 +15,7 @@ Installation and start of the development version
a. Check requirements a. Check requirements
''''''''''''''''''''' '''''''''''''''''''''
Make sure that you have installed `Python (>= 3.5) <https://www.python.org/>`_, Make sure that you have installed `Python (>= 3.6) <https://www.python.org/>`_,
`Node.js (>=4.x) <https://nodejs.org/>`_, `Yarn <https://yarnpkg.com/>`_ and `Node.js (>=4.x) <https://nodejs.org/>`_, `Yarn <https://yarnpkg.com/>`_ and
`Git <http://git-scm.com/>`_ on your system. You also need build-essential `Git <http://git-scm.com/>`_ on your system. You also need build-essential
packages and header files and a static library for Python. packages and header files and a static library for Python.
@ -99,7 +99,7 @@ Use gulp watch in a second command-line interface::
Follow the instructions above (Installation on GNU/Linux or Mac OS X) but care Follow the instructions above (Installation on GNU/Linux or Mac OS X) but care
of the following variations. of the following variations.
To get Python download and run the latest `Python 3.5 32-bit (x86) executable To get Python download and run the latest `Python 3.7 32-bit (x86) executable
installer <https://www.python.org/downloads/windows/>`_. Note that the 32-bit installer <https://www.python.org/downloads/windows/>`_. Note that the 32-bit
installer is required even on a 64-bit Windows system. If you use the 64-bit installer is required even on a 64-bit Windows system. If you use the 64-bit
installer, step d. of the instruction might fail unless you installed some installer, step d. of the instruction might fail unless you installed some

View File

@ -1,4 +1,4 @@
FROM python:3.5 FROM python:3.7
RUN apt-get -y update && apt-get -y upgrade RUN apt-get -y update && apt-get -y upgrade
RUN apt-get install -y libpq-dev supervisor curl vim RUN apt-get install -y libpq-dev supervisor curl vim
RUN useradd -m openslides RUN useradd -m openslides

View File

@ -26,7 +26,7 @@ Installation
a. Check requirements a. Check requirements
''''''''''''''''''''' '''''''''''''''''''''
Make sure that you have installed `Python (>= 3.5) <https://www.python.org/>`_ Make sure that you have installed `Python (>= 3.6) <https://www.python.org/>`_
on your system. on your system.
Additional you need build-essential packages, header files and a static Additional you need build-essential packages, header files and a static

View File

@ -2,7 +2,7 @@
import os import os
import sys import sys
from typing import Dict # noqa from typing import Dict
import django import django
from django.core.management import call_command, execute_from_command_line from django.core.management import call_command, execute_from_command_line
@ -239,7 +239,7 @@ def createsettings(args):
""" """
settings_dir = args.settings_dir settings_dir = args.settings_dir
local_installation = is_local_installation() local_installation = is_local_installation()
context = {} # type: Dict[str, str] context: Dict[str, str] = {}
if local_installation: if local_installation:
if settings_dir is None: if settings_dir is None:

View File

@ -1,4 +1,4 @@
from typing import Any, Dict, Iterable, List, Optional # noqa from typing import Any, Dict, Iterable, List, Optional
from ..utils.access_permissions import BaseAccessPermissions from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
@ -72,7 +72,7 @@ class ItemAccessPermissions(BaseAccessPermissions):
# In non internal case managers see everything and non managers see # In non internal case managers see everything and non managers see
# everything but comments. # everything but comments.
if has_perm(user, 'agenda.can_manage'): if has_perm(user, 'agenda.can_manage'):
blocked_keys_non_internal_hidden_case = [] # type: Iterable[str] blocked_keys_non_internal_hidden_case: Iterable[str] = []
can_see_hidden = True can_see_hidden = True
else: else:
blocked_keys_non_internal_hidden_case = ('comment',) blocked_keys_non_internal_hidden_case = ('comment',)

View File

@ -1,5 +1,5 @@
from collections import defaultdict from collections import defaultdict
from typing import Dict, List, Set # noqa from typing import Dict, List, Set
from django.conf import settings from django.conf import settings
from django.contrib.auth.models import AnonymousUser from django.contrib.auth.models import AnonymousUser
@ -65,7 +65,7 @@ class ItemManager(models.Manager):
all of their children. all of their children.
""" """
queryset = self.order_by('weight') queryset = self.order_by('weight')
item_children = defaultdict(list) # type: Dict[int, List[Item]] item_children: Dict[int, List[Item]] = defaultdict(list)
root_items = [] root_items = []
for item in queryset: for item in queryset:
if only_item_type is not None and item.type != only_item_type: if only_item_type is not None and item.type != only_item_type:
@ -121,7 +121,7 @@ class ItemManager(models.Manager):
yield (element['id'], parent, weight) yield (element['id'], parent, weight)
yield from walk_items(element.get('children', []), element['id']) yield from walk_items(element.get('children', []), element['id'])
touched_items = set() # type: Set[int] touched_items: Set[int] = set()
db_items = dict((item.pk, item) for item in Item.objects.all()) db_items = dict((item.pk, item) for item in Item.objects.all())
for item_id, parent_id, weight in walk_items(tree): for item_id, parent_id, weight in walk_items(tree):
# Check that the item is only once in the tree to prevent invalid trees # Check that the item is only once in the tree to prevent invalid trees

View File

@ -1,4 +1,4 @@
from typing import Set # noqa from typing import Set
from django.apps import apps from django.apps import apps
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
@ -72,7 +72,7 @@ def required_users(sender, request_user, **kwargs):
if request_user can see the agenda. This function may return an empty if request_user can see the agenda. This function may return an empty
set. set.
""" """
speakers = set() # type: Set[int] speakers: Set[int] = set()
if has_perm(request_user, 'agenda.can_see'): if has_perm(request_user, 'agenda.can_see'):
for item_collection_element in Collection(Item.get_collection_string()).element_generator(): for item_collection_element in Collection(Item.get_collection_string()).element_generator():
full_data = item_collection_element.get_full_data() full_data = item_collection_element.get_full_data()

View File

@ -1,6 +1,6 @@
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from ..utils.access_permissions import BaseAccessPermissions # noqa from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
from ..utils.collection import CollectionElement from ..utils.collection import CollectionElement

View File

@ -1,4 +1,4 @@
from typing import Dict, List, Union # noqa from typing import List
from django.apps import AppConfig from django.apps import AppConfig
from mypy_extensions import TypedDict from mypy_extensions import TypedDict
@ -48,10 +48,10 @@ class AssignmentsAppConfig(AppConfig):
def get_angular_constants(self): def get_angular_constants(self):
assignment = self.get_model('Assignment') assignment = self.get_model('Assignment')
InnerItem = TypedDict('InnerItem', {'value': int, 'display_name': str}) InnerItem = TypedDict('InnerItem', {'value': int, 'display_name': str})
Item = TypedDict('Item', {'name': str, 'value': List[InnerItem]}) # noqa Item = TypedDict('Item', {'name': str, 'value': List[InnerItem]})
data = { data: Item = {
'name': 'AssignmentPhases', 'name': 'AssignmentPhases',
'value': []} # type: Item 'value': []}
for phase in assignment.PHASES: for phase in assignment.PHASES:
data['value'].append({ data['value'].append({
'value': phase[0], 'value': phase[0],

View File

@ -1,5 +1,5 @@
from collections import OrderedDict from collections import OrderedDict
from typing import Any, Dict, List, Optional # noqa from typing import Any, Dict, List
from django.conf import settings from django.conf import settings
from django.contrib.contenttypes.fields import GenericRelation from django.contrib.contenttypes.fields import GenericRelation
@ -301,14 +301,14 @@ class Assignment(RESTModelMixin, models.Model):
Returns a table represented as a list with all candidates from all Returns a table represented as a list with all candidates from all
related polls and their vote results. related polls and their vote results.
""" """
vote_results_dict = OrderedDict() # type: Dict[Any, List[AssignmentVote]] vote_results_dict: Dict[Any, List[AssignmentVote]] = OrderedDict()
polls = self.polls.all() polls = self.polls.all()
if only_published: if only_published:
polls = polls.filter(published=True) polls = polls.filter(published=True)
# All PollOption-Objects related to this assignment # All PollOption-Objects related to this assignment
options = [] # type: List[AssignmentOption] options: List[AssignmentOption] = []
for poll in polls: for poll in polls:
options += poll.get_options() options += poll.get_options()
@ -318,7 +318,7 @@ class Assignment(RESTModelMixin, models.Model):
continue continue
vote_results_dict[candidate] = [] vote_results_dict[candidate] = []
for poll in polls: for poll in polls:
votes = {} # type: Any votes: Any = {}
try: try:
# candidate related to this poll # candidate related to this poll
poll_option = poll.get_options().get(candidate=candidate) poll_option = poll.get_options().get(candidate=candidate)
@ -333,7 +333,7 @@ class Assignment(RESTModelMixin, models.Model):
""" """
Container for runtime information for agenda app (on create or update of this instance). Container for runtime information for agenda app (on create or update of this instance).
""" """
agenda_item_update_information = {} # type: Dict[str, Any] agenda_item_update_information: Dict[str, Any] = {}
def get_agenda_title(self): def get_agenda_title(self):
return str(self) return str(self)

View File

@ -1,4 +1,4 @@
from typing import Any, Set # noqa from typing import Any, Set
from django.apps import apps from django.apps import apps
@ -24,7 +24,7 @@ def required_users(sender, request_user, **kwargs):
options) in any assignment if request_user can see assignments. This options) in any assignment if request_user can see assignments. This
function may return an empty set. function may return an empty set.
""" """
candidates = set() # type: Set[Any] # TODO: Replace Any candidates: Set[Any] = set()
if has_perm(request_user, 'assignments.can_see'): if has_perm(request_user, 'assignments.can_see'):
for assignment_collection_element in Collection(Assignment.get_collection_string()).element_generator(): for assignment_collection_element in Collection(Assignment.get_collection_string()).element_generator():
full_data = assignment_collection_element.get_full_data() full_data = assignment_collection_element.get_full_data()

View File

@ -1,6 +1,6 @@
from collections import OrderedDict from collections import OrderedDict
from operator import attrgetter from operator import attrgetter
from typing import Any, List # noqa from typing import Any, List
from django.apps import AppConfig from django.apps import AppConfig
from django.conf import settings from django.conf import settings
@ -93,7 +93,7 @@ class CoreAppConfig(AppConfig):
'value': client_settings_dict} 'value': client_settings_dict}
# Config variables # Config variables
config_groups = [] # type: List[Any] # TODO: Replace Any by correct type config_groups: List[Any] = []
for config_variable in sorted(config.config_variables.values(), key=attrgetter('weight')): for config_variable in sorted(config.config_variables.values(), key=attrgetter('weight')):
if config_variable.is_hidden(): if config_variable.is_hidden():
# Skip hidden config variables. Do not even check groups and subgroups. # Skip hidden config variables. Do not even check groups and subgroups.

View File

@ -46,10 +46,10 @@ class ConfigHandler:
def __init__(self) -> None: def __init__(self) -> None:
# Dict, that keeps all ConfigVariable objects. Has to be set at statup. # Dict, that keeps all ConfigVariable objects. Has to be set at statup.
# See the ready() method in openslides.core.apps. # See the ready() method in openslides.core.apps.
self.config_variables = {} # type: Dict[str, ConfigVariable] self.config_variables: Dict[str, ConfigVariable] = {}
# Index to get the database id from a given config key # Index to get the database id from a given config key
self.key_to_id = None # type: Optional[Dict[str, int]] self.key_to_id: Optional[Dict[str, int]] = None
def __getitem__(self, key: str) -> Any: def __getitem__(self, key: str) -> Any:
""" """

View File

@ -1,7 +1,7 @@
import json import json
import uuid import uuid
from textwrap import dedent from textwrap import dedent
from typing import Any, Dict, List, cast # noqa from typing import Any, Dict, List, cast
from django.apps import apps from django.apps import apps
from django.conf import settings from django.conf import settings
@ -91,7 +91,7 @@ class WebclientJavaScriptView(utils_views.View):
AngularJS app for the requested realm (site or projector). Also code AngularJS app for the requested realm (site or projector). Also code
for plugins is appended. The result is not uglified. for plugins is appended. The result is not uglified.
""" """
cache = {} # type: Dict[str, str] cache: Dict[str, str] = {}
def __init__(self, *args: Any, **kwargs: Any) -> None: def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@ -102,8 +102,8 @@ class WebclientJavaScriptView(utils_views.View):
self.init_cache('projector') self.init_cache('projector')
def init_cache(self, realm: str) -> None: def init_cache(self, realm: str) -> None:
angular_modules = [] # type: List[str] angular_modules: List[str] = []
js_files = [] # type: List[str] js_files: List[str] = []
for app_config in apps.get_app_configs(): for app_config in apps.get_app_configs():
# Add the angular app if the module has one. # Add the angular app if the module has one.
if getattr(app_config, 'angular_{}_module'.format(realm), False): if getattr(app_config, 'angular_{}_module'.format(realm), False):
@ -836,16 +836,16 @@ class VersionView(utils_views.APIView):
http_method_names = ['get'] http_method_names = ['get']
def get_context_data(self, **context): def get_context_data(self, **context):
Result = TypedDict('Result', { # noqa Result = TypedDict('Result', {
'openslides_version': str, 'openslides_version': str,
'openslides_license': str, 'openslides_license': str,
'openslides_url': str, 'openslides_url': str,
'plugins': List[Dict[str, str]]}) 'plugins': List[Dict[str, str]]})
result = dict( result: Result = dict(
openslides_version=version, openslides_version=version,
openslides_license=license, openslides_license=license,
openslides_url=url, openslides_url=url,
plugins=[]) # type: Result plugins=[])
# Versions of plugins. # Versions of plugins.
for plugin in settings.INSTALLED_PLUGINS: for plugin in settings.INSTALLED_PLUGINS:
result['plugins'].append({ result['plugins'].append({

View File

@ -1,6 +1,6 @@
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from ..utils.access_permissions import BaseAccessPermissions # noqa from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
from ..utils.collection import CollectionElement from ..utils.collection import CollectionElement

View File

@ -2,7 +2,7 @@ from copy import deepcopy
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from ..core.config import config from ..core.config import config
from ..utils.access_permissions import BaseAccessPermissions # noqa from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import has_perm from ..utils.auth import has_perm
from ..utils.collection import CollectionElement from ..utils.collection import CollectionElement

View File

@ -1,4 +1,4 @@
from typing import Any, Dict # noqa from typing import Any, Dict
from django.conf import settings from django.conf import settings
from django.contrib.auth.models import AnonymousUser from django.contrib.auth.models import AnonymousUser
@ -692,7 +692,7 @@ class Motion(RESTModelMixin, models.Model):
""" """
Container for runtime information for agenda app (on create or update of this instance). Container for runtime information for agenda app (on create or update of this instance).
""" """
agenda_item_update_information = {} # type: Dict[str, Any] agenda_item_update_information: Dict[str, Any] = {}
def get_agenda_title(self): def get_agenda_title(self):
""" """
@ -1052,7 +1052,7 @@ class MotionBlock(RESTModelMixin, models.Model):
""" """
Container for runtime information for agenda app (on create or update of this instance). Container for runtime information for agenda app (on create or update of this instance).
""" """
agenda_item_update_information = {} # type: Dict[str, Any] agenda_item_update_information: Dict[str, Any] = {}
@property @property
def agenda_item(self): def agenda_item(self):

View File

@ -1,4 +1,4 @@
from typing import Dict # noqa from typing import Dict
from django.db import transaction from django.db import transaction
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
@ -233,7 +233,7 @@ class MotionPollSerializer(ModelSerializer):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
# The following dictionary is just a cache for several votes. # The following dictionary is just a cache for several votes.
self._votes_dicts = {} # type: Dict[int, Dict[int, int]] self._votes_dicts: Dict[int, Dict[int, int]] = {}
return super().__init__(*args, **kwargs) return super().__init__(*args, **kwargs)
def get_yes(self, obj): def get_yes(self, obj):

View File

@ -1,4 +1,4 @@
from typing import Set # noqa from typing import Set
from django.apps import apps from django.apps import apps
from django.utils.translation import ugettext_noop from django.utils.translation import ugettext_noop
@ -126,7 +126,7 @@ def required_users(sender, request_user, **kwargs):
any motion if request_user can see motions. This function may return an any motion if request_user can see motions. This function may return an
empty set. empty set.
""" """
submitters_supporters = set() # type: Set[int] submitters_supporters: Set[int] = set()
if has_perm(request_user, 'motions.can_see'): if has_perm(request_user, 'motions.can_see'):
for motion_collection_element in Collection(Motion.get_collection_string()).element_generator(): for motion_collection_element in Collection(Motion.get_collection_string()).element_generator():
full_data = motion_collection_element.get_full_data() full_data = motion_collection_element.get_full_data()

View File

@ -1,5 +1,5 @@
import re import re
from typing import Optional # noqa from typing import Optional
from django.conf import settings from django.conf import settings
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
@ -113,9 +113,9 @@ class MotionViewSet(ModelViewSet):
# Check if parent motion exists. # Check if parent motion exists.
if request.data.get('parent_id') is not None: if request.data.get('parent_id') is not None:
try: try:
parent_motion = CollectionElement.from_values( parent_motion: Optional[CollectionElement] = CollectionElement.from_values(
Motion.get_collection_string(), Motion.get_collection_string(),
request.data['parent_id']) # type: Optional[CollectionElement] request.data['parent_id'])
except Motion.DoesNotExist: except Motion.DoesNotExist:
raise ValidationError({'detail': _('The parent motion does not exist.')}) raise ValidationError({'detail': _('The parent motion does not exist.')})
else: else:

View File

@ -1,5 +1,5 @@
import locale import locale
from typing import Type # noqa from typing import Optional, Type
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
from django.db import models from django.db import models
@ -17,7 +17,7 @@ class BaseOption(models.Model):
which has to be a subclass of BaseVote. Otherwise you have to override the which has to be a subclass of BaseVote. Otherwise you have to override the
get_vote_class method. get_vote_class method.
""" """
vote_class = None # type: Type[BaseVote] vote_class: Optional[Type['BaseVote']] = None
class Meta: class Meta:
abstract = True abstract = True

View File

@ -1,4 +1,4 @@
from typing import Any, Dict # noqa from typing import Any, Dict
from django.contrib.contenttypes.fields import GenericRelation from django.contrib.contenttypes.fields import GenericRelation
from django.db import models from django.db import models
@ -59,7 +59,7 @@ class Topic(RESTModelMixin, models.Model):
""" """
Container for runtime information for agenda app (on create or update of this instance). Container for runtime information for agenda app (on create or update of this instance).
""" """
agenda_item_update_information = {} # type: Dict[str, Any] agenda_item_update_information: Dict[str, Any] = {}
@property @property
def agenda_item(self): def agenda_item(self):

View File

@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
from django.contrib.auth.models import AnonymousUser from django.contrib.auth.models import AnonymousUser
from ..core.signals import user_data_required from ..core.signals import user_data_required
from ..utils.access_permissions import BaseAccessPermissions # noqa from ..utils.access_permissions import BaseAccessPermissions
from ..utils.auth import anonymous_is_enabled, has_perm from ..utils.auth import anonymous_is_enabled, has_perm
from ..utils.collection import CollectionElement from ..utils.collection import CollectionElement
@ -172,7 +172,7 @@ class PersonalNoteAccessPermissions(BaseAccessPermissions):
""" """
# Parse data. # Parse data.
if user is None: if user is None:
data = [] # type: List[Dict[str, Any]] data: List[Dict[str, Any]] = []
else: else:
for full in full_data: for full in full_data:
if full['user_id'] == user.id: if full['user_id'] == user.id:

View File

@ -1,5 +1,5 @@
import smtplib import smtplib
from typing import List # noqa from typing import List
from asgiref.sync import async_to_sync from asgiref.sync import async_to_sync
from django.conf import settings from django.conf import settings
@ -325,7 +325,7 @@ class GroupViewSet(ModelViewSet):
# Some permissions are added. # Some permissions are added.
if len(new_permissions) > 0: if len(new_permissions) > 0:
collection_elements = [] # type: List[CollectionElement] collection_elements: List[CollectionElement] = []
signal_results = permission_change.send(None, permissions=new_permissions, action='added') signal_results = permission_change.send(None, permissions=new_permissions, action='added')
for receiver, signal_collections in signal_results: for receiver, signal_collections in signal_results:
for cachable in signal_collections: for cachable in signal_collections:

View File

@ -1,9 +1,9 @@
from argparse import Namespace from argparse import Namespace
from typing import Any, Union # noqa from typing import Any, Optional
class OpenSlidesArguments(): class OpenSlidesArguments():
args = None # type: Union[None, Namespace] args: Optional[Namespace] = None
def __getitem__(self, key: str) -> Any: def __getitem__(self, key: str) -> Any:
if not self.args: if not self.args:

View File

@ -16,7 +16,7 @@ def to_ordered_dict(d: Optional[Dict]) -> Optional[OrderedDict]:
Little helper to hash information dict in inform_*_data. Little helper to hash information dict in inform_*_data.
""" """
if isinstance(d, dict): if isinstance(d, dict):
result = OrderedDict([(key, to_ordered_dict(d[key])) for key in sorted(d.keys())]) # type: Optional[OrderedDict] result: Optional[OrderedDict] = OrderedDict([(key, to_ordered_dict(d[key])) for key in sorted(d.keys())])
else: else:
result = d result = d
return result return result
@ -64,7 +64,7 @@ def inform_deleted_data(elements: Iterable[Tuple[str, int]], information: Dict[s
The argument information is added to each collection element. The argument information is added to each collection element.
""" """
collection_elements = {} # type: Dict[str, Any] collection_elements: Dict[str, Any] = {}
for element in elements: for element in elements:
collection_element = CollectionElement.from_values( collection_element = CollectionElement.from_values(
collection_string=element[0], collection_string=element[0],
@ -106,7 +106,7 @@ def inform_data_collection_element_list(collection_elements: List[CollectionElem
""" """
Global container for autoupdate bundles Global container for autoupdate bundles
""" """
autoupdate_bundle = {} # type: Dict[int, Dict[str, CollectionElement]] autoupdate_bundle: Dict[int, Dict[str, CollectionElement]] = {}
class AutoupdateBundleMiddleware: class AutoupdateBundleMiddleware:
@ -123,7 +123,7 @@ class AutoupdateBundleMiddleware:
response = self.get_response(request) response = self.get_response(request)
bundle = autoupdate_bundle.pop(thread_id) # type: Dict[str, CollectionElement] bundle: Dict[str, CollectionElement] = autoupdate_bundle.pop(thread_id)
async_to_sync(send_autoupdate)(bundle.values()) async_to_sync(send_autoupdate)(bundle.values())
return response return response
@ -138,7 +138,7 @@ async def send_autoupdate(collection_elements: Iterable[CollectionElement]) -> N
Does nothing if collection_elements is empty. Does nothing if collection_elements is empty.
""" """
if collection_elements: if collection_elements:
cache_elements = {} # type: Dict[str, Optional[Dict[str, Any]]] cache_elements: Dict[str, Optional[Dict[str, Any]]] = {}
for element in collection_elements: for element in collection_elements:
element_id = get_element_id(element.collection_string, element.id) element_id = get_element_id(element.collection_string, element.id)
if element.is_deleted(): if element.is_deleted():

View File

@ -73,7 +73,7 @@ class ElementCache:
self.use_restricted_data_cache = use_restricted_data_cache self.use_restricted_data_cache = use_restricted_data_cache
self.cache_provider = cache_provider_class(redis) self.cache_provider = cache_provider_class(redis)
self.cachable_provider = cachable_provider self.cachable_provider = cachable_provider
self._cachables = None # type: Optional[Dict[str, Cachable]] self._cachables: Optional[Dict[str, Cachable]] = None
# Start time is used as first change_id if there is non in redis # Start time is used as first change_id if there is non in redis
if start_time is None: if start_time is None:
@ -81,7 +81,7 @@ class ElementCache:
self.start_time = start_time self.start_time = start_time
# Contains Futures to controll, that only one client updates the restricted_data. # Contains Futures to controll, that only one client updates the restricted_data.
self.restricted_data_cache_updater = {} # type: Dict[int, asyncio.Future] self.restricted_data_cache_updater: Dict[int, asyncio.Future] = {}
@property @property
def cachables(self) -> Dict[str, Cachable]: def cachables(self) -> Dict[str, Cachable]:
@ -109,7 +109,7 @@ class ElementCache:
""" """
Build or rebuild the full_data cache. Build or rebuild the full_data cache.
""" """
db_data = {} # type: Dict[str, List[Dict[str, Any]]] db_data = {}
for collection_string, cachable in self.cachables.items(): for collection_string, cachable in self.cachables.items():
db_data[collection_string] = await database_sync_to_async(cachable.get_elements)() db_data[collection_string] = await database_sync_to_async(cachable.get_elements)()
await self.save_full_data(db_data) await self.save_full_data(db_data)
@ -262,7 +262,7 @@ class ElementCache:
# If this succeeds, there is noone else currently updating the cache. # If this succeeds, there is noone else currently updating the cache.
# TODO: Make a timeout. Else this could block forever # TODO: Make a timeout. Else this could block forever
if await self.cache_provider.set_lock_restricted_data(get_user_id(user)): if await self.cache_provider.set_lock_restricted_data(get_user_id(user)):
future = asyncio.Future() # type: asyncio.Future future: asyncio.Future = asyncio.Future()
self.restricted_data_cache_updater[get_user_id(user)] = future self.restricted_data_cache_updater[get_user_id(user)] = future
# Get change_id for this user # Get change_id for this user
value = await self.cache_provider.get_change_id_user(get_user_id(user)) value = await self.cache_provider.get_change_id_user(get_user_id(user))
@ -318,7 +318,7 @@ class ElementCache:
await self.update_restricted_data(user) await self.update_restricted_data(user)
out = defaultdict(list) # type: Dict[str, List[Dict[str, Any]]] out: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
restricted_data = await self.cache_provider.get_all_data(get_user_id(user)) restricted_data = await self.cache_provider.get_all_data(get_user_id(user))
for element_id, data in restricted_data.items(): for element_id, data in restricted_data.items():
if element_id.decode().startswith('_config'): if element_id.decode().startswith('_config'):

View File

@ -1,5 +1,4 @@
from collections import defaultdict from collections import defaultdict
from typing import Set # noqa
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
@ -8,6 +7,7 @@ from typing import (
Iterable, Iterable,
List, List,
Optional, Optional,
Set,
Tuple, Tuple,
Union, Union,
) )
@ -108,7 +108,7 @@ class RedisCacheProvider(BaseCacheProvider):
""" """
Cache provider that loads and saves the data to redis. Cache provider that loads and saves the data to redis.
""" """
redis_pool = None # type: Optional[aioredis.RedisConnection] redis_pool: Optional[aioredis.RedisConnection] = None
def __init__(self, redis: str) -> None: def __init__(self, redis: str) -> None:
self.redis_address = redis self.redis_address = redis
@ -232,8 +232,8 @@ class RedisCacheProvider(BaseCacheProvider):
""" """
# TODO: rewrite with lua to get all elements with one request # TODO: rewrite with lua to get all elements with one request
redis = await self.get_connection() redis = await self.get_connection()
changed_elements = defaultdict(list) # type: Dict[str, List[bytes]] changed_elements: Dict[str, List[bytes]] = defaultdict(list)
deleted_elements = [] # type: List[str] deleted_elements: List[str] = []
for element_id in await redis.zrangebyscore(self.get_change_id_cache_key(), min=change_id): for element_id in await redis.zrangebyscore(self.get_change_id_cache_key(), min=change_id):
if element_id.startswith(b'_config'): if element_id.startswith(b'_config'):
continue continue
@ -335,9 +335,9 @@ class MemmoryCacheProvider(BaseCacheProvider):
self.clear_cache() self.clear_cache()
def clear_cache(self) -> None: def clear_cache(self) -> None:
self.full_data = {} # type: Dict[str, str] self.full_data: Dict[str, str] = {}
self.restricted_data = {} # type: Dict[int, Dict[str, str]] self.restricted_data: Dict[int, Dict[str, str]] = {}
self.change_id_data = {} # type: Dict[int, Set[str]] self.change_id_data: Dict[int, Set[str]] = {}
async def reset_full_cache(self, data: Dict[str, str]) -> None: async def reset_full_cache(self, data: Dict[str, str]) -> None:
self.full_data = data self.full_data = data
@ -392,8 +392,8 @@ class MemmoryCacheProvider(BaseCacheProvider):
async def get_data_since( async def get_data_since(
self, change_id: int, user_id: Optional[int] = None) -> Tuple[Dict[str, List[bytes]], List[str]]: self, change_id: int, user_id: Optional[int] = None) -> Tuple[Dict[str, List[bytes]], List[str]]:
changed_elements = defaultdict(list) # type: Dict[str, List[bytes]] changed_elements: Dict[str, List[bytes]] = defaultdict(list)
deleted_elements = [] # type: List[str] deleted_elements: List[str] = []
if user_id is None: if user_id is None:
cache_dict = self.full_data cache_dict = self.full_data
else: else:
@ -495,7 +495,7 @@ def get_all_cachables() -> List[Cachable]:
""" """
Returns all element of OpenSlides. Returns all element of OpenSlides.
""" """
out = [] # type: List[Cachable] out: List[Cachable] = []
for app in apps.get_app_configs(): for app in apps.get_app_configs():
try: try:
# Get the method get_startup_elements() from an app. # Get the method get_startup_elements() from an app.

View File

@ -21,7 +21,7 @@ from .cache_providers import Cachable
if TYPE_CHECKING: if TYPE_CHECKING:
from .access_permissions import BaseAccessPermissions # noqa from .access_permissions import BaseAccessPermissions
AutoupdateFormat = TypedDict( AutoupdateFormat = TypedDict(
@ -200,7 +200,7 @@ class CollectionElement:
if self.full_data is None: if self.full_data is None:
if self.instance is None: if self.instance is None:
# The type of data has to be set for mypy # The type of data has to be set for mypy
data = None # type: Optional[Dict[str, Any]] data: Optional[Dict[str, Any]] = None
if getattr(settings, 'SKIP_CACHE', False): if getattr(settings, 'SKIP_CACHE', False):
# Hack for django 2.0 and channels 2.1 to stay in the same thread. # Hack for django 2.0 and channels 2.1 to stay in the same thread.
# This is needed for the tests. # This is needed for the tests.
@ -277,7 +277,7 @@ class Collection(Cachable):
""" """
if self.full_data is None: if self.full_data is None:
# The type of all_full_data has to be set for mypy # The type of all_full_data has to be set for mypy
all_full_data = {} # type: Dict[str, List[Dict[str, Any]]] all_full_data: Dict[str, List[Dict[str, Any]]] = {}
if getattr(settings, 'SKIP_CACHE', False): if getattr(settings, 'SKIP_CACHE', False):
# Hack for django 2.0 and channels 2.1 to stay in the same thread. # Hack for django 2.0 and channels 2.1 to stay in the same thread.
# This is needed for the tests. # This is needed for the tests.
@ -316,7 +316,7 @@ class Collection(Cachable):
return self.get_model().get_access_permissions().get_restricted_data(user, elements) return self.get_model().get_access_permissions().get_restricted_data(user, elements)
_models_to_collection_string = {} # type: Dict[str, Type[Model]] _models_to_collection_string: Dict[str, Type[Model]] = {}
def get_model_from_collection_string(collection_string: str) -> Type[Model]: def get_model_from_collection_string(collection_string: str) -> Type[Model]:

View File

@ -8,7 +8,6 @@ from ..core.config import config
from ..core.models import Projector from ..core.models import Projector
from .auth import async_anonymous_is_enabled, has_perm from .auth import async_anonymous_is_enabled, has_perm
from .cache import element_cache, split_element_id from .cache import element_cache, split_element_id
from .collection import AutoupdateFormat # noqa
from .collection import ( from .collection import (
Collection, Collection,
CollectionElement, CollectionElement,
@ -236,7 +235,7 @@ def projector_startup_data(projector_id: int) -> Any:
projector = Projector.objects.get(pk=config['projector_broadcast']) projector = Projector.objects.get(pk=config['projector_broadcast'])
# Collect all elements that are on the projector. # Collect all elements that are on the projector.
output = [] # type: List[AutoupdateFormat] output = []
for requirement in projector.get_all_requirements(): for requirement in projector.get_all_requirements():
required_collection_element = CollectionElement.from_instance(requirement) required_collection_element = CollectionElement.from_instance(requirement)
output.append(required_collection_element.as_autoupdate_for_projector()) output.append(required_collection_element.as_autoupdate_for_projector())

View File

@ -9,7 +9,7 @@ from .utils import convert_camel_case_to_pseudo_snake_case
if TYPE_CHECKING: if TYPE_CHECKING:
# Dummy import Collection for mypy, can be fixed with python 3.7 # Dummy import Collection for mypy, can be fixed with python 3.7
from .collection import Collection, CollectionElement # noqa from .collection import CollectionElement # noqa
class MinMaxIntegerField(models.IntegerField): class MinMaxIntegerField(models.IntegerField):
@ -32,7 +32,7 @@ class RESTModelMixin:
Mixin for Django models which are used in our REST API. Mixin for Django models which are used in our REST API.
""" """
access_permissions = None # type: BaseAccessPermissions access_permissions: Optional[BaseAccessPermissions] = None
def get_root_rest_element(self) -> models.Model: def get_root_rest_element(self) -> models.Model:
""" """

View File

@ -146,7 +146,7 @@ def get_all_plugin_urlpatterns() -> List[Any]:
Helper function to return all urlpatterns of all plugins listed in Helper function to return all urlpatterns of all plugins listed in
settings.INSTALLED_PLUGINS. settings.INSTALLED_PLUGINS.
""" """
urlpatterns = [] # type: List[Any] urlpatterns: List[Any] = []
for plugin in settings.INSTALLED_PLUGINS: for plugin in settings.INSTALLED_PLUGINS:
plugin_urlpatterns = get_plugin_urlpatterns(plugin) plugin_urlpatterns = get_plugin_urlpatterns(plugin)
if plugin_urlpatterns: if plugin_urlpatterns:

View File

@ -1,4 +1,4 @@
from typing import Any, Dict, Generator, Iterable, List, Type from typing import Any, Dict, Generator, Iterable, List, Optional, Type
from .collection import CollectionElement from .collection import CollectionElement
@ -11,7 +11,7 @@ class ProjectorElement:
subclassing from this base class with different names. The name attribute subclassing from this base class with different names. The name attribute
has to be set. has to be set.
""" """
name = None # type: str name: Optional[str] = None
def check_and_update_data(self, projector_object: Any, config_entry: Any) -> Any: def check_and_update_data(self, projector_object: Any, config_entry: Any) -> Any:
""" """
@ -84,7 +84,7 @@ class ProjectorElement:
return output return output
projector_elements = {} # type: Dict[str, ProjectorElement] projector_elements: Dict[str, ProjectorElement] = {}
def register_projector_elements(elements: Generator[Type[ProjectorElement], None, None]) -> None: def register_projector_elements(elements: Generator[Type[ProjectorElement], None, None]) -> None:
@ -95,7 +95,7 @@ def register_projector_elements(elements: Generator[Type[ProjectorElement], None
""" """
for Element in elements: for Element in elements:
element = Element() element = Element()
projector_elements[element.name] = element projector_elements[element.name] = element # type: ignore
def get_all_projector_elements() -> Dict[str, ProjectorElement]: def get_all_projector_elements() -> Dict[str, ProjectorElement]:

View File

@ -1,11 +1,11 @@
from collections import OrderedDict from collections import OrderedDict
from typing import Any, Dict, Iterable, Optional, Type # noqa from typing import Any, Dict, Iterable, Optional, Type
from django.http import Http404 from django.http import Http404
from rest_framework import status # noqa from rest_framework import status
from rest_framework.decorators import detail_route, list_route # noqa from rest_framework.decorators import detail_route, list_route
from rest_framework.metadata import SimpleMetadata # noqa from rest_framework.metadata import SimpleMetadata
from rest_framework.mixins import ( # noqa from rest_framework.mixins import (
CreateModelMixin, CreateModelMixin,
DestroyModelMixin, DestroyModelMixin,
ListModelMixin as _ListModelMixin, ListModelMixin as _ListModelMixin,
@ -15,7 +15,7 @@ from rest_framework.mixins import ( # noqa
from rest_framework.relations import MANY_RELATION_KWARGS from rest_framework.relations import MANY_RELATION_KWARGS
from rest_framework.response import Response from rest_framework.response import Response
from rest_framework.routers import DefaultRouter from rest_framework.routers import DefaultRouter
from rest_framework.serializers import ( # noqa from rest_framework.serializers import (
CharField, CharField,
DictField, DictField,
Field, Field,
@ -32,7 +32,7 @@ from rest_framework.serializers import ( # noqa
SerializerMethodField, SerializerMethodField,
ValidationError, ValidationError,
) )
from rest_framework.viewsets import ( # noqa from rest_framework.viewsets import (
GenericViewSet as _GenericViewSet, GenericViewSet as _GenericViewSet,
ModelViewSet as _ModelViewSet, ModelViewSet as _ModelViewSet,
ViewSet as _ViewSet, ViewSet as _ViewSet,
@ -43,6 +43,12 @@ from .auth import user_to_collection_user
from .collection import Collection, CollectionElement from .collection import Collection, CollectionElement
__all__ = ['status', 'detail_route', 'list_route', 'SimpleMetadata', 'CreateModelMixin',
'DestroyModelMixin', 'UpdateModelMixin', 'CharField', 'DictField', 'FileField',
'IntegerField', 'JSONField', 'ListField', 'ListSerializer', 'RelatedField',
'SerializerMethodField', 'ValidationError']
router = DefaultRouter() router = DefaultRouter()
@ -112,7 +118,7 @@ class PermissionMixin:
Also connects container to handle access permissions for model and Also connects container to handle access permissions for model and
viewset. viewset.
""" """
access_permissions = None # type: Optional[BaseAccessPermissions] access_permissions: Optional[BaseAccessPermissions] = None
def get_permissions(self) -> Iterable[str]: def get_permissions(self) -> Iterable[str]:
""" """
@ -165,7 +171,7 @@ class ModelSerializer(_ModelSerializer):
""" """
Returns all fields of the serializer. Returns all fields of the serializer.
""" """
fields = OrderedDict() # type: Dict[str, Field] fields: Dict[str, Field] = OrderedDict()
for field_name, field in super().get_fields().items(): for field_name, field in super().get_fields().items():
try: try:

View File

@ -6,7 +6,7 @@ import roman
if TYPE_CHECKING: if TYPE_CHECKING:
# Dummy import Collection for mypy, can be fixed with python 3.7 # Dummy import Collection for mypy, can be fixed with python 3.7
from .collection import Collection, CollectionElement # noqa from .collection import CollectionElement # noqa
CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_1 = re.compile('(.)([A-Z][a-z]+)') CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_1 = re.compile('(.)([A-Z][a-z]+)')
CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_2 = re.compile('([a-z0-9])([A-Z])') CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_2 = re.compile('([a-z0-9])([A-Z])')

View File

@ -1,5 +1,5 @@
import base64 import base64
from typing import Any, Dict, List # noqa from typing import Any, Dict, List, Optional
from django.contrib.staticfiles import finders from django.contrib.staticfiles import finders
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
@ -28,7 +28,7 @@ class APIView(_APIView):
The Django Rest framework APIView with improvements for OpenSlides. The Django Rest framework APIView with improvements for OpenSlides.
""" """
http_method_names = [] # type: List[str] http_method_names: List[str] = []
""" """
The allowed actions have to be explicitly defined. The allowed actions have to be explicitly defined.
@ -60,8 +60,8 @@ class TemplateView(View):
is not allowed to change. So the State has to be saved in this dict. Search for 'Borg design is not allowed to change. So the State has to be saved in this dict. Search for 'Borg design
pattern' for more information. pattern' for more information.
""" """
template_name = None # type: str template_name: Optional[str] = None
state = {} # type: Dict[str, str] state: Dict[str, str] = {}
def __init__(self, *args: Any, **kwargs: Any) -> None: def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@ -78,7 +78,7 @@ class TemplateView(View):
return template.read() return template.read()
def get(self, *args: Any, **kwargs: Any) -> HttpResponse: def get(self, *args: Any, **kwargs: Any) -> HttpResponse:
return HttpResponse(self.state[self.template_name]) return HttpResponse(self.state[self.template_name]) # type: ignore
class BinaryTemplateView(TemplateView): class BinaryTemplateView(TemplateView):

View File

@ -8,6 +8,7 @@ coverage
git+https://gitlab.com/pycqa/flake8.git git+https://gitlab.com/pycqa/flake8.git
isort isort
mypy mypy
pytest>=3.7.2
pytest-django pytest-django
pytest-asyncio pytest-asyncio
pytest-cov pytest-cov

View File

@ -33,7 +33,6 @@ setup(
'Framework :: Django', 'Framework :: Django',
'License :: OSI Approved :: MIT License', 'License :: OSI Approved :: MIT License',
'Operating System :: OS Independent', 'Operating System :: OS Independent',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7', ], 'Programming Language :: Python :: 3.7', ],
packages=find_packages(exclude=['tests', 'tests.*']), packages=find_packages(exclude=['tests', 'tests.*']),

View File

@ -43,17 +43,10 @@ def prepare_element_cache(settings):
@pytest.fixture @pytest.fixture
def communicator(request, event_loop): async def communicator(request, event_loop):
communicator = WebsocketCommunicator(application, "/ws/site/") communicator = WebsocketCommunicator(application, "/ws/site/")
yield communicator
# This style is needed for python 3.5. Use the generaor style when 3.5 ist dropped await communicator.disconnect()
def fin():
async def afin():
await communicator.disconnect()
event_loop.run_until_complete(afin())
request.addfinalizer(fin)
return communicator
@pytest.mark.asyncio @pytest.mark.asyncio

View File

@ -1,12 +1,12 @@
import asyncio # noqa import asyncio
from typing import Any, Callable, Dict, List, Optional from typing import Any, Callable, Dict, List, Optional
from openslides.utils.cache_providers import Cachable, MemmoryCacheProvider from openslides.utils.cache_providers import Cachable, MemmoryCacheProvider
from openslides.utils.collection import CollectionElement # noqa from openslides.utils.collection import CollectionElement
def restrict_elements( def restrict_elements(
user: Optional['CollectionElement'], user: Optional[CollectionElement],
elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
""" """
Adds the prefix 'restricted_' to all values except id. Adds the prefix 'restricted_' to all values except id.
@ -32,7 +32,7 @@ class Collection1(Cachable):
{'id': 1, 'value': 'value1'}, {'id': 1, 'value': 'value1'},
{'id': 2, 'value': 'value2'}] {'id': 2, 'value': 'value2'}]
def restrict_elements(self, user: Optional['CollectionElement'], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: def restrict_elements(self, user: Optional[CollectionElement], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
return restrict_elements(user, elements) return restrict_elements(user, elements)
@ -45,7 +45,7 @@ class Collection2(Cachable):
{'id': 1, 'key': 'value1'}, {'id': 1, 'key': 'value1'},
{'id': 2, 'key': 'value2'}] {'id': 2, 'key': 'value2'}]
def restrict_elements(self, user: Optional['CollectionElement'], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: def restrict_elements(self, user: Optional[CollectionElement], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
return restrict_elements(user, elements) return restrict_elements(user, elements)

View File

@ -397,7 +397,7 @@ async def test_update_restricted_data_second_worker_on_same_server(element_cache
""" """
element_cache.use_restricted_data_cache = True element_cache.use_restricted_data_cache = True
element_cache.cache_provider.restricted_data = {0: {}} element_cache.cache_provider.restricted_data = {0: {}}
future = asyncio.Future() # type: asyncio.Future future: asyncio.Future = asyncio.Future()
element_cache.restricted_data_cache_updater[0] = future element_cache.restricted_data_cache_updater[0] = future
await element_cache.cache_provider.set_lock_restricted_data(0) await element_cache.cache_provider.set_lock_restricted_data(0)
await element_cache.cache_provider.del_lock_restricted_data_after_wait(0, future) await element_cache.cache_provider.del_lock_restricted_data_after_wait(0, future)