From aac9dcabf5853bc90e2684917ee2289694955d5c Mon Sep 17 00:00:00 2001 From: Oskar Hahn Date: Wed, 22 Aug 2018 22:00:08 +0200 Subject: [PATCH] drop python 3.5 --- .travis.yml | 1 - CHANGELOG.rst | 3 ++- DEVELOPMENT.rst | 4 ++-- Dockerfile | 2 +- README.rst | 2 +- openslides/__main__.py | 4 ++-- openslides/agenda/access_permissions.py | 4 ++-- openslides/agenda/models.py | 6 ++--- openslides/agenda/signals.py | 4 ++-- openslides/assignments/access_permissions.py | 2 +- openslides/assignments/apps.py | 8 +++---- openslides/assignments/models.py | 10 ++++---- openslides/assignments/signals.py | 4 ++-- openslides/core/apps.py | 4 ++-- openslides/core/config.py | 4 ++-- openslides/core/views.py | 14 ++++++------ openslides/mediafiles/access_permissions.py | 2 +- openslides/motions/access_permissions.py | 2 +- openslides/motions/models.py | 6 ++--- openslides/motions/serializers.py | 4 ++-- openslides/motions/signals.py | 4 ++-- openslides/motions/views.py | 6 ++--- openslides/poll/models.py | 4 ++-- openslides/topics/models.py | 4 ++-- openslides/users/access_permissions.py | 4 ++-- openslides/users/views.py | 4 ++-- openslides/utils/arguments.py | 4 ++-- openslides/utils/autoupdate.py | 10 ++++---- openslides/utils/cache.py | 10 ++++---- openslides/utils/cache_providers.py | 20 ++++++++-------- openslides/utils/collection.py | 8 +++---- openslides/utils/consumers.py | 3 +-- openslides/utils/models.py | 4 ++-- openslides/utils/plugins.py | 2 +- openslides/utils/projector.py | 8 +++---- openslides/utils/rest_api.py | 24 ++++++++++++-------- openslides/utils/utils.py | 2 +- openslides/utils/views.py | 10 ++++---- requirements.txt | 1 + setup.py | 1 - tests/integration/utils/test_consumers.py | 13 +++-------- tests/unit/utils/cache_provider.py | 10 ++++---- tests/unit/utils/test_cache.py | 2 +- 43 files changed, 123 insertions(+), 125 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed4d13386..4474118d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ cache: pip: true yarn: true python: - - "3.5" - "3.6" - "3.7" env: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa6b8d022..b9cb67c8e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,7 +9,8 @@ Version 3.0 (unreleased) Core: - Change URL schema [#3798]. - - Update to channels2 + - Update to channels2 [#3796]. + - Drop Python 3.5 support[#3805]. Version 2.3 (unreleased) diff --git a/DEVELOPMENT.rst b/DEVELOPMENT.rst index c178e5d81..1f7f97a54 100644 --- a/DEVELOPMENT.rst +++ b/DEVELOPMENT.rst @@ -15,7 +15,7 @@ Installation and start of the development version a. Check requirements ''''''''''''''''''''' -Make sure that you have installed `Python (>= 3.5) `_, +Make sure that you have installed `Python (>= 3.6) `_, `Node.js (>=4.x) `_, `Yarn `_ and `Git `_ on your system. You also need build-essential packages and header files and a static library for Python. @@ -99,7 +99,7 @@ Use gulp watch in a second command-line interface:: Follow the instructions above (Installation on GNU/Linux or Mac OS X) but care of the following variations. -To get Python download and run the latest `Python 3.5 32-bit (x86) executable +To get Python download and run the latest `Python 3.7 32-bit (x86) executable installer `_. Note that the 32-bit installer is required even on a 64-bit Windows system. If you use the 64-bit installer, step d. of the instruction might fail unless you installed some diff --git a/Dockerfile b/Dockerfile index 2609d37b6..4e5d0d89e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.5 +FROM python:3.7 RUN apt-get -y update && apt-get -y upgrade RUN apt-get install -y libpq-dev supervisor curl vim RUN useradd -m openslides diff --git a/README.rst b/README.rst index 7719accc8..02c27ce27 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ Installation a. Check requirements ''''''''''''''''''''' -Make sure that you have installed `Python (>= 3.5) `_ +Make sure that you have installed `Python (>= 3.6) `_ on your system. Additional you need build-essential packages, header files and a static diff --git a/openslides/__main__.py b/openslides/__main__.py index 698a19cca..56b572d69 100644 --- a/openslides/__main__.py +++ b/openslides/__main__.py @@ -2,7 +2,7 @@ import os import sys -from typing import Dict # noqa +from typing import Dict import django from django.core.management import call_command, execute_from_command_line @@ -239,7 +239,7 @@ def createsettings(args): """ settings_dir = args.settings_dir local_installation = is_local_installation() - context = {} # type: Dict[str, str] + context: Dict[str, str] = {} if local_installation: if settings_dir is None: diff --git a/openslides/agenda/access_permissions.py b/openslides/agenda/access_permissions.py index 89a47a3b7..a90f73002 100644 --- a/openslides/agenda/access_permissions.py +++ b/openslides/agenda/access_permissions.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Iterable, List, Optional # noqa +from typing import Any, Dict, Iterable, List, Optional from ..utils.access_permissions import BaseAccessPermissions from ..utils.auth import has_perm @@ -72,7 +72,7 @@ class ItemAccessPermissions(BaseAccessPermissions): # In non internal case managers see everything and non managers see # everything but comments. if has_perm(user, 'agenda.can_manage'): - blocked_keys_non_internal_hidden_case = [] # type: Iterable[str] + blocked_keys_non_internal_hidden_case: Iterable[str] = [] can_see_hidden = True else: blocked_keys_non_internal_hidden_case = ('comment',) diff --git a/openslides/agenda/models.py b/openslides/agenda/models.py index 8abe6d614..a6d5a161d 100644 --- a/openslides/agenda/models.py +++ b/openslides/agenda/models.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Dict, List, Set # noqa +from typing import Dict, List, Set from django.conf import settings from django.contrib.auth.models import AnonymousUser @@ -65,7 +65,7 @@ class ItemManager(models.Manager): all of their children. """ queryset = self.order_by('weight') - item_children = defaultdict(list) # type: Dict[int, List[Item]] + item_children: Dict[int, List[Item]] = defaultdict(list) root_items = [] for item in queryset: if only_item_type is not None and item.type != only_item_type: @@ -121,7 +121,7 @@ class ItemManager(models.Manager): yield (element['id'], parent, weight) yield from walk_items(element.get('children', []), element['id']) - touched_items = set() # type: Set[int] + touched_items: Set[int] = set() db_items = dict((item.pk, item) for item in Item.objects.all()) for item_id, parent_id, weight in walk_items(tree): # Check that the item is only once in the tree to prevent invalid trees diff --git a/openslides/agenda/signals.py b/openslides/agenda/signals.py index 1b3e2e787..36208d139 100644 --- a/openslides/agenda/signals.py +++ b/openslides/agenda/signals.py @@ -1,4 +1,4 @@ -from typing import Set # noqa +from typing import Set from django.apps import apps from django.contrib.contenttypes.models import ContentType @@ -72,7 +72,7 @@ def required_users(sender, request_user, **kwargs): if request_user can see the agenda. This function may return an empty set. """ - speakers = set() # type: Set[int] + speakers: Set[int] = set() if has_perm(request_user, 'agenda.can_see'): for item_collection_element in Collection(Item.get_collection_string()).element_generator(): full_data = item_collection_element.get_full_data() diff --git a/openslides/assignments/access_permissions.py b/openslides/assignments/access_permissions.py index c3dac3029..7c0f883e7 100644 --- a/openslides/assignments/access_permissions.py +++ b/openslides/assignments/access_permissions.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional -from ..utils.access_permissions import BaseAccessPermissions # noqa +from ..utils.access_permissions import BaseAccessPermissions from ..utils.auth import has_perm from ..utils.collection import CollectionElement diff --git a/openslides/assignments/apps.py b/openslides/assignments/apps.py index 04e5398b6..984776b40 100644 --- a/openslides/assignments/apps.py +++ b/openslides/assignments/apps.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Union # noqa +from typing import List from django.apps import AppConfig from mypy_extensions import TypedDict @@ -48,10 +48,10 @@ class AssignmentsAppConfig(AppConfig): def get_angular_constants(self): assignment = self.get_model('Assignment') InnerItem = TypedDict('InnerItem', {'value': int, 'display_name': str}) - Item = TypedDict('Item', {'name': str, 'value': List[InnerItem]}) # noqa - data = { + Item = TypedDict('Item', {'name': str, 'value': List[InnerItem]}) + data: Item = { 'name': 'AssignmentPhases', - 'value': []} # type: Item + 'value': []} for phase in assignment.PHASES: data['value'].append({ 'value': phase[0], diff --git a/openslides/assignments/models.py b/openslides/assignments/models.py index f3eabc666..35b8b03d3 100644 --- a/openslides/assignments/models.py +++ b/openslides/assignments/models.py @@ -1,5 +1,5 @@ from collections import OrderedDict -from typing import Any, Dict, List, Optional # noqa +from typing import Any, Dict, List from django.conf import settings from django.contrib.contenttypes.fields import GenericRelation @@ -301,14 +301,14 @@ class Assignment(RESTModelMixin, models.Model): Returns a table represented as a list with all candidates from all related polls and their vote results. """ - vote_results_dict = OrderedDict() # type: Dict[Any, List[AssignmentVote]] + vote_results_dict: Dict[Any, List[AssignmentVote]] = OrderedDict() polls = self.polls.all() if only_published: polls = polls.filter(published=True) # All PollOption-Objects related to this assignment - options = [] # type: List[AssignmentOption] + options: List[AssignmentOption] = [] for poll in polls: options += poll.get_options() @@ -318,7 +318,7 @@ class Assignment(RESTModelMixin, models.Model): continue vote_results_dict[candidate] = [] for poll in polls: - votes = {} # type: Any + votes: Any = {} try: # candidate related to this poll poll_option = poll.get_options().get(candidate=candidate) @@ -333,7 +333,7 @@ class Assignment(RESTModelMixin, models.Model): """ Container for runtime information for agenda app (on create or update of this instance). """ - agenda_item_update_information = {} # type: Dict[str, Any] + agenda_item_update_information: Dict[str, Any] = {} def get_agenda_title(self): return str(self) diff --git a/openslides/assignments/signals.py b/openslides/assignments/signals.py index d36589243..48bc9cdb8 100644 --- a/openslides/assignments/signals.py +++ b/openslides/assignments/signals.py @@ -1,4 +1,4 @@ -from typing import Any, Set # noqa +from typing import Any, Set from django.apps import apps @@ -24,7 +24,7 @@ def required_users(sender, request_user, **kwargs): options) in any assignment if request_user can see assignments. This function may return an empty set. """ - candidates = set() # type: Set[Any] # TODO: Replace Any + candidates: Set[Any] = set() if has_perm(request_user, 'assignments.can_see'): for assignment_collection_element in Collection(Assignment.get_collection_string()).element_generator(): full_data = assignment_collection_element.get_full_data() diff --git a/openslides/core/apps.py b/openslides/core/apps.py index da4a8990e..0c342ea4a 100644 --- a/openslides/core/apps.py +++ b/openslides/core/apps.py @@ -1,6 +1,6 @@ from collections import OrderedDict from operator import attrgetter -from typing import Any, List # noqa +from typing import Any, List from django.apps import AppConfig from django.conf import settings @@ -93,7 +93,7 @@ class CoreAppConfig(AppConfig): 'value': client_settings_dict} # Config variables - config_groups = [] # type: List[Any] # TODO: Replace Any by correct type + config_groups: List[Any] = [] for config_variable in sorted(config.config_variables.values(), key=attrgetter('weight')): if config_variable.is_hidden(): # Skip hidden config variables. Do not even check groups and subgroups. diff --git a/openslides/core/config.py b/openslides/core/config.py index 6783ee056..bf28fae1d 100644 --- a/openslides/core/config.py +++ b/openslides/core/config.py @@ -46,10 +46,10 @@ class ConfigHandler: def __init__(self) -> None: # Dict, that keeps all ConfigVariable objects. Has to be set at statup. # See the ready() method in openslides.core.apps. - self.config_variables = {} # type: Dict[str, ConfigVariable] + self.config_variables: Dict[str, ConfigVariable] = {} # Index to get the database id from a given config key - self.key_to_id = None # type: Optional[Dict[str, int]] + self.key_to_id: Optional[Dict[str, int]] = None def __getitem__(self, key: str) -> Any: """ diff --git a/openslides/core/views.py b/openslides/core/views.py index dfaa981f9..23264b37b 100644 --- a/openslides/core/views.py +++ b/openslides/core/views.py @@ -1,7 +1,7 @@ import json import uuid from textwrap import dedent -from typing import Any, Dict, List, cast # noqa +from typing import Any, Dict, List, cast from django.apps import apps from django.conf import settings @@ -91,7 +91,7 @@ class WebclientJavaScriptView(utils_views.View): AngularJS app for the requested realm (site or projector). Also code for plugins is appended. The result is not uglified. """ - cache = {} # type: Dict[str, str] + cache: Dict[str, str] = {} def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -102,8 +102,8 @@ class WebclientJavaScriptView(utils_views.View): self.init_cache('projector') def init_cache(self, realm: str) -> None: - angular_modules = [] # type: List[str] - js_files = [] # type: List[str] + angular_modules: List[str] = [] + js_files: List[str] = [] for app_config in apps.get_app_configs(): # Add the angular app if the module has one. if getattr(app_config, 'angular_{}_module'.format(realm), False): @@ -836,16 +836,16 @@ class VersionView(utils_views.APIView): http_method_names = ['get'] def get_context_data(self, **context): - Result = TypedDict('Result', { # noqa + Result = TypedDict('Result', { 'openslides_version': str, 'openslides_license': str, 'openslides_url': str, 'plugins': List[Dict[str, str]]}) - result = dict( + result: Result = dict( openslides_version=version, openslides_license=license, openslides_url=url, - plugins=[]) # type: Result + plugins=[]) # Versions of plugins. for plugin in settings.INSTALLED_PLUGINS: result['plugins'].append({ diff --git a/openslides/mediafiles/access_permissions.py b/openslides/mediafiles/access_permissions.py index ab23be913..93c84c8b4 100644 --- a/openslides/mediafiles/access_permissions.py +++ b/openslides/mediafiles/access_permissions.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional -from ..utils.access_permissions import BaseAccessPermissions # noqa +from ..utils.access_permissions import BaseAccessPermissions from ..utils.auth import has_perm from ..utils.collection import CollectionElement diff --git a/openslides/motions/access_permissions.py b/openslides/motions/access_permissions.py index 57ccba291..37c629358 100644 --- a/openslides/motions/access_permissions.py +++ b/openslides/motions/access_permissions.py @@ -2,7 +2,7 @@ from copy import deepcopy from typing import Any, Dict, List, Optional from ..core.config import config -from ..utils.access_permissions import BaseAccessPermissions # noqa +from ..utils.access_permissions import BaseAccessPermissions from ..utils.auth import has_perm from ..utils.collection import CollectionElement diff --git a/openslides/motions/models.py b/openslides/motions/models.py index c2b0f2e8c..21d0664a4 100644 --- a/openslides/motions/models.py +++ b/openslides/motions/models.py @@ -1,4 +1,4 @@ -from typing import Any, Dict # noqa +from typing import Any, Dict from django.conf import settings from django.contrib.auth.models import AnonymousUser @@ -692,7 +692,7 @@ class Motion(RESTModelMixin, models.Model): """ Container for runtime information for agenda app (on create or update of this instance). """ - agenda_item_update_information = {} # type: Dict[str, Any] + agenda_item_update_information: Dict[str, Any] = {} def get_agenda_title(self): """ @@ -1052,7 +1052,7 @@ class MotionBlock(RESTModelMixin, models.Model): """ Container for runtime information for agenda app (on create or update of this instance). """ - agenda_item_update_information = {} # type: Dict[str, Any] + agenda_item_update_information: Dict[str, Any] = {} @property def agenda_item(self): diff --git a/openslides/motions/serializers.py b/openslides/motions/serializers.py index 6f2791bc2..27c2fca43 100644 --- a/openslides/motions/serializers.py +++ b/openslides/motions/serializers.py @@ -1,4 +1,4 @@ -from typing import Dict # noqa +from typing import Dict from django.db import transaction from django.utils.translation import ugettext as _ @@ -233,7 +233,7 @@ class MotionPollSerializer(ModelSerializer): def __init__(self, *args, **kwargs): # The following dictionary is just a cache for several votes. - self._votes_dicts = {} # type: Dict[int, Dict[int, int]] + self._votes_dicts: Dict[int, Dict[int, int]] = {} return super().__init__(*args, **kwargs) def get_yes(self, obj): diff --git a/openslides/motions/signals.py b/openslides/motions/signals.py index 4bc838522..41605faf6 100644 --- a/openslides/motions/signals.py +++ b/openslides/motions/signals.py @@ -1,4 +1,4 @@ -from typing import Set # noqa +from typing import Set from django.apps import apps from django.utils.translation import ugettext_noop @@ -126,7 +126,7 @@ def required_users(sender, request_user, **kwargs): any motion if request_user can see motions. This function may return an empty set. """ - submitters_supporters = set() # type: Set[int] + submitters_supporters: Set[int] = set() if has_perm(request_user, 'motions.can_see'): for motion_collection_element in Collection(Motion.get_collection_string()).element_generator(): full_data = motion_collection_element.get_full_data() diff --git a/openslides/motions/views.py b/openslides/motions/views.py index 13b9d6ba6..6b021963d 100644 --- a/openslides/motions/views.py +++ b/openslides/motions/views.py @@ -1,5 +1,5 @@ import re -from typing import Optional # noqa +from typing import Optional from django.conf import settings from django.contrib.auth import get_user_model @@ -113,9 +113,9 @@ class MotionViewSet(ModelViewSet): # Check if parent motion exists. if request.data.get('parent_id') is not None: try: - parent_motion = CollectionElement.from_values( + parent_motion: Optional[CollectionElement] = CollectionElement.from_values( Motion.get_collection_string(), - request.data['parent_id']) # type: Optional[CollectionElement] + request.data['parent_id']) except Motion.DoesNotExist: raise ValidationError({'detail': _('The parent motion does not exist.')}) else: diff --git a/openslides/poll/models.py b/openslides/poll/models.py index 545e3f111..051f94f8c 100644 --- a/openslides/poll/models.py +++ b/openslides/poll/models.py @@ -1,5 +1,5 @@ import locale -from typing import Type # noqa +from typing import Optional, Type from django.core.exceptions import ObjectDoesNotExist from django.db import models @@ -17,7 +17,7 @@ class BaseOption(models.Model): which has to be a subclass of BaseVote. Otherwise you have to override the get_vote_class method. """ - vote_class = None # type: Type[BaseVote] + vote_class: Optional[Type['BaseVote']] = None class Meta: abstract = True diff --git a/openslides/topics/models.py b/openslides/topics/models.py index 1f869304a..ecee8331f 100644 --- a/openslides/topics/models.py +++ b/openslides/topics/models.py @@ -1,4 +1,4 @@ -from typing import Any, Dict # noqa +from typing import Any, Dict from django.contrib.contenttypes.fields import GenericRelation from django.db import models @@ -59,7 +59,7 @@ class Topic(RESTModelMixin, models.Model): """ Container for runtime information for agenda app (on create or update of this instance). """ - agenda_item_update_information = {} # type: Dict[str, Any] + agenda_item_update_information: Dict[str, Any] = {} @property def agenda_item(self): diff --git a/openslides/users/access_permissions.py b/openslides/users/access_permissions.py index 545133b5a..3648ca991 100644 --- a/openslides/users/access_permissions.py +++ b/openslides/users/access_permissions.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from django.contrib.auth.models import AnonymousUser from ..core.signals import user_data_required -from ..utils.access_permissions import BaseAccessPermissions # noqa +from ..utils.access_permissions import BaseAccessPermissions from ..utils.auth import anonymous_is_enabled, has_perm from ..utils.collection import CollectionElement @@ -172,7 +172,7 @@ class PersonalNoteAccessPermissions(BaseAccessPermissions): """ # Parse data. if user is None: - data = [] # type: List[Dict[str, Any]] + data: List[Dict[str, Any]] = [] else: for full in full_data: if full['user_id'] == user.id: diff --git a/openslides/users/views.py b/openslides/users/views.py index d6993d1e4..e0aed75bf 100644 --- a/openslides/users/views.py +++ b/openslides/users/views.py @@ -1,5 +1,5 @@ import smtplib -from typing import List # noqa +from typing import List from asgiref.sync import async_to_sync from django.conf import settings @@ -325,7 +325,7 @@ class GroupViewSet(ModelViewSet): # Some permissions are added. if len(new_permissions) > 0: - collection_elements = [] # type: List[CollectionElement] + collection_elements: List[CollectionElement] = [] signal_results = permission_change.send(None, permissions=new_permissions, action='added') for receiver, signal_collections in signal_results: for cachable in signal_collections: diff --git a/openslides/utils/arguments.py b/openslides/utils/arguments.py index cfd0d99f3..74321fa46 100644 --- a/openslides/utils/arguments.py +++ b/openslides/utils/arguments.py @@ -1,9 +1,9 @@ from argparse import Namespace -from typing import Any, Union # noqa +from typing import Any, Optional class OpenSlidesArguments(): - args = None # type: Union[None, Namespace] + args: Optional[Namespace] = None def __getitem__(self, key: str) -> Any: if not self.args: diff --git a/openslides/utils/autoupdate.py b/openslides/utils/autoupdate.py index f68770c13..e0757ced1 100644 --- a/openslides/utils/autoupdate.py +++ b/openslides/utils/autoupdate.py @@ -16,7 +16,7 @@ def to_ordered_dict(d: Optional[Dict]) -> Optional[OrderedDict]: Little helper to hash information dict in inform_*_data. """ if isinstance(d, dict): - result = OrderedDict([(key, to_ordered_dict(d[key])) for key in sorted(d.keys())]) # type: Optional[OrderedDict] + result: Optional[OrderedDict] = OrderedDict([(key, to_ordered_dict(d[key])) for key in sorted(d.keys())]) else: result = d return result @@ -64,7 +64,7 @@ def inform_deleted_data(elements: Iterable[Tuple[str, int]], information: Dict[s The argument information is added to each collection element. """ - collection_elements = {} # type: Dict[str, Any] + collection_elements: Dict[str, Any] = {} for element in elements: collection_element = CollectionElement.from_values( collection_string=element[0], @@ -106,7 +106,7 @@ def inform_data_collection_element_list(collection_elements: List[CollectionElem """ Global container for autoupdate bundles """ -autoupdate_bundle = {} # type: Dict[int, Dict[str, CollectionElement]] +autoupdate_bundle: Dict[int, Dict[str, CollectionElement]] = {} class AutoupdateBundleMiddleware: @@ -123,7 +123,7 @@ class AutoupdateBundleMiddleware: response = self.get_response(request) - bundle = autoupdate_bundle.pop(thread_id) # type: Dict[str, CollectionElement] + bundle: Dict[str, CollectionElement] = autoupdate_bundle.pop(thread_id) async_to_sync(send_autoupdate)(bundle.values()) return response @@ -138,7 +138,7 @@ async def send_autoupdate(collection_elements: Iterable[CollectionElement]) -> N Does nothing if collection_elements is empty. """ if collection_elements: - cache_elements = {} # type: Dict[str, Optional[Dict[str, Any]]] + cache_elements: Dict[str, Optional[Dict[str, Any]]] = {} for element in collection_elements: element_id = get_element_id(element.collection_string, element.id) if element.is_deleted(): diff --git a/openslides/utils/cache.py b/openslides/utils/cache.py index ac2a162fd..178488936 100644 --- a/openslides/utils/cache.py +++ b/openslides/utils/cache.py @@ -73,7 +73,7 @@ class ElementCache: self.use_restricted_data_cache = use_restricted_data_cache self.cache_provider = cache_provider_class(redis) self.cachable_provider = cachable_provider - self._cachables = None # type: Optional[Dict[str, Cachable]] + self._cachables: Optional[Dict[str, Cachable]] = None # Start time is used as first change_id if there is non in redis if start_time is None: @@ -81,7 +81,7 @@ class ElementCache: self.start_time = start_time # Contains Futures to controll, that only one client updates the restricted_data. - self.restricted_data_cache_updater = {} # type: Dict[int, asyncio.Future] + self.restricted_data_cache_updater: Dict[int, asyncio.Future] = {} @property def cachables(self) -> Dict[str, Cachable]: @@ -109,7 +109,7 @@ class ElementCache: """ Build or rebuild the full_data cache. """ - db_data = {} # type: Dict[str, List[Dict[str, Any]]] + db_data = {} for collection_string, cachable in self.cachables.items(): db_data[collection_string] = await database_sync_to_async(cachable.get_elements)() await self.save_full_data(db_data) @@ -262,7 +262,7 @@ class ElementCache: # If this succeeds, there is noone else currently updating the cache. # TODO: Make a timeout. Else this could block forever if await self.cache_provider.set_lock_restricted_data(get_user_id(user)): - future = asyncio.Future() # type: asyncio.Future + future: asyncio.Future = asyncio.Future() self.restricted_data_cache_updater[get_user_id(user)] = future # Get change_id for this user value = await self.cache_provider.get_change_id_user(get_user_id(user)) @@ -318,7 +318,7 @@ class ElementCache: await self.update_restricted_data(user) - out = defaultdict(list) # type: Dict[str, List[Dict[str, Any]]] + out: Dict[str, List[Dict[str, Any]]] = defaultdict(list) restricted_data = await self.cache_provider.get_all_data(get_user_id(user)) for element_id, data in restricted_data.items(): if element_id.decode().startswith('_config'): diff --git a/openslides/utils/cache_providers.py b/openslides/utils/cache_providers.py index 1be8d867d..e2150d4d4 100644 --- a/openslides/utils/cache_providers.py +++ b/openslides/utils/cache_providers.py @@ -1,5 +1,4 @@ from collections import defaultdict -from typing import Set # noqa from typing import ( TYPE_CHECKING, Any, @@ -8,6 +7,7 @@ from typing import ( Iterable, List, Optional, + Set, Tuple, Union, ) @@ -108,7 +108,7 @@ class RedisCacheProvider(BaseCacheProvider): """ Cache provider that loads and saves the data to redis. """ - redis_pool = None # type: Optional[aioredis.RedisConnection] + redis_pool: Optional[aioredis.RedisConnection] = None def __init__(self, redis: str) -> None: self.redis_address = redis @@ -232,8 +232,8 @@ class RedisCacheProvider(BaseCacheProvider): """ # TODO: rewrite with lua to get all elements with one request redis = await self.get_connection() - changed_elements = defaultdict(list) # type: Dict[str, List[bytes]] - deleted_elements = [] # type: List[str] + changed_elements: Dict[str, List[bytes]] = defaultdict(list) + deleted_elements: List[str] = [] for element_id in await redis.zrangebyscore(self.get_change_id_cache_key(), min=change_id): if element_id.startswith(b'_config'): continue @@ -335,9 +335,9 @@ class MemmoryCacheProvider(BaseCacheProvider): self.clear_cache() def clear_cache(self) -> None: - self.full_data = {} # type: Dict[str, str] - self.restricted_data = {} # type: Dict[int, Dict[str, str]] - self.change_id_data = {} # type: Dict[int, Set[str]] + self.full_data: Dict[str, str] = {} + self.restricted_data: Dict[int, Dict[str, str]] = {} + self.change_id_data: Dict[int, Set[str]] = {} async def reset_full_cache(self, data: Dict[str, str]) -> None: self.full_data = data @@ -392,8 +392,8 @@ class MemmoryCacheProvider(BaseCacheProvider): async def get_data_since( self, change_id: int, user_id: Optional[int] = None) -> Tuple[Dict[str, List[bytes]], List[str]]: - changed_elements = defaultdict(list) # type: Dict[str, List[bytes]] - deleted_elements = [] # type: List[str] + changed_elements: Dict[str, List[bytes]] = defaultdict(list) + deleted_elements: List[str] = [] if user_id is None: cache_dict = self.full_data else: @@ -495,7 +495,7 @@ def get_all_cachables() -> List[Cachable]: """ Returns all element of OpenSlides. """ - out = [] # type: List[Cachable] + out: List[Cachable] = [] for app in apps.get_app_configs(): try: # Get the method get_startup_elements() from an app. diff --git a/openslides/utils/collection.py b/openslides/utils/collection.py index fc43430f6..d0555a3ea 100644 --- a/openslides/utils/collection.py +++ b/openslides/utils/collection.py @@ -21,7 +21,7 @@ from .cache_providers import Cachable if TYPE_CHECKING: - from .access_permissions import BaseAccessPermissions # noqa + from .access_permissions import BaseAccessPermissions AutoupdateFormat = TypedDict( @@ -200,7 +200,7 @@ class CollectionElement: if self.full_data is None: if self.instance is None: # The type of data has to be set for mypy - data = None # type: Optional[Dict[str, Any]] + data: Optional[Dict[str, Any]] = None if getattr(settings, 'SKIP_CACHE', False): # Hack for django 2.0 and channels 2.1 to stay in the same thread. # This is needed for the tests. @@ -277,7 +277,7 @@ class Collection(Cachable): """ if self.full_data is None: # The type of all_full_data has to be set for mypy - all_full_data = {} # type: Dict[str, List[Dict[str, Any]]] + all_full_data: Dict[str, List[Dict[str, Any]]] = {} if getattr(settings, 'SKIP_CACHE', False): # Hack for django 2.0 and channels 2.1 to stay in the same thread. # This is needed for the tests. @@ -316,7 +316,7 @@ class Collection(Cachable): return self.get_model().get_access_permissions().get_restricted_data(user, elements) -_models_to_collection_string = {} # type: Dict[str, Type[Model]] +_models_to_collection_string: Dict[str, Type[Model]] = {} def get_model_from_collection_string(collection_string: str) -> Type[Model]: diff --git a/openslides/utils/consumers.py b/openslides/utils/consumers.py index 820b1ad36..5f67bf86a 100644 --- a/openslides/utils/consumers.py +++ b/openslides/utils/consumers.py @@ -8,7 +8,6 @@ from ..core.config import config from ..core.models import Projector from .auth import async_anonymous_is_enabled, has_perm from .cache import element_cache, split_element_id -from .collection import AutoupdateFormat # noqa from .collection import ( Collection, CollectionElement, @@ -236,7 +235,7 @@ def projector_startup_data(projector_id: int) -> Any: projector = Projector.objects.get(pk=config['projector_broadcast']) # Collect all elements that are on the projector. - output = [] # type: List[AutoupdateFormat] + output = [] for requirement in projector.get_all_requirements(): required_collection_element = CollectionElement.from_instance(requirement) output.append(required_collection_element.as_autoupdate_for_projector()) diff --git a/openslides/utils/models.py b/openslides/utils/models.py index 08bc033fb..8cdb92396 100644 --- a/openslides/utils/models.py +++ b/openslides/utils/models.py @@ -9,7 +9,7 @@ from .utils import convert_camel_case_to_pseudo_snake_case if TYPE_CHECKING: # Dummy import Collection for mypy, can be fixed with python 3.7 - from .collection import Collection, CollectionElement # noqa + from .collection import CollectionElement # noqa class MinMaxIntegerField(models.IntegerField): @@ -32,7 +32,7 @@ class RESTModelMixin: Mixin for Django models which are used in our REST API. """ - access_permissions = None # type: BaseAccessPermissions + access_permissions: Optional[BaseAccessPermissions] = None def get_root_rest_element(self) -> models.Model: """ diff --git a/openslides/utils/plugins.py b/openslides/utils/plugins.py index 987d00b6f..6719415f4 100644 --- a/openslides/utils/plugins.py +++ b/openslides/utils/plugins.py @@ -146,7 +146,7 @@ def get_all_plugin_urlpatterns() -> List[Any]: Helper function to return all urlpatterns of all plugins listed in settings.INSTALLED_PLUGINS. """ - urlpatterns = [] # type: List[Any] + urlpatterns: List[Any] = [] for plugin in settings.INSTALLED_PLUGINS: plugin_urlpatterns = get_plugin_urlpatterns(plugin) if plugin_urlpatterns: diff --git a/openslides/utils/projector.py b/openslides/utils/projector.py index c46153d33..14db3d7da 100644 --- a/openslides/utils/projector.py +++ b/openslides/utils/projector.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Generator, Iterable, List, Type +from typing import Any, Dict, Generator, Iterable, List, Optional, Type from .collection import CollectionElement @@ -11,7 +11,7 @@ class ProjectorElement: subclassing from this base class with different names. The name attribute has to be set. """ - name = None # type: str + name: Optional[str] = None def check_and_update_data(self, projector_object: Any, config_entry: Any) -> Any: """ @@ -84,7 +84,7 @@ class ProjectorElement: return output -projector_elements = {} # type: Dict[str, ProjectorElement] +projector_elements: Dict[str, ProjectorElement] = {} def register_projector_elements(elements: Generator[Type[ProjectorElement], None, None]) -> None: @@ -95,7 +95,7 @@ def register_projector_elements(elements: Generator[Type[ProjectorElement], None """ for Element in elements: element = Element() - projector_elements[element.name] = element + projector_elements[element.name] = element # type: ignore def get_all_projector_elements() -> Dict[str, ProjectorElement]: diff --git a/openslides/utils/rest_api.py b/openslides/utils/rest_api.py index e8c40733a..6d42cfbd5 100644 --- a/openslides/utils/rest_api.py +++ b/openslides/utils/rest_api.py @@ -1,11 +1,11 @@ from collections import OrderedDict -from typing import Any, Dict, Iterable, Optional, Type # noqa +from typing import Any, Dict, Iterable, Optional, Type from django.http import Http404 -from rest_framework import status # noqa -from rest_framework.decorators import detail_route, list_route # noqa -from rest_framework.metadata import SimpleMetadata # noqa -from rest_framework.mixins import ( # noqa +from rest_framework import status +from rest_framework.decorators import detail_route, list_route +from rest_framework.metadata import SimpleMetadata +from rest_framework.mixins import ( CreateModelMixin, DestroyModelMixin, ListModelMixin as _ListModelMixin, @@ -15,7 +15,7 @@ from rest_framework.mixins import ( # noqa from rest_framework.relations import MANY_RELATION_KWARGS from rest_framework.response import Response from rest_framework.routers import DefaultRouter -from rest_framework.serializers import ( # noqa +from rest_framework.serializers import ( CharField, DictField, Field, @@ -32,7 +32,7 @@ from rest_framework.serializers import ( # noqa SerializerMethodField, ValidationError, ) -from rest_framework.viewsets import ( # noqa +from rest_framework.viewsets import ( GenericViewSet as _GenericViewSet, ModelViewSet as _ModelViewSet, ViewSet as _ViewSet, @@ -43,6 +43,12 @@ from .auth import user_to_collection_user from .collection import Collection, CollectionElement +__all__ = ['status', 'detail_route', 'list_route', 'SimpleMetadata', 'CreateModelMixin', + 'DestroyModelMixin', 'UpdateModelMixin', 'CharField', 'DictField', 'FileField', + 'IntegerField', 'JSONField', 'ListField', 'ListSerializer', 'RelatedField', + 'SerializerMethodField', 'ValidationError'] + + router = DefaultRouter() @@ -112,7 +118,7 @@ class PermissionMixin: Also connects container to handle access permissions for model and viewset. """ - access_permissions = None # type: Optional[BaseAccessPermissions] + access_permissions: Optional[BaseAccessPermissions] = None def get_permissions(self) -> Iterable[str]: """ @@ -165,7 +171,7 @@ class ModelSerializer(_ModelSerializer): """ Returns all fields of the serializer. """ - fields = OrderedDict() # type: Dict[str, Field] + fields: Dict[str, Field] = OrderedDict() for field_name, field in super().get_fields().items(): try: diff --git a/openslides/utils/utils.py b/openslides/utils/utils.py index 8fc4a6dd9..1c64c4ec5 100644 --- a/openslides/utils/utils.py +++ b/openslides/utils/utils.py @@ -6,7 +6,7 @@ import roman if TYPE_CHECKING: # Dummy import Collection for mypy, can be fixed with python 3.7 - from .collection import Collection, CollectionElement # noqa + from .collection import CollectionElement # noqa CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_1 = re.compile('(.)([A-Z][a-z]+)') CAMEL_CASE_TO_PSEUDO_SNAKE_CASE_CONVERSION_REGEX_2 = re.compile('([a-z0-9])([A-Z])') diff --git a/openslides/utils/views.py b/openslides/utils/views.py index dc8411ba5..02e4efd4f 100644 --- a/openslides/utils/views.py +++ b/openslides/utils/views.py @@ -1,5 +1,5 @@ import base64 -from typing import Any, Dict, List # noqa +from typing import Any, Dict, List, Optional from django.contrib.staticfiles import finders from django.core.exceptions import ImproperlyConfigured @@ -28,7 +28,7 @@ class APIView(_APIView): The Django Rest framework APIView with improvements for OpenSlides. """ - http_method_names = [] # type: List[str] + http_method_names: List[str] = [] """ The allowed actions have to be explicitly defined. @@ -60,8 +60,8 @@ class TemplateView(View): is not allowed to change. So the State has to be saved in this dict. Search for 'Borg design pattern' for more information. """ - template_name = None # type: str - state = {} # type: Dict[str, str] + template_name: Optional[str] = None + state: Dict[str, str] = {} def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -78,7 +78,7 @@ class TemplateView(View): return template.read() def get(self, *args: Any, **kwargs: Any) -> HttpResponse: - return HttpResponse(self.state[self.template_name]) + return HttpResponse(self.state[self.template_name]) # type: ignore class BinaryTemplateView(TemplateView): diff --git a/requirements.txt b/requirements.txt index cc82a1270..77450b5f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ coverage git+https://gitlab.com/pycqa/flake8.git isort mypy +pytest>=3.7.2 pytest-django pytest-asyncio pytest-cov diff --git a/setup.py b/setup.py index b769b8d49..9fd34dfe4 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ setup( 'Framework :: Django', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', ], packages=find_packages(exclude=['tests', 'tests.*']), diff --git a/tests/integration/utils/test_consumers.py b/tests/integration/utils/test_consumers.py index c510f79ab..5ae29ef4c 100644 --- a/tests/integration/utils/test_consumers.py +++ b/tests/integration/utils/test_consumers.py @@ -43,17 +43,10 @@ def prepare_element_cache(settings): @pytest.fixture -def communicator(request, event_loop): +async def communicator(request, event_loop): communicator = WebsocketCommunicator(application, "/ws/site/") - - # This style is needed for python 3.5. Use the generaor style when 3.5 ist dropped - def fin(): - async def afin(): - await communicator.disconnect() - event_loop.run_until_complete(afin()) - - request.addfinalizer(fin) - return communicator + yield communicator + await communicator.disconnect() @pytest.mark.asyncio diff --git a/tests/unit/utils/cache_provider.py b/tests/unit/utils/cache_provider.py index 3db52363b..49a81f091 100644 --- a/tests/unit/utils/cache_provider.py +++ b/tests/unit/utils/cache_provider.py @@ -1,12 +1,12 @@ -import asyncio # noqa +import asyncio from typing import Any, Callable, Dict, List, Optional from openslides.utils.cache_providers import Cachable, MemmoryCacheProvider -from openslides.utils.collection import CollectionElement # noqa +from openslides.utils.collection import CollectionElement def restrict_elements( - user: Optional['CollectionElement'], + user: Optional[CollectionElement], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ Adds the prefix 'restricted_' to all values except id. @@ -32,7 +32,7 @@ class Collection1(Cachable): {'id': 1, 'value': 'value1'}, {'id': 2, 'value': 'value2'}] - def restrict_elements(self, user: Optional['CollectionElement'], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def restrict_elements(self, user: Optional[CollectionElement], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: return restrict_elements(user, elements) @@ -45,7 +45,7 @@ class Collection2(Cachable): {'id': 1, 'key': 'value1'}, {'id': 2, 'key': 'value2'}] - def restrict_elements(self, user: Optional['CollectionElement'], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def restrict_elements(self, user: Optional[CollectionElement], elements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: return restrict_elements(user, elements) diff --git a/tests/unit/utils/test_cache.py b/tests/unit/utils/test_cache.py index aa1a902f1..9c7218b22 100644 --- a/tests/unit/utils/test_cache.py +++ b/tests/unit/utils/test_cache.py @@ -397,7 +397,7 @@ async def test_update_restricted_data_second_worker_on_same_server(element_cache """ element_cache.use_restricted_data_cache = True element_cache.cache_provider.restricted_data = {0: {}} - future = asyncio.Future() # type: asyncio.Future + future: asyncio.Future = asyncio.Future() element_cache.restricted_data_cache_updater[0] = future await element_cache.cache_provider.set_lock_restricted_data(0) await element_cache.cache_provider.del_lock_restricted_data_after_wait(0, future)