From e225a57f97b766a608376691d147ae3275d5472c Mon Sep 17 00:00:00 2001 From: FinnStutzenstein Date: Thu, 28 May 2020 11:40:41 +0200 Subject: [PATCH] OpenSlides3+: External Autoupdate Service - Removing channels. Going back to a wsgi deployment - Removed server projector code - Autoupdate throttling is now in the client - New communication stack in the client - Adopted all deployment methods: Docker stack and docker compose (prod and dev) - Added autoupdate service as submodule --- .github/workflows/run-tests.yml | 2 +- .gitignore | 9 +- .gitmodules | 3 + DEVELOPMENT.rst | 211 ++----- Makefile | 17 + README.rst | 2 + autoupdate | 1 + client/docker/Dockerfile | 4 +- client/docker/Dockerfile.dev | 11 + client/docker/nginx.conf | 23 - client/ngsw-config.json | 2 +- client/package.json | 1 + client/proxy.conf.json | 4 + client/src/app/app.component.ts | 8 +- .../autoupdate-throttle.service.ts | 164 +++++ .../core/core-services/autoupdate.service.ts | 150 ++--- .../communication-manager.service.ts | 186 ++++++ .../core/core-services/constants.service.ts | 33 +- .../core/core-services/data-store.service.ts | 7 +- .../app/core/core-services/http.service.ts | 29 +- .../app/core/core-services/notify.service.ts | 128 +++- .../offline-broadcast.service.ts | 37 ++ .../app/core/core-services/offline.service.ts | 151 +++-- .../core/core-services/openslides.service.ts | 88 ++- .../core/core-services/operator.service.ts | 11 +- .../core/core-services/ping.service.spec.ts | 17 - .../app/core/core-services/ping.service.ts | 86 --- .../core-services/prioritize.service.spec.ts | 17 - .../core/core-services/prioritize.service.ts | 46 -- .../core-services/projector-data.service.ts | 75 ++- .../core/core-services/projector.service.ts | 6 - .../streaming-communication.service.ts | 281 +++++++++ .../core/core-services/time-travel.service.ts | 6 +- .../core-services/websocket.service.spec.ts | 17 - .../core/core-services/websocket.service.ts | 551 ---------------- .../app/core/definitions/autoupdate-format.ts | 43 ++ .../src/app/core/definitions/http-methods.ts | 10 + client/src/app/core/promises/deferred.ts | 12 +- client/src/app/core/promises/sleep.ts | 10 + .../app/core/rxjs/trailing-throttle-time.ts | 6 + .../app/core/ui-services/banner.service.ts | 21 + .../core/ui-services/count-users.service.ts | 10 +- .../app/core/ui-services/overlay.service.ts | 15 +- .../copyright-sign.component.ts | 20 +- .../list-view-table.component.ts | 11 +- .../projector/projector.component.ts | 8 +- .../components/start/start.component.html | 1 + .../motion-detail/motion-detail.component.ts | 4 +- .../components/base-poll-dialog.component.ts | 2 - client/src/app/site/site.component.ts | 6 +- dc-dev.sh | 3 + docker/.env | 5 + docker/build.sh | 6 +- docker/docker-compose.dev.yml | 47 ++ docker/docker-compose.yml.m4 | 76 ++- docker/docker-stack.yml.m4 | 76 ++- haproxy/Dockerfile | 5 + haproxy/Dockerfile.dev | 5 + haproxy/Makefile | 3 + haproxy/build.sh | 6 + haproxy/prepare-cert.sh | 27 + haproxy/src/haproxy.common.cfg | 31 + haproxy/src/haproxy.dev.cfg | 20 + haproxy/src/haproxy.prod.cfg | 26 + server/.dockerignore | 3 + server/docker/Dockerfile | 3 +- server/docker/Dockerfile.dev | 30 + server/docker/entrypoint-db-setup | 1 - server/docker/entrypoint-dev | 14 + server/docker/settings.py | 19 +- server/openslides/agenda/apps.py | 4 - server/openslides/agenda/projector.py | 309 --------- server/openslides/assignments/apps.py | 4 - server/openslides/assignments/projector.py | 113 ---- server/openslides/core/apps.py | 6 +- server/openslides/core/projector.py | 68 -- server/openslides/core/serializers.py | 5 - server/openslides/core/urls.py | 3 +- server/openslides/core/views.py | 16 +- server/openslides/global_settings.py | 9 - server/openslides/mediafiles/apps.py | 5 - server/openslides/mediafiles/projector.py | 29 - server/openslides/motions/apps.py | 5 - server/openslides/motions/projector.py | 437 ------------- server/openslides/routing.py | 15 - server/openslides/topics/apps.py | 4 - server/openslides/topics/projector.py | 32 - server/openslides/users/apps.py | 4 - server/openslides/users/projector.py | 44 -- server/openslides/utils/autoupdate.py | 37 +- server/openslides/utils/cache.py | 2 + server/openslides/utils/cache_providers.py | 49 +- .../utils/consumer_autoupdate_strategy.py | 102 --- server/openslides/utils/consumers.py | 175 ------ server/openslides/utils/middleware.py | 73 --- server/openslides/utils/projector.py | 172 ----- server/openslides/utils/redis.py | 11 +- .../openslides/utils/redis_connection_pool.py | 126 +++- server/openslides/utils/settings.py.tpl | 56 +- server/openslides/utils/stream.py | 50 ++ server/openslides/utils/websocket.py | 267 -------- .../utils/websocket_client_messages.py | 215 ------- server/openslides/{asgi.py => wsgi.py} | 8 +- server/requirements/big_mode.txt | 6 +- server/requirements/production.txt | 2 - server/tests/integration/utils/__init__.py | 0 .../tests/integration/utils/test_consumers.py | 594 ------------------ server/tests/integration/websocket.py | 38 -- server/tests/unit/agenda/test_projector.py | 134 ---- server/tests/unit/core/__init__.py | 0 server/tests/unit/core/test_websocket.py | 44 -- server/tests/unit/motions/test_projector.py | 362 ----------- 112 files changed, 1849 insertions(+), 4755 deletions(-) create mode 100644 .gitmodules create mode 100644 Makefile create mode 160000 autoupdate create mode 100644 client/docker/Dockerfile.dev create mode 100644 client/src/app/core/core-services/autoupdate-throttle.service.ts create mode 100644 client/src/app/core/core-services/communication-manager.service.ts create mode 100644 client/src/app/core/core-services/offline-broadcast.service.ts delete mode 100644 client/src/app/core/core-services/ping.service.spec.ts delete mode 100644 client/src/app/core/core-services/ping.service.ts delete mode 100644 client/src/app/core/core-services/prioritize.service.spec.ts delete mode 100644 client/src/app/core/core-services/prioritize.service.ts create mode 100644 client/src/app/core/core-services/streaming-communication.service.ts delete mode 100644 client/src/app/core/core-services/websocket.service.spec.ts delete mode 100644 client/src/app/core/core-services/websocket.service.ts create mode 100644 client/src/app/core/definitions/autoupdate-format.ts create mode 100644 client/src/app/core/definitions/http-methods.ts create mode 100644 client/src/app/core/promises/sleep.ts create mode 100644 client/src/app/core/rxjs/trailing-throttle-time.ts create mode 100755 dc-dev.sh create mode 100644 docker/docker-compose.dev.yml create mode 100644 haproxy/Dockerfile create mode 100644 haproxy/Dockerfile.dev create mode 100644 haproxy/Makefile create mode 100755 haproxy/build.sh create mode 100755 haproxy/prepare-cert.sh create mode 100644 haproxy/src/haproxy.common.cfg create mode 100644 haproxy/src/haproxy.dev.cfg create mode 100644 haproxy/src/haproxy.prod.cfg create mode 100644 server/docker/Dockerfile.dev create mode 100755 server/docker/entrypoint-dev delete mode 100644 server/openslides/agenda/projector.py delete mode 100644 server/openslides/assignments/projector.py delete mode 100644 server/openslides/core/projector.py delete mode 100644 server/openslides/mediafiles/projector.py delete mode 100644 server/openslides/motions/projector.py delete mode 100644 server/openslides/routing.py delete mode 100644 server/openslides/topics/projector.py delete mode 100644 server/openslides/users/projector.py delete mode 100644 server/openslides/utils/consumer_autoupdate_strategy.py delete mode 100644 server/openslides/utils/consumers.py delete mode 100644 server/openslides/utils/middleware.py delete mode 100644 server/openslides/utils/projector.py create mode 100644 server/openslides/utils/stream.py delete mode 100644 server/openslides/utils/websocket.py delete mode 100644 server/openslides/utils/websocket_client_messages.py rename server/openslides/{asgi.py => wsgi.py} (70%) delete mode 100644 server/tests/integration/utils/__init__.py delete mode 100644 server/tests/integration/utils/test_consumers.py delete mode 100644 server/tests/integration/websocket.py delete mode 100644 server/tests/unit/agenda/test_projector.py delete mode 100644 server/tests/unit/core/__init__.py delete mode 100644 server/tests/unit/core/test_websocket.py delete mode 100644 server/tests/unit/motions/test_projector.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index cbe945e30..f9275339b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -40,7 +40,7 @@ jobs: run: mypy openslides/ tests/ - name: test using pytest - run: pytest --cov --cov-fail-under=75 + run: pytest --cov --cov-fail-under=74 install-client-dependencies: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 12d1656d2..05dd1bb4a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,14 +18,12 @@ .DS_Store Thumbs.db # Virtual Environment -.virtualenv*/* +.virtualenv* .venv -server/.venv ## Compatibility -# OS4-Submodules +# OS4-Submodules and aux-directories /openslides-*/ -/haproxy/ /docker/keys/ /docs/ # OS3+-Submodules @@ -79,7 +77,8 @@ cypress.json ## Deployment # Docker build artifacts -/docker/docker-compose.yml +docker/docker-compose.yml *-version.txt +*.pem # secrets docker/secrets/*.env diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..54f8d912f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "autoupdate"] + path = autoupdate + url = https://github.com/OpenSlides/openslides3-autoupdate-service.git diff --git a/DEVELOPMENT.rst b/DEVELOPMENT.rst index f4f239d2a..482eab1bf 100644 --- a/DEVELOPMENT.rst +++ b/DEVELOPMENT.rst @@ -2,187 +2,82 @@ OpenSlides Development ======================== -This instruction helps you to setup a development environment for OpenSlides. A -simple dev setup will be configured without the need of the docker-compose -setup. There are only the server running without a cache and a sqlite database -and the client as an development server. - - -1. Installation on GNU/Linux or Mac OS X ----------------------------------------- - -a. Check requirements +Check requirements ''''''''''''''''''''' -Make sure that you have installed `Python (>= 3.6) `_, -`Node.js (>=10.x) `_ and `Git `_ on -your system. You also need build-essential packages and header files and a -static library for Python. +- ``docker`` +- ``docker-compose`` +- ``git`` +- ``make`` -For Debian based systems (Ubuntu, etc) run:: - - $ sudo apt-get install git nodejs npm build-essential python3-dev +Note about migrating from previous OpenSlides3 development +setups: You must set the ``OPENSLIDES_USER_DATA_DIR`` variable in +your ``server/personal_data/var/settings.py`` to ``'/app/personal_data/var'`` -b. Get OpenSlides source code +Get OpenSlides source code ''''''''''''''''''''''''''''' Clone current master version from `OpenSlides GitHub repository `_:: - $ git clone https://github.com/OpenSlides/OpenSlides.git - $ cd OpenSlides + git clone https://github.com/OpenSlides/OpenSlides.git + cd OpenSlides + +TODO: submodules. + +Start the development setup +'''''''''''''''''''''''''''''' + + make run-dev -c. Setup a virtual Python environment (optional) -'''''''''''''''''''''''''''''''''''''''''''''''' +All you data (database, config, mediafiles) is stored in ``server/personal_data/var``. -You can setup a virtual Python environment using the virtual environment -(venv) package for Python to install OpenSlides as non-root user. This will -allow for encapsulated dependencies. They will be installed in the virtual -environment and not globally on your system. - -Setup and activate the virtual environment:: - - $ python3 -m venv .virtualenv - $ source .virtualenv/bin/activate - -You can exit the environment with:: - - $ deactivate - -d. Server -''''''''' - -Go into the server's directory:: - - $ cd server/ - -Install all required Python packages:: - - $ pip install --upgrade setuptools pip - $ pip install --requirement requirements.txt - -Create a settings file, run migrations and start the server:: - - $ python manage.py createsettings - $ python manage.py migrate - $ python manage.py runserver - -All you data (database, config, mediafiles) are stored in ``personal_data/var``. -To get help on the command line options run:: - - $ python manage.py --help - -Later you might want to restart the server with one of the following commands. - -To run the OpenSlides server execute:: - - $ python manage.py runserver - -When debugging something email related change the email backend to console:: - - $ python manage.py runserver --debug-email - -The server is available under http://localhost:8000. Especially the rest interface -might be important during development: http://localhost:8000/rest/ (The trailing -slash is important!). - -e. Client -''''''''' - -Go in the client's directory:: - - $ cd client/ - -Install all dependencies and start the development server:: - - $ npm install - $ npm start - -After a while, the client is available under http://localhost:4200. - - -2. Installation on Windows --------------------------- - -Follow the instructions above (Installation on GNU/Linux or Mac OS X) but care -of the following variations. - -To get Python download and run the latest `Python 3.7 32-bit (x86) executable -installer `_. Note that the 32-bit -installer is required even on a 64-bit Windows system. If you use the 64-bit -installer, step d. of the instruction might fail unless you installed some -packages manually. - -In some cases you have to install `MS Visual C++ 2015 build tools -`_ before you -install the required python packages for OpenSlides (unfortunately Twisted -needs it). - -To setup and activate the virtual environment in step c. use:: - - > .virtualenv\Scripts\activate.bat - -All other commands are the same as for GNU/Linux and Mac OS X. - - -3. Running the test cases +Running the test cases ------------------------- -a. Running server tests +For all services in submodules check out the documentation there. + +Server tests andscripts ''''''''''''''''''''''' +You need to have python (>=3.8) and python-venv installed. Change your workdirectory to the server:: -To run some server tests see `.travis.yml -`_. + cd server -b. Client tests and commands -'''''''''''''''''''''''''''' +Setup an python virtual environment. If you have already done it, you can skip this step: -Change to the client's directory to run every client related command. Run -client tests:: + python3 -m venv .venv + source .venv/bin/activate + pip install -U -r requirements.txt - $ npm test +Make sure you are using the correct python version (e.g. try with explicit minor version: ``python3.8``). Activate it:: + + source .venv/bin/activate + +To deactivate it type ``deactivate``. Running all tests and linters: + + black openslides/ tests/ + flake8 openslides/ tests/ + mypy openslides/ tests/ + isort -rc openslides/ tests/ + pytest tests/ + +Client tests +'''''''''''' +You need `node` and `npm` installed. Change to the client's directory. For the first time, install all dependencies:: + + cd client/ + npm install + +Run client tests:: + + npm test Fix the code format and lint it with:: - $ npm run prettify-write - $ npm run lint + npm run cleanup To extract translations run:: - $ npm run extract - -When updating, adding or changing used packages from npm, please update the -README.md using following command:: - - $ npm run licenses - - -4. Notes for running OpenSlides in larger setups ------------------------------------------------- - -For productive setups refer to the docker-compose setup described in the main -`README `_. - -While develpment it might be handy to use a cache and another database. -PostgreSQL is recommended and Redis necessary as a cache. Both can be set up in -the ``settings.py``. Please consider reading the `OpenSlides configuration -`_ page -to find out about all configurations, especially when using OpenSlides for big -assemblies. - -If you followed the instructions and installed the pip requirements form the -``requirements.py`` all needed dependencies for another worker are installed. -Instead of running ``python manage.py runserver`` you can use daphne or gunicorn -(the latter is used in the prod setup):: - - $ export DJANGO_SETTINGS_MODULE=settings - $ export PYTHONPATH=personal_data/var/ - $ daphne -b 0.0.0.0 -p 8000 openslides.asgi:application - -The last line may be interchangeable with gunicorn and uvicorn as protocol -server:: - - $ gunicorn -w 4 -b 0.0.0.0:8000 -k uvicorn.workers.UvicornWorker openslides.asgi:application - + npm run extract diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..a0e4a32c6 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +build-dev: + make -C haproxy build-dev + git submodule foreach 'make build-dev' + docker-compose -f docker/docker-compose.dev.yml build + +run-dev: | build-dev + UID=$$(id -u $${USER}) GID=$$(id -g $${USER}) docker-compose -f docker/docker-compose.dev.yml up + +stop-dev: + docker-compose -f docker/docker-compose.dev.yml down + +reload-haproxy: + docker-compose -f docker/docker-compose.dev.yml kill -s HUP haproxy + +get-server-shell: + docker-compose -f docker/docker-compose.dev.yml run server bash + diff --git a/README.rst b/README.rst index d0472636e..b104bc9c1 100644 --- a/README.rst +++ b/README.rst @@ -25,6 +25,8 @@ First, you have to clone this repository:: $ git clone https://github.com/OpenSlides/OpenSlides.git $ cd OpenSlides/docker/ +TODO: submodules. + You need to build the Docker images for the client and server with this script:: diff --git a/autoupdate b/autoupdate new file mode 160000 index 000000000..b187dd439 --- /dev/null +++ b/autoupdate @@ -0,0 +1 @@ +Subproject commit b187dd439bd7456e105901ca96cd0995862d4419 diff --git a/client/docker/Dockerfile b/client/docker/Dockerfile index 80b70a693..cc8556755 100644 --- a/client/docker/Dockerfile +++ b/client/docker/Dockerfile @@ -10,8 +10,8 @@ RUN npm install -g @angular/cli@^10 RUN ng config -g cli.warnings.versionMismatch false USER openslides -COPY package.json . -COPY package-lock.json . +COPY package.json package-lock.json ./ +RUN npm -v RUN npm ci COPY browserslist *.json ./ COPY src ./src diff --git a/client/docker/Dockerfile.dev b/client/docker/Dockerfile.dev new file mode 100644 index 000000000..ac7ce9ee4 --- /dev/null +++ b/client/docker/Dockerfile.dev @@ -0,0 +1,11 @@ +FROM node:13 + +WORKDIR /app + +COPY package.json . +RUN npm install +RUN npm run postinstall + +COPY . . + +CMD npm start diff --git a/client/docker/nginx.conf b/client/docker/nginx.conf index 7ac70cf3b..f96659452 100644 --- a/client/docker/nginx.conf +++ b/client/docker/nginx.conf @@ -25,29 +25,6 @@ http { gzip_proxied expired no-cache no-store private auth; gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; - location /apps { - proxy_pass http://server:8000; - } - location /media/ { - proxy_pass http://media:8000; - } - location /rest { - proxy_pass http://server:8000; - } - location /ws { - proxy_pass http://server:8000; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - } - location /server-version.txt { - proxy_pass http://server:8000; - } - - location = /basic_status { - stub_status; - } - location / { try_files $uri $uri/ /index.html; } diff --git a/client/ngsw-config.json b/client/ngsw-config.json index ff033e355..640d7e348 100644 --- a/client/ngsw-config.json +++ b/client/ngsw-config.json @@ -29,7 +29,7 @@ "dataGroups": [ { "name": "api", - "urls": ["/rest/*", "/apps/*"], + "urls": ["/rest/*", "/apps/*", "/system/*", "/stats"], "cacheConfig": { "maxSize": 0, "maxAge": "0u", diff --git a/client/package.json b/client/package.json index 87ce2ff17..d8e80e4f4 100644 --- a/client/package.json +++ b/client/package.json @@ -11,6 +11,7 @@ "scripts": { "ng": "ng", "start": "ng serve --proxy-config proxy.conf.json --host=0.0.0.0", + "start-https": "ng serve --ssl --ssl-cert localhost.pem --ssl-key localhost-key.pem --proxy-config proxy.conf.json --host=0.0.0.0", "start-es5": "ng serve --proxy-config proxy.conf.json --host=0.0.0.0 --configuration es5", "build": "ng build --prod", "build-to-dir": "npm run build -- --output-path", diff --git a/client/proxy.conf.json b/client/proxy.conf.json index 74366d271..7c27c143d 100644 --- a/client/proxy.conf.json +++ b/client/proxy.conf.json @@ -15,5 +15,9 @@ "target": "ws://localhost:8000", "secure": false, "ws": true + }, + "/system/": { + "target": "https://localhost:8002", + "secure": false } } diff --git a/client/src/app/app.component.ts b/client/src/app/app.component.ts index 15f70a46d..b28e470fb 100644 --- a/client/src/app/app.component.ts +++ b/client/src/app/app.component.ts @@ -10,10 +10,9 @@ import { CountUsersService } from './core/ui-services/count-users.service'; import { DataStoreUpgradeService } from './core/core-services/data-store-upgrade.service'; import { LoadFontService } from './core/ui-services/load-font.service'; import { LoginDataService } from './core/ui-services/login-data.service'; +import { OfflineService } from './core/core-services/offline.service'; import { OperatorService } from './core/core-services/operator.service'; import { OverlayService } from './core/ui-services/overlay.service'; -import { PingService } from './core/core-services/ping.service'; -import { PrioritizeService } from './core/core-services/prioritize.service'; import { RoutingStateService } from './core/ui-services/routing-state.service'; import { ServertimeService } from './core/core-services/servertime.service'; import { ThemeService } from './core/ui-services/theme.service'; @@ -75,17 +74,16 @@ export class AppComponent { appRef: ApplicationRef, servertimeService: ServertimeService, router: Router, + offlineService: OfflineService, operator: OperatorService, loginDataService: LoginDataService, - constantsService: ConstantsService, // Needs to be started, so it can register itself to the WebsocketService + constantsService: ConstantsService, themeService: ThemeService, overlayService: OverlayService, countUsersService: CountUsersService, // Needed to register itself. configService: ConfigService, loadFontService: LoadFontService, dataStoreUpgradeService: DataStoreUpgradeService, // to start it. - prioritizeService: PrioritizeService, - pingService: PingService, routingState: RoutingStateService, votingBannerService: VotingBannerService // needed for initialisation ) { diff --git a/client/src/app/core/core-services/autoupdate-throttle.service.ts b/client/src/app/core/core-services/autoupdate-throttle.service.ts new file mode 100644 index 000000000..f4dc66faa --- /dev/null +++ b/client/src/app/core/core-services/autoupdate-throttle.service.ts @@ -0,0 +1,164 @@ +import { EventEmitter, Injectable } from '@angular/core'; + +import { Observable, Subject } from 'rxjs'; + +import { AutoupdateFormat } from 'app/core/definitions/autoupdate-format'; +import { trailingThrottleTime } from 'app/core/rxjs/trailing-throttle-time'; +import { Identifiable } from 'app/shared/models/base/identifiable'; +import { ConstantsService } from './constants.service'; +import { HttpService } from './http.service'; + +interface ThrottleSettings { + AUTOUPDATE_DELAY?: number; +} + +@Injectable({ + providedIn: 'root' +}) +export class AutoupdateThrottleService { + private readonly _autoupdatesToInject = new Subject(); + + public get autoupdatesToInject(): Observable { + return this._autoupdatesToInject.asObservable(); + } + + private readonly receivedAutoupdate = new EventEmitter(); + + private pendingAutoupdates = []; + + private disabledUntil: number | null = null; + + private delay = 0; + + private maxSeenChangeId = 0; + + private get isActive(): boolean { + return this.delay !== 0; + } + + public constructor(private constantsService: ConstantsService, private httpService: HttpService) { + this.constantsService.get('Settings').subscribe(settings => { + // This is a one-shot. If the delay was set one time >0, it cannot be changed afterwards. + // A change is more complicated since you have to unsubscribe, clean pending autoupdates, + // subscribe again and make sure, that no autoupdate is missed. + if (this.delay === 0 && settings.AUTOUPDATE_DELAY) { + this.delay = 1000 * settings.AUTOUPDATE_DELAY; + console.log(`Configured autoupdate delay: ${this.delay}ms`); + this.receivedAutoupdate + .pipe(trailingThrottleTime(this.delay)) + .subscribe(() => this.processPendingAutoupdates()); + } else if (this.delay === 0) { + console.log('No autoupdate delay'); + } + }); + + this.httpService.responseChangeIds.subscribe(changeId => this.disableUntil(changeId)); + } + + public newAutoupdate(autoupdate: AutoupdateFormat): void { + if (autoupdate.to_change_id > this.maxSeenChangeId) { + this.maxSeenChangeId = autoupdate.to_change_id; + } + + if (!this.isActive) { + this._autoupdatesToInject.next(autoupdate); + } else if (this.disabledUntil !== null) { + this._autoupdatesToInject.next(autoupdate); + if (autoupdate.to_change_id >= this.disabledUntil) { + this.disabledUntil = null; + console.log('Throttling autoupdates again'); + } + } else { + this.pendingAutoupdates.push(autoupdate); + this.receivedAutoupdate.emit(); + } + } + + public disableUntil(changeId: number): void { + // Wait for an autoupdate with to_id >= changeId. + console.log(this.delay); + if (!this.isActive) { + return; + } + this.processPendingAutoupdates(); + // Checking with maxSeenChangeId is for the following race condition: + // If the autoupdate comes before the response, it must not be throttled. + // But flushing pending autoupdates is important since *if* the autoupdate + // was early, it is in the pending queue. + if (changeId <= this.maxSeenChangeId) { + return; + } + console.log('Disable autoupdate until change id', changeId); + this.disabledUntil = changeId; + } + + private processPendingAutoupdates(): void { + const autoupdates = this.pendingAutoupdates; + if (autoupdates.length === 0) { + return; + } + this.pendingAutoupdates = []; + + const autoupdate = this.mergeAutoupdates(autoupdates); + this._autoupdatesToInject.next(autoupdate); + } + + private mergeAutoupdates(autoupdates: AutoupdateFormat[]): AutoupdateFormat { + const mergedAutoupdate: AutoupdateFormat = { + changed: {}, + deleted: {}, + from_change_id: autoupdates[0].from_change_id, + to_change_id: autoupdates[autoupdates.length - 1].to_change_id, + all_data: false + }; + + let lastToChangeId = null; + for (const au of autoupdates) { + if (lastToChangeId === null) { + lastToChangeId = au.to_change_id; + } else { + if (au.from_change_id !== lastToChangeId) { + console.warn('!!!', autoupdates, au); + } + lastToChangeId = au.to_change_id; + } + + this.applyAutoupdate(au, mergedAutoupdate); + } + + return mergedAutoupdate; + } + + private applyAutoupdate(from: AutoupdateFormat, into: AutoupdateFormat): void { + if (from.all_data) { + into.all_data = true; + into.changed = from.changed; + into.deleted = from.deleted; + return; + } + + for (const collection of Object.keys(from.deleted)) { + for (const id of from.deleted[collection]) { + if (into.changed[collection]) { + into.changed[collection] = into.changed[collection].filter(obj => (obj as Identifiable).id !== id); + } + if (!into.deleted[collection]) { + into.deleted[collection] = []; + } + into.deleted[collection].push(id); + } + } + + for (const collection of Object.keys(from.changed)) { + for (const obj of from.changed[collection]) { + if (into.deleted[collection]) { + into.deleted[collection] = into.deleted[collection].filter(id => id !== (obj as Identifiable).id); + } + if (!into.changed[collection]) { + into.changed[collection] = []; + } + into.changed[collection].push(obj); + } + } + } +} diff --git a/client/src/app/core/core-services/autoupdate.service.ts b/client/src/app/core/core-services/autoupdate.service.ts index a286bc3e9..e478d796a 100644 --- a/client/src/app/core/core-services/autoupdate.service.ts +++ b/client/src/app/core/core-services/autoupdate.service.ts @@ -1,57 +1,16 @@ import { Injectable } from '@angular/core'; +import { AutoupdateFormat } from '../definitions/autoupdate-format'; +import { AutoupdateThrottleService } from './autoupdate-throttle.service'; import { BaseModel } from '../../shared/models/base/base-model'; import { CollectionStringMapperService } from './collection-string-mapper.service'; +import { CommunicationManagerService, OfflineError } from './communication-manager.service'; import { DataStoreService, DataStoreUpdateManagerService } from './data-store.service'; +import { HttpService } from './http.service'; import { Mutex } from '../promises/mutex'; -import { WebsocketService, WEBSOCKET_ERROR_CODES } from './websocket.service'; - -export interface AutoupdateFormat { - /** - * All changed (and created) items as their full/restricted data grouped by their collection. - */ - changed: { - [collectionString: string]: object[]; - }; - - /** - * All deleted items (by id) grouped by their collection. - */ - deleted: { - [collectionString: string]: number[]; - }; - - /** - * The lower change id bond for this autoupdate - */ - from_change_id: number; - - /** - * The upper change id bound for this autoupdate - */ - to_change_id: number; - - /** - * Flag, if this autoupdate contains all data. If so, the DS needs to be resetted. - */ - all_data: boolean; -} - -export function isAutoupdateFormat(obj: any): obj is AutoupdateFormat { - const format = obj as AutoupdateFormat; - return ( - obj && - typeof obj === 'object' && - format.changed !== undefined && - format.deleted !== undefined && - format.from_change_id !== undefined && - format.to_change_id !== undefined && - format.all_data !== undefined - ); -} /** - * Handles the initial update and automatic updates using the {@link WebsocketService} + * Handles the initial update and automatic updates * Incoming objects, usually BaseModels, will be saved in the dataStore (`this.DS`) * This service usually creates all models */ @@ -61,32 +20,49 @@ export function isAutoupdateFormat(obj: any): obj is AutoupdateFormat { export class AutoupdateService { private mutex = new Mutex(); - /** - * Constructor to create the AutoupdateService. Calls the constructor of the parent class. - * @param websocketService - * @param DS - * @param modelMapper - */ + private streamCloseFn: () => void | null = null; + + private lastMessageContainedAllData = false; + public constructor( - private websocketService: WebsocketService, private DS: DataStoreService, private modelMapper: CollectionStringMapperService, - private DSUpdateManager: DataStoreUpdateManagerService + private DSUpdateManager: DataStoreUpdateManagerService, + private communicationManager: CommunicationManagerService, + private autoupdateThrottle: AutoupdateThrottleService ) { - this.websocketService.getOberservable('autoupdate').subscribe(response => { - this.storeResponse(response); - }); + this.communicationManager.startCommunicationEvent.subscribe(() => this.startAutoupdate()); - // Check for too high change id-errors. If this happens, reset the DS and get fresh data. - this.websocketService.errorResponseObservable.subscribe(error => { - if (error.code === WEBSOCKET_ERROR_CODES.CHANGE_ID_TOO_HIGH) { - this.doFullUpdate(); + this.autoupdateThrottle.autoupdatesToInject.subscribe(autoupdate => this.storeAutoupdate(autoupdate)); + } + + public async startAutoupdate(changeId?: number): Promise { + this.stopAutoupdate(); + + try { + this.streamCloseFn = await this.communicationManager.subscribe( + '/system/autoupdate', + autoupdate => { + this.autoupdateThrottle.newAutoupdate(autoupdate); + }, + () => ({ change_id: (changeId ? changeId : this.DS.maxChangeId).toString() }) + ); + } catch (e) { + if (!(e instanceof OfflineError)) { + console.error(e); } - }); + } + } + + public stopAutoupdate(): void { + if (this.streamCloseFn) { + this.streamCloseFn(); + this.streamCloseFn = null; + } } /** - * Handle the answer of incoming data via {@link WebsocketService}. + * Handle the answer of incoming data, after it was throttled. * * Detects the Class of an incomming model, creates a new empty object and assigns * the data to it using the deserialize function. Also models that are flagged as deleted @@ -94,8 +70,9 @@ export class AutoupdateService { * * Handles the change ids of all autoupdates. */ - private async storeResponse(autoupdate: AutoupdateFormat): Promise { + private async storeAutoupdate(autoupdate: AutoupdateFormat): Promise { const unlock = await this.mutex.lock(); + this.lastMessageContainedAllData = autoupdate.all_data; if (autoupdate.all_data) { await this.storeAllData(autoupdate); } else { @@ -138,17 +115,10 @@ export class AutoupdateService { } else { // autoupdate fully in the future. we are missing something! console.log('Autoupdate in the future', maxChangeId, autoupdate.from_change_id, autoupdate.to_change_id); - this.requestChanges(); + this.startAutoupdate(); // restarts it. } } - public async injectAutoupdateIgnoreChangeId(autoupdate: AutoupdateFormat): Promise { - const unlock = await this.mutex.lock(); - console.debug('inject autoupdate', autoupdate); - await this.injectAutupdateIntoDS(autoupdate, false); - unlock(); - } - private async injectAutupdateIntoDS(autoupdate: AutoupdateFormat, flush: boolean): Promise { const updateSlot = await this.DSUpdateManager.getNewUpdateSlot(this.DS); @@ -187,35 +157,21 @@ export class AutoupdateService { } } - /** - * Sends a WebSocket request to the Server with the maxChangeId of the DataStore. - * The server should return an autoupdate with all new data. - */ - public requestChanges(): void { - console.log(`requesting changed objects with DS max change id ${this.DS.maxChangeId}`); - this.websocketService.send('getElements', { change_id: this.DS.maxChangeId }); - } - /** * Does a full update: Requests all data from the server and sets the DS to the fresh data. */ public async doFullUpdate(): Promise { - const oldChangeId = this.DS.maxChangeId; - const response = await this.websocketService.sendAndGetResponse<{}, AutoupdateFormat>('getElements', {}); - - const updateSlot = await this.DSUpdateManager.getNewUpdateSlot(this.DS); - let allModels: BaseModel[] = []; - for (const collection of Object.keys(response.changed)) { - if (this.modelMapper.isCollectionRegistered(collection)) { - allModels = allModels.concat(this.mapObjectsToBaseModels(collection, response.changed[collection])); - } else { - console.error(`Unregistered collection "${collection}". Ignore it.`); - } + if (this.lastMessageContainedAllData) { + console.log('full update requested. Skipping, last message already contained all data'); + } else { + console.log('requesting full update.'); + // The mutex is needed, so the DS is not cleared, if there is + // another autoupdate running. + const unlock = await this.mutex.lock(); + this.stopAutoupdate(); + await this.DS.clear(); + this.startAutoupdate(); + unlock(); } - - await this.DS.set(allModels, response.to_change_id); - this.DSUpdateManager.commit(updateSlot, response.to_change_id, true); - - console.log(`Full update done from ${oldChangeId} to ${response.to_change_id}`); } } diff --git a/client/src/app/core/core-services/communication-manager.service.ts b/client/src/app/core/core-services/communication-manager.service.ts new file mode 100644 index 000000000..0a576ecaf --- /dev/null +++ b/client/src/app/core/core-services/communication-manager.service.ts @@ -0,0 +1,186 @@ +import { HttpParams } from '@angular/common/http'; +import { EventEmitter, Injectable } from '@angular/core'; + +import { Observable } from 'rxjs'; + +import { HttpService } from './http.service'; +import { OfflineBroadcastService, OfflineReason } from './offline-broadcast.service'; +import { OperatorService } from './operator.service'; +import { SleepPromise } from '../promises/sleep'; +import { + CommunicationError, + ErrorType, + Stream, + StreamContainer, + StreamingCommunicationService, + verboseErrorType +} from './streaming-communication.service'; + +type HttpParamsGetter = () => HttpParams | { [param: string]: string | string[] }; + +const MAX_STREAM_FAILURE_RETRIES = 3; + +export class OfflineError extends Error { + public constructor() { + super(''); + this.name = 'OfflineError'; + } +} + +interface StreamConnectionWrapper { + id: number; + url: string; + messageHandler: (message: any) => void; + params: HttpParamsGetter; + stream?: Stream; + hasErroredAmount: number; +} + +@Injectable({ + providedIn: 'root' +}) +export class CommunicationManagerService { + private communicationAllowed = false; + + private readonly _startCommunicationEvent = new EventEmitter(); + + public get startCommunicationEvent(): Observable { + return this._startCommunicationEvent.asObservable(); + } + + private readonly _stopCommunicationEvent = new EventEmitter(); + + public get stopCommunicationEvent(): Observable { + return this._stopCommunicationEvent.asObservable(); + } + + private streamContainers: { [id: number]: StreamContainer } = {}; + + public constructor( + private streamingCommunicationService: StreamingCommunicationService, + private offlineBroadcastService: OfflineBroadcastService, + private http: HttpService, + private operatorService: OperatorService + ) { + this.offlineBroadcastService.goOfflineObservable.subscribe(() => this.closeConnections()); + } + + public async subscribe( + url: string, + messageHandler: (message: T) => void, + params?: HttpParamsGetter + ): Promise<() => void> { + if (!params) { + params = () => null; + } + + const streamContainer = new StreamContainer(url, messageHandler, params); + return await this.connectWithWrapper(streamContainer); + } + + public startCommunication(): void { + if (this.communicationAllowed) { + console.error('Illegal state! Do not emit this event multiple times'); + } else { + this.communicationAllowed = true; + this._startCommunicationEvent.emit(); + } + } + + private async connectWithWrapper(streamContainer: StreamContainer): Promise<() => void> { + console.log('connect', streamContainer, streamContainer.stream); + const errorHandler = (type: ErrorType, error: CommunicationError, message: string) => + this.handleError(streamContainer, type, error, message); + this.streamingCommunicationService.subscribe(streamContainer, errorHandler); + this.streamContainers[streamContainer.id] = streamContainer; + return () => this.close(streamContainer); + } + + private async handleError( + streamContainer: StreamContainer, + type: ErrorType, + error: CommunicationError, + message: string + ): Promise { + console.log('handle Error', streamContainer, streamContainer.stream, verboseErrorType(type), error, message); + streamContainer.stream.close(); + streamContainer.stream = null; + + streamContainer.hasErroredAmount++; + if (streamContainer.hasErroredAmount > MAX_STREAM_FAILURE_RETRIES) { + this.goOffline(streamContainer, OfflineReason.ConnectionLost); + } else if (type === ErrorType.Client && error.type === 'ErrorAuth') { + this.goOffline(streamContainer, OfflineReason.WhoAmIFailed); + } else { + // retry it after some time: + console.log( + `Retry no. ${streamContainer.hasErroredAmount} of ${MAX_STREAM_FAILURE_RETRIES} for ${streamContainer.url}` + ); + try { + await this.delayAndCheckReconnection(); + await this.connectWithWrapper(streamContainer); + } catch (e) { + // delayAndCheckReconnection can throw an OfflineError, + // which are just an 'abord mission' signal. Here, those errors can be ignored. + } + } + } + + private async delayAndCheckReconnection(): Promise { + const delay = Math.floor(Math.random() * 3000 + 2000); + console.log(`retry again in ${delay} ms`); + + await SleepPromise(delay); + + // do not continue, if we are offline! + if (this.offlineBroadcastService.isOffline()) { + console.log('we are offline?'); + throw new OfflineError(); + } + + // do not continue, if we are offline! + if (!this.shouldRetryConnecting()) { + console.log('operator changed, do not rety'); + throw new OfflineError(); // TODO: This error is not really good.... + } + } + + public closeConnections(): void { + for (const streamWrapper of Object.values(this.streamContainers)) { + if (streamWrapper.stream) { + streamWrapper.stream.close(); + } + } + this.streamContainers = {}; + this.communicationAllowed = false; + this._stopCommunicationEvent.emit(); + } + + private goOffline(streamContainer: StreamContainer, reason: OfflineReason): void { + delete this.streamContainers[streamContainer.id]; + this.closeConnections(); // here we close the connections early. + this.offlineBroadcastService.goOffline(reason); + } + + private close(streamConnectionWrapper: StreamConnectionWrapper): void { + if (this.streamContainers[streamConnectionWrapper.id]) { + this.streamContainers[streamConnectionWrapper.id].stream.close(); + delete this.streamContainers[streamConnectionWrapper.id]; + } + } + + // Checks the operator: If we do not have a valid user, + // do not even try to connect again.. + private shouldRetryConnecting(): boolean { + return this.operatorService.guestsEnabled || !!this.operatorService.user; + } + + public async isCommunicationServiceOnline(): Promise { + try { + const response = await this.http.get<{ healthy: boolean }>('/system/health'); + return !!response.healthy; + } catch (e) { + return false; + } + } +} diff --git a/client/src/app/core/core-services/constants.service.ts b/client/src/app/core/core-services/constants.service.ts index 00b90d4d0..1faea4de3 100644 --- a/client/src/app/core/core-services/constants.service.ts +++ b/client/src/app/core/core-services/constants.service.ts @@ -1,9 +1,11 @@ import { Injectable } from '@angular/core'; +import { environment } from 'environments/environment'; import { BehaviorSubject, Observable } from 'rxjs'; import { filter } from 'rxjs/operators'; -import { WebsocketService } from './websocket.service'; +import { CommunicationManagerService } from './communication-manager.service'; +import { HttpService } from './http.service'; /** * constants have a key associated with the data. @@ -36,24 +38,15 @@ export class ConstantsService { */ private subjects: { [key: string]: BehaviorSubject } = {}; - /** - * @param websocketService - */ - public constructor(private websocketService: WebsocketService) { - // The hook for recieving constants. - websocketService.getOberservable('constants').subscribe(constants => { - this.constants = constants; + public constructor(communicationManager: CommunicationManagerService, private http: HttpService) { + communicationManager.startCommunicationEvent.subscribe(async () => { + console.log('start communication'); + this.constants = await this.http.get(environment.urlPrefix + '/core/constants/'); + console.log('constants:', this.constants); Object.keys(this.subjects).forEach(key => { this.subjects[key].next(this.constants[key]); }); }); - - // We can request constants, if the websocket connection opens. - // On retries, the `refresh()` method is called by the OpenSlidesService, so - // here we do not need to take care about this. - websocketService.noRetryConnectEvent.subscribe(() => { - this.refresh(); - }); } /** @@ -66,14 +59,4 @@ export class ConstantsService { } return this.subjects[key].asObservable().pipe(filter(x => !!x)); } - - /** - * Refreshed the constants - */ - public refresh(): Promise { - if (!this.websocketService.isConnected) { - return; - } - this.websocketService.send('constants', {}); - } } diff --git a/client/src/app/core/core-services/data-store.service.ts b/client/src/app/core/core-services/data-store.service.ts index 206bf3b08..1f4d5cf89 100644 --- a/client/src/app/core/core-services/data-store.service.ts +++ b/client/src/app/core/core-services/data-store.service.ts @@ -361,12 +361,12 @@ export class DataStoreService { * * @returns The max change id. */ - public async initFromStorage(): Promise { + public async initFromStorage(): Promise { // This promise will be resolved with cached datastore. const store = await this.storageService.get(DataStoreService.cachePrefix + 'DS'); if (!store) { await this.clear(); - return this.maxChangeId; + return; } const updateSlot = await this.DSUpdateManager.getNewUpdateSlot(this); @@ -395,7 +395,6 @@ export class DataStoreService { this.DSUpdateManager.dropUpdateSlot(); await this.clear(); } - return this.maxChangeId; } /** @@ -670,6 +669,6 @@ export class DataStoreService { public print(): void { console.log('Max change id', this.maxChangeId); console.log(JSON.stringify(this.jsonStore)); - console.log(this.modelStore); + console.log(JSON.parse(JSON.stringify(this.modelStore))); } } diff --git a/client/src/app/core/core-services/http.service.ts b/client/src/app/core/core-services/http.service.ts index 7e10cbcbd..2c90183cf 100644 --- a/client/src/app/core/core-services/http.service.ts +++ b/client/src/app/core/core-services/http.service.ts @@ -2,22 +2,14 @@ import { HttpClient, HttpErrorResponse, HttpHeaders } from '@angular/common/http import { Injectable } from '@angular/core'; import { TranslateService } from '@ngx-translate/core'; +import { Subject } from 'rxjs'; -import { AutoupdateFormat, AutoupdateService, isAutoupdateFormat } from './autoupdate.service'; +import { AutoupdateFormat } from '../definitions/autoupdate-format'; +import { AutoupdateThrottleService } from './autoupdate-throttle.service'; +import { HTTPMethod } from '../definitions/http-methods'; import { OpenSlidesStatusService } from './openslides-status.service'; import { formatQueryParams, QueryParams } from '../definitions/query-params'; -/** - * Enum for different HTTPMethods - */ -export enum HTTPMethod { - GET = 'get', - POST = 'post', - PUT = 'put', - PATCH = 'patch', - DELETE = 'delete' -} - export interface ErrorDetailResponse { detail: string | string[]; args?: string[]; @@ -33,12 +25,12 @@ function isErrorDetailResponse(obj: any): obj is ErrorDetailResponse { } interface AutoupdateResponse { - autoupdate: AutoupdateFormat; + change_id: number; data?: any; } function isAutoupdateReponse(obj: any): obj is AutoupdateResponse { - return obj && typeof obj === 'object' && isAutoupdateFormat((obj as AutoupdateResponse).autoupdate); + return obj && typeof obj === 'object' && typeof (obj as AutoupdateResponse).change_id === 'number'; } /** @@ -53,6 +45,8 @@ export class HttpService { */ private defaultHeaders: HttpHeaders; + public readonly responseChangeIds = new Subject(); + /** * Construct a HttpService * @@ -65,8 +59,7 @@ export class HttpService { public constructor( private http: HttpClient, private translate: TranslateService, - private OSStatus: OpenSlidesStatusService, - private autoupdateService: AutoupdateService + private OSStatus: OpenSlidesStatusService ) { this.defaultHeaders = new HttpHeaders().set('Content-Type', 'application/json'); } @@ -205,8 +198,8 @@ export class HttpService { private processResponse(responseData: T): T { if (isAutoupdateReponse(responseData)) { - this.autoupdateService.injectAutoupdateIgnoreChangeId(responseData.autoupdate); - responseData = responseData.data; + this.responseChangeIds.next(responseData.change_id); + return responseData.data; } return responseData; } diff --git a/client/src/app/core/core-services/notify.service.ts b/client/src/app/core/core-services/notify.service.ts index e09ddee7f..b19e1c633 100644 --- a/client/src/app/core/core-services/notify.service.ts +++ b/client/src/app/core/core-services/notify.service.ts @@ -2,8 +2,9 @@ import { Injectable } from '@angular/core'; import { Observable, Subject } from 'rxjs'; +import { CommunicationManagerService, OfflineError } from './communication-manager.service'; +import { HttpService } from './http.service'; import { OperatorService } from './operator.service'; -import { WebsocketService } from './websocket.service'; /** * Encapslates the name and content of every message regardless of being a request or response. @@ -17,7 +18,12 @@ interface NotifyBase { /** * The content to send. */ - content: T; + message: T; +} + +function isNotifyBase(obj: object): obj is NotifyResponse { + const base = obj as NotifyBase; + return !!obj && base.message !== undefined && base.name !== undefined; } /** @@ -26,15 +32,18 @@ interface NotifyBase { * channel names. */ export interface NotifyRequest extends NotifyBase { + channel_id: string; + to_all?: boolean; + /** * User ids (or `true` for all users) to send this message to. */ - users?: number[] | boolean; + to_users?: number[]; /** * An array of channels to send this message to. */ - replyChannels?: string[]; + to_channels?: string[]; } /** @@ -45,12 +54,12 @@ export interface NotifyResponse extends NotifyBase { * This is the channel name of the one, who sends this message. Can be use to directly * answer this message. */ - senderChannelName: string; + sender_channel_id: string; /** * The user id of the user who sends this message. It is 0 for Anonymous. */ - senderUserId: number; + sender_user_id: number; /** * This is validated here and is true, if the senderUserId matches the current operator's id. @@ -59,6 +68,20 @@ export interface NotifyResponse extends NotifyBase { sendByThisUser: boolean; } +function isNotifyResponse(obj: object): obj is NotifyResponse { + const response = obj as NotifyResponse; + // Note: we do not test for sendByThisUser, since it is set later in our code. + return isNotifyBase(obj) && response.sender_channel_id !== undefined && response.sender_user_id !== undefined; +} + +interface ChannelIdResponse { + channel_id: string; +} + +function isChannelIdResponse(obj: object): obj is ChannelIdResponse { + return !!obj && (obj as ChannelIdResponse).channel_id !== undefined; +} + /** * Handles all incoming and outgoing notify messages via {@link WebsocketService}. */ @@ -78,18 +101,41 @@ export class NotifyService { [name: string]: Subject>; } = {}; - /** - * Constructor to create the NotifyService. Registers itself to the WebsocketService. - * @param websocketService - */ - public constructor(private websocketService: WebsocketService, private operator: OperatorService) { - websocketService.getOberservable>('notify').subscribe(notify => { - notify.sendByThisUser = notify.senderUserId === (this.operator.user ? this.operator.user.id : 0); - this.notifySubject.next(notify); - if (this.messageSubjects[notify.name]) { - this.messageSubjects[notify.name].next(notify); + private channelId: string; + + public constructor( + private communicationManager: CommunicationManagerService, + private http: HttpService, + private operator: OperatorService + ) { + this.communicationManager.startCommunicationEvent.subscribe(() => this.startListening()); + this.communicationManager.stopCommunicationEvent.subscribe(() => (this.channelId = null)); + } + + private async startListening(): Promise { + try { + await this.communicationManager.subscribe | ChannelIdResponse>( + '/system/notify', + notify => { + if (isChannelIdResponse(notify)) { + this.channelId = notify.channel_id; + } else if (isNotifyResponse(notify)) { + notify.sendByThisUser = + notify.sender_user_id === (this.operator.user ? this.operator.user.id : 0); + this.notifySubject.next(notify); + if (this.messageSubjects[notify.name]) { + this.messageSubjects[notify.name].next(notify); + } + } else { + console.error('Unknwon notify message', notify); + } + } + ); + } catch (e) { + if (!(e instanceof OfflineError)) { + console.log(e); } - }); + } } /** @@ -97,8 +143,8 @@ export class NotifyService { * @param name The name of the notify message * @param content The payload to send */ - public sendToAllUsers(name: string, content: T): void { - this.send(name, content); + public async sendToAllUsers(name: string, content: T): Promise { + await this.send(name, content, true); } /** @@ -107,8 +153,11 @@ export class NotifyService { * @param content The payload to send. * @param users Multiple user ids. */ - public sendToUsers(name: string, content: T, ...users: number[]): void { - this.send(name, content, users); + public async sendToUsers(name: string, content: T, ...users: number[]): Promise { + if (users.length < 1) { + throw new Error('You have to provide at least one user'); + } + await this.send(name, content, false, users); } /** @@ -117,35 +166,48 @@ export class NotifyService { * @param content The payload to send. * @param channels Multiple channels to send this message to. */ - public sendToChannels(name: string, content: T, ...channels: string[]): void { + public async sendToChannels(name: string, content: T, ...channels: string[]): Promise { if (channels.length < 1) { throw new Error('You have to provide at least one channel'); } - this.send(name, content, null, channels); + await this.send(name, content, false, null, channels); } /** * General send function for notify messages. * @param name The name of the notify message - * @param content The payload to send. + * @param message The payload to send. * @param users Either an array of IDs or `true` meaning of sending this message to all online users clients. * @param channels An array of channels to send this message to. */ - public send(name: string, content: T, users?: number[] | boolean, channels?: string[]): void { + private async send( + name: string, + message: T, + toAll?: boolean, + users?: number[], + channels?: string[] + ): Promise { + if (!this.channelId) { + throw new Error('No channel id!'); + } + const notify: NotifyRequest = { name: name, - content: content + message: message, + channel_id: this.channelId }; - if (typeof users === 'boolean' && users !== true) { - throw new Error('You just can give true as a boolean to send this message to all users.'); + if (toAll === true) { + notify.to_all = true; } - if (users !== null) { - notify.users = users; + if (users) { + notify.to_users = users; } - if (channels !== null) { - notify.replyChannels = channels; + if (channels) { + notify.to_channels = channels; } - this.websocketService.send('notify', notify); + + console.debug('send notify', notify); + await this.http.post('/system/notify/send', notify); } /** diff --git a/client/src/app/core/core-services/offline-broadcast.service.ts b/client/src/app/core/core-services/offline-broadcast.service.ts new file mode 100644 index 000000000..eb72f4323 --- /dev/null +++ b/client/src/app/core/core-services/offline-broadcast.service.ts @@ -0,0 +1,37 @@ +import { EventEmitter, Injectable } from '@angular/core'; + +import { BehaviorSubject, Observable } from 'rxjs'; + +export enum OfflineReason { + WhoAmIFailed, + ConnectionLost +} + +@Injectable({ + providedIn: 'root' +}) +export class OfflineBroadcastService { + public readonly isOfflineSubject = new BehaviorSubject(false); + public get isOfflineObservable(): Observable { + return this.isOfflineSubject.asObservable(); + } + + private readonly _goOffline = new EventEmitter(); + public get goOfflineObservable(): Observable { + return this._goOffline.asObservable(); + } + + public constructor() {} + + public goOffline(reason: OfflineReason): void { + this._goOffline.emit(reason); + } + + public isOffline(): boolean { + return this.isOfflineSubject.getValue(); + } + + public isOnline(): boolean { + return !this.isOffline(); + } +} diff --git a/client/src/app/core/core-services/offline.service.ts b/client/src/app/core/core-services/offline.service.ts index 9689c018e..b80fe756c 100644 --- a/client/src/app/core/core-services/offline.service.ts +++ b/client/src/app/core/core-services/offline.service.ts @@ -1,10 +1,9 @@ import { Injectable } from '@angular/core'; -import { marker as _ } from '@biesbjerg/ngx-translate-extract-marker'; -import { TranslateService } from '@ngx-translate/core'; -import { BehaviorSubject, Observable } from 'rxjs'; - -import { BannerDefinition, BannerService } from '../ui-services/banner.service'; +import { CommunicationManagerService } from './communication-manager.service'; +import { OfflineBroadcastService, OfflineReason } from './offline-broadcast.service'; +import { OpenSlidesService } from './openslides.service'; +import { OperatorService, WhoAmI } from './operator.service'; /** * This service handles everything connected with being offline. @@ -16,63 +15,111 @@ import { BannerDefinition, BannerService } from '../ui-services/banner.service'; providedIn: 'root' }) export class OfflineService { - /** - * BehaviorSubject to receive further status values. - */ - private offline = new BehaviorSubject(false); - private bannerDefinition: BannerDefinition = { - text: _('Offline mode'), - icon: 'cloud_off' - }; + private reason: OfflineReason | null; - public constructor(private banner: BannerService, translate: TranslateService) { - translate.onLangChange.subscribe(() => { - this.bannerDefinition.text = translate.instant(this.bannerDefinition.text); - }); - } - - /** - * Determines of you are either in Offline mode or not connected via websocket - * - * @returns whether the client is offline or not connected - */ - public isOffline(): Observable { - return this.offline; - } - - /** - * Sets the offline flag. Restores the DataStoreService to the last known configuration. - */ - public goOfflineBecauseFailedWhoAmI(): void { - if (!this.offline.getValue()) { - console.log('offline because whoami failed.'); - } - this.goOffline(); - } - - /** - * Sets the offline flag, because there is no connection to the server. - */ - public goOfflineBecauseConnectionLost(): void { - if (!this.offline.getValue()) { - console.log('offline because connection lost.'); - } - this.goOffline(); + public constructor( + private OpenSlides: OpenSlidesService, + private offlineBroadcastService: OfflineBroadcastService, + private operatorService: OperatorService, + private communicationManager: CommunicationManagerService + ) { + this.offlineBroadcastService.goOfflineObservable.subscribe((reason: OfflineReason) => this.goOffline(reason)); } /** * Helper function to set offline status */ - private goOffline(): void { - this.offline.next(true); - this.banner.addBanner(this.bannerDefinition); + public goOffline(reason: OfflineReason): void { + if (this.offlineBroadcastService.isOffline()) { + return; + } + this.reason = reason; + + if (reason === OfflineReason.ConnectionLost) { + console.log('offline because connection lost.'); + } else if (reason === OfflineReason.WhoAmIFailed) { + console.log('offline because whoami failed.'); + } else { + console.error('No such offline reason', reason); + } + + this.offlineBroadcastService.isOfflineSubject.next(true); + this.checkStillOffline(); + } + + private checkStillOffline(): void { + const timeout = Math.floor(Math.random() * 3000 + 2000); + console.log(`Try to go online in ${timeout} ms`); + + setTimeout(async () => { + let online: boolean; + let whoami: WhoAmI | null = null; + + if (this.reason === OfflineReason.ConnectionLost) { + online = await this.communicationManager.isCommunicationServiceOnline(); + console.log('is communication online? ', online); + } else if (this.reason === OfflineReason.WhoAmIFailed) { + const result = await this.operatorService.whoAmI(); + online = result.online; + whoami = result.whoami; + console.log('is whoami reachable?', online); + } + + if (online) { + await this.goOnline(whoami); + // TODO: check all other reasons -> e.g. if the + // connection was lost, the operator must be checked and the other way + // around the comminucation must be started!! + + // stop trying. + } else { + // continue trying. + this.checkStillOffline(); + } + }, timeout); } /** * Function to return to online-status. + * + * First, we have to check, if all other sources (except this.reason) are online, too. + * This results in definetly having a whoami response at this point. + * If this is the case, we need to setup everything again: + * 1) check the operator. If this allowes for an logged in state (or anonymous is OK), do + * step 2, otherwise done. + * 2) enable communications. */ - public goOnline(): void { - this.offline.next(false); - this.banner.removeBanner(this.bannerDefinition); + private async goOnline(whoami?: WhoAmI): Promise { + console.log('go online!', this.reason, whoami); + if (this.reason === OfflineReason.ConnectionLost) { + // now we have to check whoami + const result = await this.operatorService.whoAmI(); + if (!result.online) { + console.log('whoami down.'); + this.reason = OfflineReason.WhoAmIFailed; + this.checkStillOffline(); + return; + } + whoami = result.whoami; + } else if (this.reason === OfflineReason.WhoAmIFailed) { + const online = await this.communicationManager.isCommunicationServiceOnline(); + if (!online) { + console.log('communication down.'); + this.reason = OfflineReason.ConnectionLost; + this.checkStillOffline(); + return; + } + } + console.log('we are online!'); + + // Ok, we are online now! + const isLoggedIn = await this.OpenSlides.checkWhoAmI(whoami); + console.log('logged in:', isLoggedIn); + if (isLoggedIn) { + this.communicationManager.startCommunication(); + } + console.log('done'); + + this.offlineBroadcastService.isOfflineSubject.next(false); } } diff --git a/client/src/app/core/core-services/openslides.service.ts b/client/src/app/core/core-services/openslides.service.ts index a35c47ed8..0ff38c224 100644 --- a/client/src/app/core/core-services/openslides.service.ts +++ b/client/src/app/core/core-services/openslides.service.ts @@ -3,12 +3,11 @@ import { Router } from '@angular/router'; import { BehaviorSubject } from 'rxjs'; -import { AutoupdateService } from './autoupdate.service'; -import { ConstantsService } from './constants.service'; +import { CommunicationManagerService } from './communication-manager.service'; import { DataStoreService } from './data-store.service'; -import { OperatorService } from './operator.service'; +import { OfflineBroadcastService, OfflineReason } from './offline-broadcast.service'; +import { OperatorService, WhoAmI } from './operator.service'; import { StorageService } from './storage.service'; -import { WebsocketService } from './websocket.service'; /** * Handles the bootup/showdown of this application. @@ -35,29 +34,19 @@ export class OpenSlidesService { return this.booted.value; } - /** - * Constructor to create the OpenSlidesService. Registers itself to the WebsocketService. - * @param storageService - * @param operator - * @param websocketService - * @param router - * @param autoupdateService - * @param DS - */ public constructor( private storageService: StorageService, private operator: OperatorService, - private websocketService: WebsocketService, private router: Router, - private autoupdateService: AutoupdateService, private DS: DataStoreService, - private constantsService: ConstantsService + private communicationManager: CommunicationManagerService, + private offlineBroadcastService: OfflineBroadcastService ) { // Handler that gets called, if the websocket connection reconnects after a disconnection. // There might have changed something on the server, so we check the operator, if he changed. - websocketService.retryReconnectEvent.subscribe(() => { + /*websocketService.retryReconnectEvent.subscribe(() => { this.checkOperator(); - }); + });*/ this.bootup(); } @@ -68,20 +57,24 @@ export class OpenSlidesService { */ public async bootup(): Promise { // start autoupdate if the user is logged in: - let response = await this.operator.whoAmIFromStorage(); - const needToCheckOperator = !!response; + let whoami = await this.operator.whoAmIFromStorage(); + const needToCheckOperator = !!whoami; - if (!response) { - response = await this.operator.whoAmI(); + if (!whoami) { + const response = await this.operator.whoAmI(); + if (!response.online) { + this.offlineBroadcastService.goOffline(OfflineReason.WhoAmIFailed); + } + whoami = response.whoami; } - if (!response.user && !response.guest_enabled) { + if (!whoami.user && !whoami.guest_enabled) { if (!location.pathname.includes('error')) { this.redirectUrl = location.pathname; } this.redirectToLoginIfNotSubpage(); } else { - await this.afterLoginBootup(response.user_id); + await this.afterLoginBootup(whoami.user_id); } if (needToCheckOperator) { @@ -121,7 +114,7 @@ export class OpenSlidesService { await this.DS.clear(); await this.storageService.set('lastUserLoggedIn', userId); } - await this.setupDataStoreAndWebSocket(); + await this.setupDataStoreAndStartCommunication(); // Now finally booted. this.booted.next(true); } @@ -129,23 +122,16 @@ export class OpenSlidesService { /** * Init DS from cache and after this start the websocket service. */ - private async setupDataStoreAndWebSocket(): Promise { - const changeId = await this.DS.initFromStorage(); - // disconnect the WS connection, if there was one. This is needed - // to update the connection parameters, namely the cookies. If the user - // is changed, the WS needs to reconnect, so the new connection holds the new - // user information. - if (this.websocketService.isConnected) { - await this.websocketService.close(); // Wait for the disconnect. - } - await this.websocketService.connect(changeId); // Request changes after changeId. + private async setupDataStoreAndStartCommunication(): Promise { + await this.DS.initFromStorage(); + this.communicationManager.startCommunication(); } /** - * Shuts down OpenSlides. The websocket connection is closed and the operator is not set. + * Shuts down OpenSlides. */ public async shutdown(): Promise { - await this.websocketService.close(); + this.communicationManager.closeConnections(); this.booted.next(false); } @@ -167,29 +153,37 @@ export class OpenSlidesService { await this.bootup(); } + public async checkOperator(requestChanges: boolean = true): Promise { + const response = await this.operator.whoAmI(); + if (!response.online) { + this.offlineBroadcastService.goOffline(OfflineReason.WhoAmIFailed); + } + await this.checkWhoAmI(response.whoami, requestChanges); + } + /** * Verify that the operator is the same as it was before. Should be alled on a reconnect. + * + * @returns true, if the user is still logged in */ - private async checkOperator(requestChanges: boolean = true): Promise { - const response = await this.operator.whoAmI(); + public async checkWhoAmI(whoami: WhoAmI, requestChanges: boolean = true): Promise { + let isLoggedIn = false; // User logged off. - if (!response.user && !response.guest_enabled) { - this.websocketService.cancelReconnectenRetry(); + if (!whoami.user && !whoami.guest_enabled) { await this.shutdown(); this.redirectToLoginIfNotSubpage(); } else { + isLoggedIn = true; if ( - (this.operator.user && this.operator.user.id !== response.user_id) || - (!this.operator.user && response.user_id) + (this.operator.user && this.operator.user.id !== whoami.user_id) || + (!this.operator.user && whoami.user_id) ) { // user changed await this.DS.clear(); await this.reboot(); - } else if (requestChanges) { - // User is still the same, but check for missed autoupdates. - this.autoupdateService.requestChanges(); - this.constantsService.refresh(); } } + + return isLoggedIn; } } diff --git a/client/src/app/core/core-services/operator.service.ts b/client/src/app/core/core-services/operator.service.ts index 3e9b1f804..8ad5c5af8 100644 --- a/client/src/app/core/core-services/operator.service.ts +++ b/client/src/app/core/core-services/operator.service.ts @@ -10,7 +10,6 @@ import { CollectionStringMapperService } from './collection-string-mapper.servic import { DataStoreService } from './data-store.service'; import { Deferred } from '../promises/deferred'; import { HttpService } from './http.service'; -import { OfflineService } from './offline.service'; import { OnAfterAppsLoaded } from '../definitions/on-after-apps-loaded'; import { OpenSlidesStatusService } from './openslides-status.service'; import { StorageService } from './storage.service'; @@ -207,7 +206,6 @@ export class OperatorService implements OnAfterAppsLoaded { public constructor( private http: HttpService, private DS: DataStoreService, - private offlineService: OfflineService, private collectionStringMapper: CollectionStringMapperService, private storageService: StorageService, private OSStatus: OpenSlidesStatusService @@ -306,18 +304,19 @@ export class OperatorService implements OnAfterAppsLoaded { * * @returns The response of the WhoAmI request. */ - public async whoAmI(): Promise { + public async whoAmI(): Promise<{ whoami: WhoAmI; online: boolean }> { + let online = true; try { const response = await this.http.get(environment.urlPrefix + '/users/whoami/'); if (isWhoAmI(response)) { await this.updateCurrentWhoAmI(response); } else { - this.offlineService.goOfflineBecauseFailedWhoAmI(); + online = false; } } catch (e) { - this.offlineService.goOfflineBecauseFailedWhoAmI(); + online = false; } - return this.currentWhoAmI; + return { whoami: this.currentWhoAmI, online }; } /** diff --git a/client/src/app/core/core-services/ping.service.spec.ts b/client/src/app/core/core-services/ping.service.spec.ts deleted file mode 100644 index 39e2e713f..000000000 --- a/client/src/app/core/core-services/ping.service.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { inject, TestBed } from '@angular/core/testing'; - -import { E2EImportsModule } from '../../../e2e-imports.module'; -import { PingService } from './ping.service'; - -describe('PingService', () => { - beforeEach(() => { - TestBed.configureTestingModule({ - imports: [E2EImportsModule], - providers: [PingService] - }); - }); - - it('should be created', inject([PingService], (service: PingService) => { - expect(service).toBeTruthy(); - })); -}); diff --git a/client/src/app/core/core-services/ping.service.ts b/client/src/app/core/core-services/ping.service.ts deleted file mode 100644 index 5fddd7873..000000000 --- a/client/src/app/core/core-services/ping.service.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { Injectable } from '@angular/core'; - -import { ConstantsService } from './constants.service'; -import { Deferred } from '../promises/deferred'; -import { TimeoutPromise } from '../promises/timeout-promise'; -import { WebsocketService } from './websocket.service'; - -interface OpenSlidesSettings { - PING_INTERVAL?: number; - PING_TIMEOUT?: number; -} - -@Injectable({ - providedIn: 'root' -}) -export class PingService { - /** - * The interval. - */ - private pingInterval: any; - - private intervalTime = 30000; - - private timeoutTime = 5000; - - private lastLatency: number | null = null; - - public constructor(private websocketService: WebsocketService, private constantsService: ConstantsService) { - this.setup(); - } - - private async setup(): Promise { - const gotConstants = new Deferred(); - - this.constantsService.get('Settings').subscribe(settings => { - this.intervalTime = settings.PING_INTERVAL || 30000; - this.timeoutTime = settings.PING_TIMEOUT || 5000; - gotConstants.resolve(); - }); - await gotConstants; - - // Connects the ping-pong mechanism to the opening and closing of the connection. - this.websocketService.closeEvent.subscribe(() => this.stopPing()); - this.websocketService.generalConnectEvent.subscribe(() => this.startPing()); - if (this.websocketService.isConnected) { - this.startPing(); - } - } - - /** - * Starts the ping-mechanism - */ - private startPing(): void { - if (this.pingInterval) { - return; - } - - this.pingInterval = setInterval(async () => { - const start = performance.now(); - try { - await TimeoutPromise( - this.websocketService.sendAndGetResponse('ping', this.lastLatency), - this.timeoutTime - ); - this.lastLatency = performance.now() - start; - if (this.lastLatency > 1000) { - console.warn(`Ping took ${this.lastLatency / 1000} seconds.`); - } - } catch (e) { - console.warn(`The server didn't respond to ping within ${this.timeoutTime / 1000} seconds.`); - this.stopPing(); - this.websocketService.simulateAbnormalClose(); - } - }, this.intervalTime); - } - - /** - * Clears the ping interval - */ - private stopPing(): void { - if (this.pingInterval) { - clearInterval(this.pingInterval); - this.pingInterval = null; - } - } -} diff --git a/client/src/app/core/core-services/prioritize.service.spec.ts b/client/src/app/core/core-services/prioritize.service.spec.ts deleted file mode 100644 index 08b34266c..000000000 --- a/client/src/app/core/core-services/prioritize.service.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { inject, TestBed } from '@angular/core/testing'; - -import { E2EImportsModule } from '../../../e2e-imports.module'; -import { PrioritizeService } from './prioritize.service'; - -describe('PrioritizeService', () => { - beforeEach(() => { - TestBed.configureTestingModule({ - imports: [E2EImportsModule], - providers: [PrioritizeService] - }); - }); - - it('should be created', inject([PrioritizeService], (service: PrioritizeService) => { - expect(service).toBeTruthy(); - })); -}); diff --git a/client/src/app/core/core-services/prioritize.service.ts b/client/src/app/core/core-services/prioritize.service.ts deleted file mode 100644 index e7677ab3b..000000000 --- a/client/src/app/core/core-services/prioritize.service.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Injectable } from '@angular/core'; - -import { ConstantsService } from './constants.service'; -import { DataStoreService } from './data-store.service'; -import { OpenSlidesStatusService } from './openslides-status.service'; -import { OperatorService } from './operator.service'; -import { WebsocketService } from './websocket.service'; - -interface OpenSlidesSettings { - PRIORITIZED_GROUP_IDS?: number[]; -} - -/** - * Cares about prioritizing a client. Checks, if the operator is in one of - * some prioritized groups. These group ids come from the server. If the prio- - * ritization changes, the websocket connection gets reconnected. - */ -@Injectable({ - providedIn: 'root' -}) -export class PrioritizeService { - private prioritizedGroupIds: number[] = []; - - public constructor( - constantsService: ConstantsService, - private websocketService: WebsocketService, - private DS: DataStoreService, - private openSlidesStatusService: OpenSlidesStatusService, - private operator: OperatorService - ) { - constantsService.get('Settings').subscribe(settings => { - this.prioritizedGroupIds = settings.PRIORITIZED_GROUP_IDS || []; - this.checkPrioritization(); - }); - operator.getUserObservable().subscribe(() => this.checkPrioritization()); - } - - private checkPrioritization(): void { - const opPrioritized = this.operator.isInGroupIdsNonAdminCheck(...this.prioritizedGroupIds); - if (this.openSlidesStatusService.isPrioritizedClient !== opPrioritized) { - console.log('Alter prioritization:', opPrioritized); - this.openSlidesStatusService.isPrioritizedClient = opPrioritized; - this.websocketService.reconnect(this.DS.maxChangeId); - } - } -} diff --git a/client/src/app/core/core-services/projector-data.service.ts b/client/src/app/core/core-services/projector-data.service.ts index 0374110f2..f12e459b3 100644 --- a/client/src/app/core/core-services/projector-data.service.ts +++ b/client/src/app/core/core-services/projector-data.service.ts @@ -3,8 +3,8 @@ import { Injectable } from '@angular/core'; import { BehaviorSubject, Observable, Subject } from 'rxjs'; import { auditTime } from 'rxjs/operators'; -import { WebsocketService } from 'app/core/core-services/websocket.service'; import { Projector, ProjectorElement } from 'app/shared/models/core/projector'; +import { CommunicationManagerService, OfflineError } from './communication-manager.service'; export interface SlideData { data: T; @@ -15,13 +15,13 @@ export interface SlideData { - if (this.currentChangeId > update.change_id) { - return; - } - Object.keys(update.data).forEach(_id => { - const id = parseInt(_id, 10); - if (this.currentProjectorData[id]) { - this.currentProjectorData[id].next(update.data[id] as ProjectorData); - } - }); - this.currentChangeId = update.change_id; - }); + private streamCloseFn: () => void | null = null; - // The service need to re-register, if the websocket connection was lost. - this.websocketService.generalConnectEvent.subscribe(() => this.updateProjectorDataSubscription()); + public constructor(private communicationManager: CommunicationManagerService) { + this.communicationManager.startCommunicationEvent.subscribe(() => this.updateProjectorDataSubscription()); // With a bit of debounce, update the needed projectors. this.updateProjectorDataDebounceSubject.pipe(auditTime(10)).subscribe(() => { const allActiveProjectorIds = Object.keys(this.openProjectorInstances) .map(id => parseInt(id, 10)) .filter(id => this.openProjectorInstances[id] > 0); - this.websocketService.send('listenToProjectors', { projector_ids: allActiveProjectorIds }); + this.requestProjectors(allActiveProjectorIds); }); } + public async requestProjectors(allActiveProjectorIds: number[]): Promise { + this.cancelCurrentServerSubscription(); + + if (allActiveProjectorIds.length === 0) { + return; + } + + try { + this.streamCloseFn = await this.communicationManager.subscribe( + '/system/projector', + message => { + this.handleMesage(message); + }, + () => ({ projector_ids: allActiveProjectorIds.join(',') }) + ); + } catch (e) { + if (!(e instanceof OfflineError)) { + console.log(e); + } + } + } + + public cancelCurrentServerSubscription(): void { + if (this.streamCloseFn) { + this.streamCloseFn(); + this.streamCloseFn = null; + } + } + + private handleMesage(message: ProjectorDataMessage): void { + if (this.currentChangeId > message.change_id) { + console.log('Projector: Change id too low:', this.currentChangeId, message.change_id); + return; + } + Object.keys(message.data).forEach(_id => { + const id = parseInt(_id, 10); + if (this.currentProjectorData[id]) { + this.currentProjectorData[id].next(message.data[id] as ProjectorData); + } + }); + this.currentChangeId = message.change_id; + } + /** * Gets an observable for the projector data. * diff --git a/client/src/app/core/core-services/projector.service.ts b/client/src/app/core/core-services/projector.service.ts index 73d7979c6..bdd209070 100644 --- a/client/src/app/core/core-services/projector.service.ts +++ b/client/src/app/core/core-services/projector.service.ts @@ -40,12 +40,6 @@ export interface ProjectorTitle { providedIn: 'root' }) export class ProjectorService { - /** - * Constructor. - * - * @param DS - * @param dataSend - */ public constructor( private DS: DataStoreService, private http: HttpService, diff --git a/client/src/app/core/core-services/streaming-communication.service.ts b/client/src/app/core/core-services/streaming-communication.service.ts new file mode 100644 index 000000000..7ea42eda1 --- /dev/null +++ b/client/src/app/core/core-services/streaming-communication.service.ts @@ -0,0 +1,281 @@ +import { + HttpClient, + HttpDownloadProgressEvent, + HttpEvent, + HttpHeaderResponse, + HttpHeaders, + HttpParams +} from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +import { Observable, Subscription } from 'rxjs'; + +import { HTTPMethod } from '../definitions/http-methods'; + +const HEADER_EVENT_TYPE = 2; +const PROGRESS_EVENT_TYPE = 3; +const FINISH_EVENT_TYPE = 4; + +export type Params = HttpParams | { [param: string]: string | string[] }; +export enum ErrorType { + Client, + Server, // or network errors, they are the same. + Unknown +} +export function verboseErrorType(type: ErrorType): string { + switch (type) { + case ErrorType.Client: + return 'Client'; + case ErrorType.Server: + return 'Server'; + case ErrorType.Unknown: + return 'Unknown'; + } +} +export interface CommunicationError { + type: string; + msg: string; +} +export function isCommunicationError(obj: any): obj is CommunicationError { + const _obj = obj as CommunicationError; + return typeof obj === 'object' && typeof _obj.msg === 'string' && typeof _obj.type === 'string'; +} +interface CommunicationErrorWrapper { + error: CommunicationError; +} +export function isCommunicationErrorWrapper(obj: any): obj is CommunicationErrorWrapper { + return typeof obj === 'object' && isCommunicationError(obj.error); +} +export type ErrorHandler = (type: ErrorType, error: CommunicationError, message: string) => void; + +export class StreamConnectionError extends Error { + public constructor(public code: number, message: string) { + super(message); + this.name = 'StreamConnectionError'; + } +} + +export class StreamContainer { + public readonly id = Math.floor(Math.random() * (900000 - 1) + 100000); // [100000, 999999] + + public hasErroredAmount = 0; + + public stream?: Stream; + + public constructor( + public url: string, + public messageHandler: (message: any) => void, + public params: () => Params + ) {} +} + +export class Stream { + private subscription: Subscription = null; + + private hasError = false; + private reportedError = false; + + private _statuscode: number; + public get statuscode(): number { + return this._statuscode; + } + + private _errorContent: CommunicationError; + public get errorContent(): CommunicationError { + return this._errorContent; + } + + /** + * This is the index where we checked, if there is a \n in the read buffer (event.partialText) + * This position is always >= contentStartIndex and is > contentStartIndex, if the message + * was too big to fit into one buffer. So we have just a partial message. + * + * The difference between this index and contentStartIndex is that this index remembers the position + * we checked for a \n which lay in the middle of the next JOSN-packet. + */ + private checkedUntilIndex = 0; + + /** + * This index holds always the position of the current JOSN-packet, that we are receiving. + */ + private contentStartIndex = 0; + + private closed = false; + + public constructor( + observable: Observable>, + private messageHandler: (message: T) => void, + private errorHandler: ErrorHandler + ) { + this.subscription = observable.subscribe( + (event: HttpEvent) => { + if (this.closed) { + return; + } + if (event.type === HEADER_EVENT_TYPE) { + const headerResponse = event as HttpHeaderResponse; + this._statuscode = headerResponse.status; + if (headerResponse.status >= 400) { + this.hasError = true; + } + } else if ((>event).type === PROGRESS_EVENT_TYPE) { + this.handleMessage(event as HttpDownloadProgressEvent); + } else if ((>event).type === FINISH_EVENT_TYPE) { + this.errorHandler(ErrorType.Server, null, 'The stream was closed'); + } + }, + error => { + this.errorHandler(ErrorType.Server, error, 'Network error'); + }, + () => { + console.log('The stream was completed'); + } + ); + } + + private handleMessage(event: HttpDownloadProgressEvent): void { + if (this.hasError) { + if (!this.reportedError) { + this.reportedError = true; + // try to get the `error` key from object + this._errorContent = this.tryParseError(event.partialText); + this.errorHandler(this.getErrorTypeFromStatusCode(), this._errorContent, 'Reported error by server'); + } + return; + } + // Maybe we get multiple messages, so continue, until the complete buffer is checked. + while (this.checkedUntilIndex < event.loaded) { + // check if there is a \n somewhere in [checkedUntilIndex, ...] + const LF_index = event.partialText.indexOf('\n', this.checkedUntilIndex); + + if (LF_index >= 0) { + // take string in [contentStartIndex, LF_index-1]. This must be valid JSON. + // In substring, the last character is exlusive. + const content = event.partialText.substring(this.contentStartIndex, LF_index); + + // move pointer: next JSON starts at LF_index + 1 + this.checkedUntilIndex = LF_index + 1; + this.contentStartIndex = LF_index + 1; + + const parsedContent = this.tryParseJson(content); + if (isCommunicationError(parsedContent)) { + if (this.hasError && this.reportedError) { + return; + } + this.hasError = true; + this._errorContent = parsedContent; + // Do not trigger the error handler, if the connection-retry-routine is still handling this issue + if (!this.reportedError) { + this.reportedError = true; + console.error(this._errorContent); + this.errorHandler( + this.getErrorTypeFromStatusCode(), + this._errorContent, + 'Reported error by server' + ); + } + return; + } else { + // check, if we didn't get a keep alive + if (Object.keys(parsedContent).length > 0) { + console.debug('received', parsedContent); + this.messageHandler(parsedContent); + } + } + } else { + this.checkedUntilIndex = event.loaded; + } + } + } + + private getErrorTypeFromStatusCode(): ErrorType { + if (!this.statuscode) { + return ErrorType.Unknown; + } + if (this.statuscode >= 400 && this.statuscode < 500) { + return ErrorType.Client; + } + if (this.statuscode >= 500) { + return ErrorType.Server; + } + return ErrorType.Unknown; + } + + private tryParseJson(json: string): T | CommunicationError { + try { + return JSON.parse(json) as T; + } catch (e) { + return this.tryParseError(json); + } + } + + /** + * This one is a bit tricky. Error can be: + * - string with HTML, e.g. provided by proxies if the service is unreachable + * - string with json of form {"error": {"type": ..., "msg": ...}} + */ + private tryParseError(error: any): CommunicationError { + if (typeof error === 'string') { + try { + error = JSON.parse(error); + } catch (e) { + return { type: 'Unknown Error', msg: error }; + } + } + + if (isCommunicationErrorWrapper(error)) { + return error.error; + } else if (isCommunicationError(error)) { + return error; + } + + // we have something else.... ?? + console.error('Unknown error', error); + throw new Error('Unknown error: ' + error.toString()); + } + + public close(): void { + this.subscription.unsubscribe(); + this.subscription = null; + this.closed = true; + } +} + +@Injectable({ + providedIn: 'root' +}) +export class StreamingCommunicationService { + public constructor(private http: HttpClient) {} + + public subscribe(streamContainer: StreamContainer, errorHandler: ErrorHandler): void { + const options: { + body?: any; + headers?: HttpHeaders | { [header: string]: string | string[] }; + observe: 'events'; + params?: HttpParams | { [param: string]: string | string[] }; + reportProgress?: boolean; + responseType: 'text'; + withCredentials?: boolean; + } = { + headers: { + 'Content-Type': 'application/json', + 'ngsw-bypass': 'yes' + }, + responseType: 'text', + observe: 'events', + reportProgress: true + }; + const params = streamContainer.params(); + if (params) { + options.params = params; + } + const observable = this.http.request(HTTPMethod.GET, streamContainer.url, options); + + if (streamContainer.stream) { + console.error('Illegal state!'); + } + + const stream = new Stream(observable, streamContainer.messageHandler, errorHandler); + streamContainer.stream = stream; + } +} diff --git a/client/src/app/core/core-services/time-travel.service.ts b/client/src/app/core/core-services/time-travel.service.ts index 3e68f77d7..d0d8260b7 100644 --- a/client/src/app/core/core-services/time-travel.service.ts +++ b/client/src/app/core/core-services/time-travel.service.ts @@ -9,7 +9,6 @@ import { DataStoreService, DataStoreUpdateManagerService } from './data-store.se import { HttpService } from './http.service'; import { OpenSlidesStatusService } from './openslides-status.service'; import { OpenSlidesService } from './openslides.service'; -import { WebsocketService } from './websocket.service'; interface HistoryData { [collection: string]: BaseModel[]; @@ -31,7 +30,6 @@ export class TimeTravelService { * Constructs the time travel service * * @param httpService To fetch the history data - * @param webSocketService to disable websocket connection * @param modelMapperService to cast history objects into models * @param DS to overwrite the dataStore * @param OSStatus Sets the history status @@ -39,7 +37,6 @@ export class TimeTravelService { */ public constructor( private httpService: HttpService, - private webSocketService: WebsocketService, private modelMapperService: CollectionStringMapperService, private DS: DataStoreService, private OSStatus: OpenSlidesStatusService, @@ -100,7 +97,8 @@ export class TimeTravelService { * Clears the DataStore and stops the WebSocket connection */ private async stopTime(history: History): Promise { - await this.webSocketService.close(); + // await this.webSocketService.close(); + // TODO await this.DS.set(); // Same as clear, but not persistent. this.OSStatus.enterHistoryMode(history); } diff --git a/client/src/app/core/core-services/websocket.service.spec.ts b/client/src/app/core/core-services/websocket.service.spec.ts deleted file mode 100644 index 887ef988f..000000000 --- a/client/src/app/core/core-services/websocket.service.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { inject, TestBed } from '@angular/core/testing'; - -import { E2EImportsModule } from '../../../e2e-imports.module'; -import { WebsocketService } from './websocket.service'; - -describe('WebsocketService', () => { - beforeEach(() => { - TestBed.configureTestingModule({ - imports: [E2EImportsModule], - providers: [WebsocketService] - }); - }); - - it('should be created', inject([WebsocketService], (service: WebsocketService) => { - expect(service).toBeTruthy(); - })); -}); diff --git a/client/src/app/core/core-services/websocket.service.ts b/client/src/app/core/core-services/websocket.service.ts deleted file mode 100644 index b7208416c..000000000 --- a/client/src/app/core/core-services/websocket.service.ts +++ /dev/null @@ -1,551 +0,0 @@ -import { EventEmitter, Injectable, NgZone } from '@angular/core'; -import { MatSnackBarRef, SimpleSnackBar } from '@angular/material/snack-bar'; -import { Router } from '@angular/router'; - -import { compress, decompress } from 'lz4js'; -import { Observable, Subject } from 'rxjs'; -import { take } from 'rxjs/operators'; - -import { OfflineService } from './offline.service'; -import { OpenSlidesStatusService } from './openslides-status.service'; -import { formatQueryParams, QueryParams } from '../definitions/query-params'; - -/** - * The generic message format in which messages are send and recieved by the server. - */ -interface BaseWebsocketMessage { - type: string; - content: any; -} - -/** - * Outgoing messages must have an id. - */ -interface OutgoingWebsocketMessage extends BaseWebsocketMessage { - id: string; -} - -/** - * Incomming messages may have an `in_response`, if they are an answer to a previously - * submitted request. - */ -interface IncommingWebsocketMessage extends BaseWebsocketMessage { - in_response?: string; -} - -/** - * The format of a messages content, if the message type is "error" - */ -interface WebsocketErrorContent { - code: number; - message: string; -} - -function isWebsocketErrorContent(obj: any): obj is WebsocketErrorContent { - return !!obj && obj.code !== undefined && obj.message !== undefined; -} - -/** - * All (custom) error codes that are used to pass error information - * from the server to the client - */ -export const WEBSOCKET_ERROR_CODES = { - NOT_AUTHORIZED: 100, - CHANGE_ID_TOO_HIGH: 101, - WRONG_FORMAT: 102 -}; - -/** - * Service that handles WebSocket connections. Other services can register themselfs - * with {@method getOberservable} for a specific type of messages. The content will be published. - */ -@Injectable({ - providedIn: 'root' -}) -export class WebsocketService { - /** - * The reference to the snackbar entry that is shown, if the connection is lost. - */ - private connectionErrorNotice: MatSnackBarRef; - - /** - * Subjects that will be called, if a reconnect after a retry (e.g. with a previous - * connection loss) was successful. - */ - private readonly _retryReconnectEvent: EventEmitter = new EventEmitter(); - - /** - * Getter for the retry reconnect event. - */ - public get retryReconnectEvent(): EventEmitter { - return this._retryReconnectEvent; - } - - /** - * Subjects that will be called, if connect took place, but not a retry reconnect. - * THis is the complement from the generalConnectEvent to the retryReconnectEvent. - */ - private readonly _noRetryConnectEvent: EventEmitter = new EventEmitter(); - - /** - * Getter for the no-retry connect event. - */ - public get noRetryConnectEvent(): EventEmitter { - return this._noRetryConnectEvent; - } - - /** - * Listeners will be nofitied, if the wesocket connection is establiched. - */ - private readonly _generalConnectEvent: EventEmitter = new EventEmitter(); - - /** - * Getter for the connect event. - */ - public get generalConnectEvent(): EventEmitter { - return this._generalConnectEvent; - } - - /** - * Listeners will be nofitied, if the wesocket connection is closed. - */ - private readonly _closeEvent: EventEmitter = new EventEmitter(); - - /** - * Getter for the close event. - */ - public get closeEvent(): EventEmitter { - return this._closeEvent; - } - - /** - * The subject for all websocket *message* errors (no connection errors). - */ - private readonly _errorResponseSubject = new Subject(); - - /** - * The error response obersable for all websocket message errors. - */ - public get errorResponseObservable(): Observable { - return this._errorResponseSubject.asObservable(); - } - - /** - * Saves, if the connection is open - */ - private _connectionOpen = false; - - /** - * Whether the WebSocket connection is established - */ - public get isConnected(): boolean { - return this._connectionOpen; - } - - private sendQueueWhileNotConnected: (string | ArrayBuffer)[] = []; - - /** - * The websocket. - */ - private websocket: WebSocket | null; - private websocketId: string | null; - - /** - * Subjects for types of websocket messages. A subscriber can get an Observable by {@function getOberservable}. - */ - private subjects: { [type: string]: Subject } = {}; - - /** - * Callbacks for a waiting response. If any callback returns true, the message/error will not be propagated with the - * responsible subjects for the message type. - */ - private responseCallbacks: { - [id: string]: [(val: any) => boolean, (error: WebsocketErrorContent) => boolean]; - } = {}; - - /** - * Saves, if the WS Connection should be closed (e.g. after an explicit `close()`). Prohibits - * retry connection attempts. - */ - private shouldBeClosed = true; - - /** - * Counter for delaying the offline message. - */ - private retryCounter = 0; - - /** - * The timeout in the onClose-handler for the next reconnect retry. - */ - private retryTimeout: any = null; - - /** - * Constructor that handles the router - * - * @param zone - * @param router - * @param openSlidesStatusService - * @param offlineService - */ - public constructor( - private zone: NgZone, - private router: Router, - private openSlidesStatusService: OpenSlidesStatusService, - private offlineService: OfflineService - ) {} - - /** - * Creates a new WebSocket connection and handles incomming events. - * - * Uses NgZone to let all callbacks run in the angular context. - */ - public async connect(changeId: number | null = null, retry: boolean = false): Promise { - const websocketId = Math.random().toString(36).substring(7); - this.websocketId = websocketId; - - if (this.websocket) { - this.websocket.close(); - this.websocket = null; - } - - if (!retry) { - this.shouldBeClosed = false; - } - - const queryParams: QueryParams = {}; - - if (changeId !== null) { - queryParams.change_id = changeId; - } - - // Create the websocket - let socketPath = location.protocol === 'https:' ? 'wss://' : 'ws://'; - socketPath += window.location.host; - if (this.openSlidesStatusService.isPrioritizedClient) { - socketPath += '/prioritize'; - } - socketPath += '/ws/'; - socketPath += formatQueryParams(queryParams); - - this.websocket = new WebSocket(socketPath); - this.websocket.binaryType = 'arraybuffer'; - - // connection established. If this connect attept was a retry, - // The error notice will be removed and the reconnectSubject is published. - this.websocket.onopen = (event: Event) => { - if (this.websocketId !== websocketId) { - return; - } - this.zone.run(() => { - this.retryCounter = 0; - - if (this.shouldBeClosed) { - this.offlineService.goOnline(); - return; - } - - this._connectionOpen = true; - if (retry) { - this.offlineService.goOnline(); - this._retryReconnectEvent.emit(); - } else { - this._noRetryConnectEvent.emit(); - } - this._generalConnectEvent.emit(); - this.sendQueueWhileNotConnected.forEach(entry => { - this.websocket.send(entry); - }); - this.sendQueueWhileNotConnected = []; - }); - }; - - this.websocket.onmessage = (event: MessageEvent) => { - if (this.websocketId !== websocketId) { - return; - } - this.zone.run(() => { - this.handleMessage(event.data); - }); - }; - - this.websocket.onclose = (event: CloseEvent) => { - if (this.websocketId !== websocketId) { - return; - } - this.zone.run(() => { - this.onclose(); - }); - }; - - this.websocket.onerror = (event: ErrorEvent) => { - if (this.websocketId !== websocketId) { - return; - } - // place for proper error handling and debugging. - // Required to get more information about errors - this.zone.run(() => { - console.warn('WS error event:', event); - }); - }; - } - - /** - * Handles an incomming message. - * - * @param data The message - */ - private handleMessage(data: string | ArrayBuffer): void { - if (data instanceof ArrayBuffer) { - const compressedSize = data.byteLength; - const decompressedBuffer: Uint8Array = decompress(new Uint8Array(data)); - console.debug( - `Recieved ${compressedSize / 1024} KB (${ - decompressedBuffer.byteLength / 1024 - } KB uncompressed), ratio ${decompressedBuffer.byteLength / compressedSize}` - ); - data = this.arrayBufferToString(decompressedBuffer); - } - - const message: IncommingWebsocketMessage = JSON.parse(data); - console.debug('Received', message); - const type = message.type; - const inResponse = message.in_response; - const callbacks = this.responseCallbacks[inResponse]; - if (callbacks) { - delete this.responseCallbacks[inResponse]; - } - - if (type === 'error') { - if (!isWebsocketErrorContent(message.content)) { - console.error('Websocket error without standard form!', message); - return; - } - - // Print this to the console. - const error = message.content; - const errorDescription = - Object.keys(WEBSOCKET_ERROR_CODES).find(key => WEBSOCKET_ERROR_CODES[key] === error.code) || - 'unknown code'; - console.error(`Websocket error with code=${error.code} (${errorDescription}):`, error.message); - - // call the error callback, if there is any. If it returns true (means "handled"), - // the errorResponseSubject will not be called - if (inResponse && callbacks && callbacks[1] && callbacks[1](error)) { - return; - } - this._errorResponseSubject.next(error); - return; - } - - // Try to fire a response callback directly. If it returnes true, the message is handeled - // and not distributed further - if (inResponse && callbacks && callbacks[0](message.content)) { - return; - } - - if (this.subjects[type]) { - // Pass the content to the registered subscribers. - this.subjects[type].next(message.content); - } else { - console.warn( - `Got unknown websocket message type "${type}" (inResponse: ${inResponse}) with content`, - message.content - ); - } - } - - /** - * Closes the connection error notice - */ - private onclose(): void { - if (this.websocket) { - this.websocketId = null; // set to null, so now further events will be - // registered with the line below. - this.websocket.close(); // Cleanup old connection - this.websocket = null; - } - this._connectionOpen = false; - // 1000 is a normal close, like the close on logout - this._closeEvent.emit(); - if (!this.shouldBeClosed) { - // Do not show the message snackbar on the projector - // tests for /projector and /projector/ - const onProjector = this.router.url.match(/^\/projector(\/[0-9]+\/?)?$/); - if (this.retryCounter <= 3) { - this.retryCounter++; - } - - if (!this.connectionErrorNotice && !onProjector && this.retryCounter > 3) { - this.offlineService.goOfflineBecauseConnectionLost(); - } - - // A random retry timeout between 2000 and 5000 ms. - const timeout = Math.floor(Math.random() * 3000 + 2000); - this.retryTimeout = setTimeout(() => { - this.retryTimeout = null; - this.connect(null, true); - }, timeout); - } - } - - public cancelReconnectenRetry(): void { - if (this.retryTimeout) { - clearTimeout(this.retryTimeout); - this.retryTimeout = null; - } - } - - /** - * Closes the websocket connection. - */ - public async close(): Promise { - this.shouldBeClosed = true; - this.offlineService.goOnline(); - if (this.websocket) { - this.websocket.close(); - this.websocket = null; - await this.closeEvent.pipe(take(1)).toPromise(); - } - } - - /** - * Simulates an abnormal close. - * - * Internally does not set `shouldBeClosed`, so a reconnect is forced. - */ - public simulateAbnormalClose(): void { - this.onclose(); - } - - /** - * closes and reopens the connection. If the connection was closed before, - * it will be just opened. - * - * @param options The options for the new connection - */ - public async reconnect(changeId: number | null = null): Promise { - await this.close(); - await this.connect(changeId); - } - - /** - * Returns an observable for messages of the given type. - * @param type the message type - */ - public getOberservable(type: string): Observable { - if (!this.subjects[type]) { - this.subjects[type] = new Subject(); - } - return this.subjects[type].asObservable(); - } - - /** - * Sends a message to the server with the content and the given type. - * - * @param type the message type - * @param content the actual content - * @param success an optional success callback for a response. If it returns true, the message will not be - * propagated through the recieve subjects. - * @param error an optional error callback for a response. If it returns true, the error will not be propagated - * with the error subject. - * @param id an optional id for the message. If not given, a random id will be generated and returned. - * @returns the message id - */ - public send( - type: string, - content: T, - success?: (val: R) => boolean, - error?: (error: WebsocketErrorContent) => boolean, - id?: string - ): string { - if (!this.websocket) { - return; - } - - const message: OutgoingWebsocketMessage = { - type: type, - content: content, - id: id - }; - - // create message id if not given. Required by the server. - if (!message.id) { - message.id = ''; - const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'; - for (let i = 0; i < 8; i++) { - message.id += possible.charAt(Math.floor(Math.random() * possible.length)); - } - } - - if (success) { - this.responseCallbacks[message.id] = [success, error]; - } - - const jsonMessage = JSON.stringify(message); - const bytesMessage = this.stringToBuffer(jsonMessage); - - const compressedMessage: ArrayBuffer = compress(bytesMessage); - const ratio = bytesMessage.byteLength / compressedMessage.byteLength; - - const toSend = ratio > 1 ? compressedMessage : jsonMessage; - - if (this.isConnected) { - this.websocket.send(toSend); - } else { - this.sendQueueWhileNotConnected.push(toSend); - } - - return message.id; - } - - /** - * Sends a message and waits for the response - * - * @param type the message type - * @param content the actual content - * @param id an optional id for the message. If not given, a random id will be generated and returned. - */ - public sendAndGetResponse(type: string, content: T, id?: string): Promise { - return new Promise((resolve, reject) => { - this.send( - type, - content, - val => { - resolve(val); - return true; - }, - val => { - reject(val); - return true; - }, - id - ); - }); - } - - /** - * Converts an ArrayBuffer to a String. - * - * @param buffer - Buffer to convert - * @returns String - */ - private arrayBufferToString(buffer: Uint8Array): string { - return Array.from(buffer) - .map(code => String.fromCharCode(code)) - .join(''); - } - - /** - * Converts a String to an ArrayBuffer. - * - * @param str - String to convert. - * @returns bufferView. - */ - private stringToBuffer(str: string): Uint8Array { - const bufferView = new Uint8Array(); - for (let i = 0; i < str.length; i++) { - bufferView[i] = str.charCodeAt(i); - } - return bufferView; - } -} diff --git a/client/src/app/core/definitions/autoupdate-format.ts b/client/src/app/core/definitions/autoupdate-format.ts new file mode 100644 index 000000000..97539b054 --- /dev/null +++ b/client/src/app/core/definitions/autoupdate-format.ts @@ -0,0 +1,43 @@ +export interface AutoupdateFormat { + /** + * All changed (and created) items as their full/restricted data grouped by their collection. + */ + changed: { + [collectionString: string]: object[]; + }; + + /** + * All deleted items (by id) grouped by their collection. + */ + deleted: { + [collectionString: string]: number[]; + }; + + /** + * The lower change id bond for this autoupdate + */ + from_change_id: number; + + /** + * The upper change id bound for this autoupdate + */ + to_change_id: number; + + /** + * Flag, if this autoupdate contains all data. If so, the DS needs to be resetted. + */ + all_data: boolean; +} + +export function isAutoupdateFormat(obj: any): obj is AutoupdateFormat { + const format = obj as AutoupdateFormat; + return ( + obj && + typeof obj === 'object' && + format.changed !== undefined && + format.deleted !== undefined && + format.from_change_id !== undefined && + format.to_change_id !== undefined && + format.all_data !== undefined + ); +} diff --git a/client/src/app/core/definitions/http-methods.ts b/client/src/app/core/definitions/http-methods.ts new file mode 100644 index 000000000..23638f24e --- /dev/null +++ b/client/src/app/core/definitions/http-methods.ts @@ -0,0 +1,10 @@ +/** + * Enum for different HTTPMethods + */ +export enum HTTPMethod { + GET = 'get', + POST = 'post', + PUT = 'put', + PATCH = 'patch', + DELETE = 'delete' +} diff --git a/client/src/app/core/promises/deferred.ts b/client/src/app/core/promises/deferred.ts index 82be23512..258e2a6a2 100644 --- a/client/src/app/core/promises/deferred.ts +++ b/client/src/app/core/promises/deferred.ts @@ -14,16 +14,16 @@ * ``` */ export class Deferred extends Promise { - /** - * The promise to wait for - */ - public readonly promise: Promise; - /** * custom resolve function */ private _resolve: (val?: T) => void; + private _wasResolved; + public get wasResolved(): boolean { + return this._wasResolved; + } + /** * Creates the promise and overloads the resolve function */ @@ -32,6 +32,7 @@ export class Deferred extends Promise { super(resolve => { preResolve = resolve; }); + this._wasResolved = false; this._resolve = preResolve; } @@ -40,5 +41,6 @@ export class Deferred extends Promise { */ public resolve(val?: T): void { this._resolve(val); + this._wasResolved = true; } } diff --git a/client/src/app/core/promises/sleep.ts b/client/src/app/core/promises/sleep.ts new file mode 100644 index 000000000..ab3e3740a --- /dev/null +++ b/client/src/app/core/promises/sleep.ts @@ -0,0 +1,10 @@ +/** + * Wraps a promise and let it reject after the given timeout (in ms), if it was + * not resolved before this timeout. + * + * @param delay The time to sleep in miliseconds + * @returns a new Promise + */ +export function SleepPromise(delay: number): Promise { + return new Promise((resolve, _) => setTimeout(resolve, delay)); +} diff --git a/client/src/app/core/rxjs/trailing-throttle-time.ts b/client/src/app/core/rxjs/trailing-throttle-time.ts new file mode 100644 index 000000000..3052e9c51 --- /dev/null +++ b/client/src/app/core/rxjs/trailing-throttle-time.ts @@ -0,0 +1,6 @@ +import { MonoTypeOperatorFunction } from 'rxjs'; +import { throttleTime } from 'rxjs/operators'; + +export function trailingThrottleTime(time: number): MonoTypeOperatorFunction { + return throttleTime(time, undefined, { leading: false, trailing: true }); +} diff --git a/client/src/app/core/ui-services/banner.service.ts b/client/src/app/core/ui-services/banner.service.ts index 7bc1a6619..06f5699e5 100644 --- a/client/src/app/core/ui-services/banner.service.ts +++ b/client/src/app/core/ui-services/banner.service.ts @@ -1,7 +1,10 @@ import { Injectable } from '@angular/core'; +import { marker as _ } from '@biesbjerg/ngx-translate-extract-marker'; import { BehaviorSubject } from 'rxjs'; +import { OfflineBroadcastService } from '../core-services/offline-broadcast.service'; + export interface BannerDefinition { type?: string; class?: string; @@ -20,8 +23,26 @@ export interface BannerDefinition { providedIn: 'root' }) export class BannerService { + private offlineBannerDefinition: BannerDefinition = { + text: _('Offline mode'), + icon: 'cloud_off' + }; + public activeBanners: BehaviorSubject = new BehaviorSubject([]); + public constructor(/*translate: TranslateService, */ offlineBroadcastService: OfflineBroadcastService) { + /*translate.onLangChange.subscribe(() => { + this.offlineBannerDefinition.text = translate.instant(this.offlineBannerDefinition.text); + });*/ + offlineBroadcastService.isOfflineObservable.subscribe(offline => { + if (offline) { + this.addBanner(this.offlineBannerDefinition); + } else { + this.removeBanner(this.offlineBannerDefinition); + } + }); + } + /** * Adds a banner to the list of active banners. Skip the banner if it's already in the list * @param toAdd the banner to add diff --git a/client/src/app/core/ui-services/count-users.service.ts b/client/src/app/core/ui-services/count-users.service.ts index 91ea7a9ec..f1c33907c 100644 --- a/client/src/app/core/ui-services/count-users.service.ts +++ b/client/src/app/core/ui-services/count-users.service.ts @@ -42,24 +42,24 @@ export class CountUsersService { public constructor(private notifyService: NotifyService, operator: OperatorService) { // Listen for requests to send an answer. this.notifyService.getMessageObservable(REQUEST_NAME).subscribe(request => { - if (request.content.token) { + if (request.message.token) { this.notifyService.sendToChannels( RESPONSE_NAME, { - token: request.content.token, + token: request.message.token, data: { userId: this.currentUserId } }, - request.senderChannelName + request.sender_channel_id ); } }); // Listen for responses and distribute them through `activeCounts` this.notifyService.getMessageObservable(RESPONSE_NAME).subscribe(response => { - if (response.content.data && response.content.token && this.activeCounts[response.content.token]) { - this.activeCounts[response.content.token].next(response.content.data); + if (response.message.data && response.message.token && this.activeCounts[response.message.token]) { + this.activeCounts[response.message.token].next(response.message.data); } }); diff --git a/client/src/app/core/ui-services/overlay.service.ts b/client/src/app/core/ui-services/overlay.service.ts index dea05232a..36b476ac2 100644 --- a/client/src/app/core/ui-services/overlay.service.ts +++ b/client/src/app/core/ui-services/overlay.service.ts @@ -7,7 +7,7 @@ import { distinctUntilChanged } from 'rxjs/operators'; import { largeDialogSettings } from 'app/shared/utils/dialog-settings'; import { SuperSearchComponent } from 'app/site/common/components/super-search/super-search.component'; import { DataStoreUpgradeService } from '../core-services/data-store-upgrade.service'; -import { OfflineService } from '../core-services/offline.service'; +import { OfflineBroadcastService } from '../core-services/offline-broadcast.service'; import { OpenSlidesService } from '../core-services/openslides.service'; import { OperatorService } from '../core-services/operator.service'; @@ -59,7 +59,7 @@ export class OverlayService { private operator: OperatorService, OpenSlides: OpenSlidesService, upgradeService: DataStoreUpgradeService, - offlineService: OfflineService + offlineBroadcastService: OfflineBroadcastService ) { // Subscribe to the current user. operator.getViewUserObservable().subscribe(user => { @@ -79,13 +79,10 @@ export class OverlayService { this.checkConnection(); }); // Subscribe to check if we are offline - offlineService - .isOffline() - .pipe(distinctUntilChanged()) - .subscribe(offline => { - this.isOffline = offline; - this.checkConnection(); - }); + offlineBroadcastService.isOfflineObservable.pipe(distinctUntilChanged()).subscribe(offline => { + this.isOffline = offline; + this.checkConnection(); + }); } /** diff --git a/client/src/app/shared/components/copyright-sign/copyright-sign.component.ts b/client/src/app/shared/components/copyright-sign/copyright-sign.component.ts index 84cf0d2bc..88d69c6d6 100644 --- a/client/src/app/shared/components/copyright-sign/copyright-sign.component.ts +++ b/client/src/app/shared/components/copyright-sign/copyright-sign.component.ts @@ -211,15 +211,15 @@ export class C4DialogComponent implements OnInit, OnDestroy { search: { recievedSearchRequest: { handle: (notify: NotifyResponse<{ name: string }>) => { - this.replyChannel = notify.senderChannelName; - this.partnerName = notify.content.name; + this.replyChannel = notify.sender_channel_id; + this.partnerName = notify.message.name; return 'waitForResponse'; } }, recievedSearchResponse: { handle: (notify: NotifyResponse<{ name: string }>) => { - this.replyChannel = notify.senderChannelName; - this.partnerName = notify.content.name; + this.replyChannel = notify.sender_channel_id; + this.partnerName = notify.message.name; // who starts? const startPlayer = Math.random() < 0.5 ? Player.thisPlayer : Player.partner; const startPartner: boolean = startPlayer === Player.partner; @@ -232,10 +232,10 @@ export class C4DialogComponent implements OnInit, OnDestroy { waitForResponse: { recievedACK: { handle: (notify: NotifyResponse<{}>) => { - if (notify.senderChannelName !== this.replyChannel) { + if (notify.sender_channel_id !== this.replyChannel) { return null; } - return notify.content ? 'myTurn' : 'foreignTurn'; + return notify.message ? 'myTurn' : 'foreignTurn'; } }, waitTimeout: { @@ -243,7 +243,7 @@ export class C4DialogComponent implements OnInit, OnDestroy { }, recievedRagequit: { handle: (notify: NotifyResponse<{}>) => { - return notify.senderChannelName === this.replyChannel ? 'search' : null; + return notify.sender_channel_id === this.replyChannel ? 'search' : null; } } }, @@ -270,10 +270,10 @@ export class C4DialogComponent implements OnInit, OnDestroy { foreignTurn: { recievedTurn: { handle: (notify: NotifyResponse<{ col: number }>) => { - if (notify.senderChannelName !== this.replyChannel) { + if (notify.sender_channel_id !== this.replyChannel) { return null; } - const col: number = notify.content.col; + const col: number = notify.message.col; if (!this.colFree(col)) { return null; } @@ -455,7 +455,7 @@ export class C4DialogComponent implements OnInit, OnDestroy { */ public enter_waitForResponse(): void { this.caption = 'Wait for response...'; - this.notifyService.send('c4_search_response', { name: this.getPlayerName() }); + this.notifyService.sendToChannels('c4_search_response', { name: this.getPlayerName() }, this.replyChannel); if (this.waitTimout) { clearTimeout(this.waitTimout); } diff --git a/client/src/app/shared/components/list-view-table/list-view-table.component.ts b/client/src/app/shared/components/list-view-table/list-view-table.component.ts index 6ce8a1fe9..969beb22c 100644 --- a/client/src/app/shared/components/list-view-table/list-view-table.component.ts +++ b/client/src/app/shared/components/list-view-table/list-view-table.component.ts @@ -28,6 +28,7 @@ import { ViewportService } from 'app/core/ui-services/viewport.service'; import { BaseProjectableViewModel } from 'app/site/base/base-projectable-view-model'; import { BaseViewModel } from 'app/site/base/base-view-model'; import { BaseViewModelWithContentObject } from 'app/site/base/base-view-model-with-content-object'; +import { isProjectable } from 'app/site/base/projectable'; export interface CssClassDefinition { [key: string]: boolean; @@ -458,8 +459,8 @@ export class ListViewTableComponent) => { - const model = context.$implicit as V; - if (this.allowProjector && this.projectorService.isProjected(this.getProjectable(model))) { + const projectableViewModel = this.getProjectable(context.$implicit as V); + if (projectableViewModel && this.allowProjector && this.projectorService.isProjected(projectableViewModel)) { return 'projected'; } }; @@ -578,8 +579,10 @@ export class ListViewTableComponent (this.isOffline = isOffline)); + this.offlineSubscription = this.offlineBroadcastService.isOfflineObservable.subscribe( + isOffline => (this.isOffline = isOffline) + ); } /** diff --git a/client/src/app/site/common/components/start/start.component.html b/client/src/app/site/common/components/start/start.component.html index ce3b12ee1..ead7e0a93 100644 --- a/client/src/app/site/common/components/start/start.component.html +++ b/client/src/app/site/common/components/start/start.component.html @@ -13,6 +13,7 @@ +

{{ startContent.general_event_welcome_title | translate }}

diff --git a/client/src/app/site/motions/modules/motion-detail/components/motion-detail/motion-detail.component.ts b/client/src/app/site/motions/modules/motion-detail/components/motion-detail/motion-detail.component.ts index 6b57e644a..95aaee312 100644 --- a/client/src/app/site/motions/modules/motion-detail/components/motion-detail/motion-detail.component.ts +++ b/client/src/app/site/motions/modules/motion-detail/components/motion-detail/motion-detail.component.ts @@ -1639,7 +1639,7 @@ export class MotionDetailComponent extends BaseViewComponentDirective implements */ private listenToEditNotification(): Subscription { return this.notifyService.getMessageObservable(this.NOTIFICATION_EDIT_MOTION).subscribe(message => { - const content = message.content; + const content = message.message; if (this.operator.viewUser.id !== content.senderId && content.motionId === this.motion.id) { let warning = ''; @@ -1656,7 +1656,7 @@ export class MotionDetailComponent extends BaseViewComponentDirective implements if (content.type === MotionEditNotificationType.TYPE_BEGIN_EDITING_MOTION) { this.sendEditNotification( MotionEditNotificationType.TYPE_ALSO_EDITING_MOTION, - message.senderUserId + message.sender_user_id ); } break; diff --git a/client/src/app/site/polls/components/base-poll-dialog.component.ts b/client/src/app/site/polls/components/base-poll-dialog.component.ts index cbc4fc129..2d5d223b8 100644 --- a/client/src/app/site/polls/components/base-poll-dialog.component.ts +++ b/client/src/app/site/polls/components/base-poll-dialog.component.ts @@ -58,8 +58,6 @@ export abstract class BasePollDialogComponent< votes: this.getVoteData(), publish_immediately: this.publishImmediately }; - console.log('answer: ', answer); - this.dialogRef.close(answer); } diff --git a/client/src/app/site/site.component.ts b/client/src/app/site/site.component.ts index 649946bb4..9583fc41c 100644 --- a/client/src/app/site/site.component.ts +++ b/client/src/app/site/site.component.ts @@ -10,7 +10,7 @@ import { TranslateService } from '@ngx-translate/core'; import { filter } from 'rxjs/operators'; import { navItemAnim } from '../shared/animations'; -import { OfflineService } from 'app/core/core-services/offline.service'; +import { OfflineBroadcastService } from 'app/core/core-services/offline-broadcast.service'; import { OverlayService } from 'app/core/ui-services/overlay.service'; import { UpdateService } from 'app/core/ui-services/update.service'; import { BaseComponent } from '../base.component'; @@ -84,7 +84,7 @@ export class SiteComponent extends BaseComponent implements OnInit { public constructor( title: Title, protected translate: TranslateService, - offlineService: OfflineService, + offlineBroadcastService: OfflineBroadcastService, private updateService: UpdateService, private router: Router, public operator: OperatorService, @@ -99,7 +99,7 @@ export class SiteComponent extends BaseComponent implements OnInit { super(title, translate); overlayService.showSpinner(translate.instant('Loading data. Please wait ...')); - offlineService.isOffline().subscribe(offline => { + offlineBroadcastService.isOfflineObservable.subscribe(offline => { this.isOffline = offline; }); diff --git a/dc-dev.sh b/dc-dev.sh new file mode 100755 index 000000000..9cb85da85 --- /dev/null +++ b/dc-dev.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd "$(dirname $0)" +docker-compose -f docker/docker-compose.dev.yml $@ diff --git a/docker/.env b/docker/.env index c961d7095..f56a7978e 100644 --- a/docker/.env +++ b/docker/.env @@ -17,10 +17,14 @@ DEFAULT_DOCKER_REGISTRY= # Docker Images # ------------- +DOCKER_OPENSLIDES_HAPROXY_NAME= +DOCKER_OPENSLIDES_HAPROXY_TAG= DOCKER_OPENSLIDES_BACKEND_NAME= DOCKER_OPENSLIDES_BACKEND_TAG= DOCKER_OPENSLIDES_FRONTEND_NAME= DOCKER_OPENSLIDES_FRONTEND_TAG= +DOCKER_OPENSLIDES_AUTOUPDATE_NAME= +DOCKER_OPENSLIDES_AUTOUPDATE_TAG= # Database # -------- @@ -37,6 +41,7 @@ PGBOUNCER_PLACEMENT_CONSTR= # ------------------- OPENSLIDES_BACKEND_SERVICE_REPLICAS= OPENSLIDES_FRONTEND_SERVICE_REPLICAS= +OPENSLIDES_AUTOUPDATE_SERVICE_REPLICAS= REDIS_RO_SERVICE_REPLICAS= MEDIA_SERVICE_REPLICAS= diff --git a/docker/build.sh b/docker/build.sh index 5a4f5d864..7e7771b23 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -6,7 +6,9 @@ declare -A TARGETS TARGETS=( [client]="$(dirname "${BASH_SOURCE[0]}")/../client/docker/" [server]="$(dirname "${BASH_SOURCE[0]}")/../server/docker/" - [media-service]="https://github.com/OpenSlides/openslides-media-service.git" + [haproxy]="$(dirname "${BASH_SOURCE[0]}")/../haproxy/" + [autoupdate]="$(dirname "${BASH_SOURCE[0]}")/../autoupdate/" + [media]="https://github.com/OpenSlides/openslides-media-service.git" [pgbouncer]="https://github.com/OpenSlides/openslides-docker-compose.git#:pgbouncer" [postfix]="https://github.com/OpenSlides/openslides-docker-compose.git#:postfix" [repmgr]="https://github.com/OpenSlides/openslides-docker-compose.git#:repmgr" @@ -17,7 +19,7 @@ DOCKER_TAG="latest" CONFIG="/etc/osinstancectl" OPTIONS=() BUILT_IMAGES=() -DEFAULT_TARGETS=(server client) +DEFAULT_TARGETS=(server client haproxy autoupdate) usage() { cat << EOF diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 000000000..78b45db15 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,47 @@ +version: "3" +services: + client: + image: os3-client-dev + build: + context: ../client + dockerfile: docker/Dockerfile.dev + volumes: + - ../client:/app + + server: + image: os3-server-dev + user: $UID:$GID + build: + context: ../server + dockerfile: docker/Dockerfile.dev + volumes: + - ../server:/app + depends_on: + - redis + + autoupdate: + image: os3-autoupdate-dev + environment: + - MESSAGE_BUS_HOST=redis + - MESSAGE_BUS_PORT=6379 + - WORKER_HOST=server + - WORKER_PORT=8000 + depends_on: + - server + - redis + volumes: + - ../autoupdate/cmd:/root/cmd + - ../autoupdate/internal:/root/internal + redis: + image: redis:latest + + haproxy: + image: os3-haproxy-dev + volumes: + - ../haproxy/src:/usr/local/etc/haproxy + depends_on: + - client + - server + - autoupdate + ports: + - "8000:8000" \ No newline at end of file diff --git a/docker/docker-compose.yml.m4 b/docker/docker-compose.yml.m4 index 58eea725c..37ef1ebe0 100644 --- a/docker/docker-compose.yml.m4 +++ b/docker/docker-compose.yml.m4 @@ -13,12 +13,22 @@ define(`read_env', `esyscmd(`printf "\`%s'" "$$1"')') dnl return env variable if set; otherwise, return given alternative value define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))') +define(`HAPROXY_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_HAPROXY_NAME', openslides-haproxy):dnl +ifenvelse(`DOCKER_OPENSLIDES_HAPROXY_TAG', latest)) define(`BACKEND_IMAGE', -ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides/openslides-server):dnl +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides-server):dnl ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest)) define(`FRONTEND_IMAGE', -ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides/openslides-client):dnl +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides-client):dnl ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_TAG', latest)) +define(`AUTOUPDATE_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_NAME', openslides-autoupdate):dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest)) define(`PRIMARY_DB', `ifenvelse(`PGNODE_REPMGR_PRIMARY', pgnode1)') @@ -41,11 +51,9 @@ x-osserver: &default-osserver image: BACKEND_IMAGE networks: - - front - back restart: always x-osserver-env: &default-osserver-env - AMOUNT_REPLICAS: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 1) AUTOUPDATE_DELAY: ifenvelse(`AUTOUPDATE_DELAY', 1) DEMO_USERS: "ifenvelse(`DEMO_USERS',)" CONNECTION_POOL_LIMIT: ifenvelse(`CONNECTION_POOL_LIMIT', 100) @@ -54,7 +62,6 @@ x-osserver-env: &default-osserver-env DATABASE_PORT: ifenvelse(`DATABASE_PORT', 5432) DATABASE_USER: "ifenvelse(`DATABASE_USER', openslides)" DEFAULT_FROM_EMAIL: "ifenvelse(`DEFAULT_FROM_EMAIL', noreply@example.com)" - DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)" EMAIL_HOST: "ifenvelse(`EMAIL_HOST', postfix)" EMAIL_HOST_PASSWORD: "ifenvelse(`EMAIL_HOST_PASSWORD',)" EMAIL_HOST_USER: "ifenvelse(`EMAIL_HOST_USER',)" @@ -69,13 +76,11 @@ x-osserver-env: &default-osserver-env JITSI_ROOM_PASSWORD: "ifenvelse(`JITSI_ROOM_PASSWORD',)" JITSI_ROOM_NAME: "ifenvelse(`JITSI_ROOM_NAME',)" OPENSLIDES_LOG_LEVEL: "ifenvelse(`OPENSLIDES_LOG_LEVEL', INFO)" - REDIS_CHANNLES_HOST: "ifenvelse(`REDIS_CHANNLES_HOST', redis-channels)" - REDIS_CHANNLES_PORT: ifenvelse(`REDIS_CHANNLES_PORT', 6379) + DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)" REDIS_HOST: "ifenvelse(`REDIS_HOST', redis)" REDIS_PORT: ifenvelse(`REDIS_PORT', 6379) REDIS_SLAVE_HOST: "ifenvelse(`REDIS_SLAVE_HOST', redis-slave)" REDIS_SLAVE_PORT: ifenvelse(`REDIS_SLAVE_PORT', 6379) - REDIS_SLAVE_WAIT_TIMEOUT: ifenvelse(`REDIS_SLAVE_WAIT_TIMEOUT', 10000) RESET_PASSWORD_VERBOSE_ERRORS: "ifenvelse(`RESET_PASSWORD_VERBOSE_ERRORS', False)" x-pgnode: &default-pgnode image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-repmgr:latest @@ -90,15 +95,27 @@ x-pgnode-env: &default-pgnode-env REPMGR_WAL_ARCHIVE: "ifenvelse(`PGNODE_WAL_ARCHIVING', on)" services: + haproxy: + image: HAPROXY_IMAGE + depends_on: + - server + - client + - autoupdate + - media + networks: + - front + - back + ports: + - "127.0.0.1:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):8000" + server: << : *default-osserver # Below is the default command. You can uncomment it to override the # number of workers, for example: - # command: "gunicorn -w 8 --preload -b 0.0.0.0:8000 - # -k uvicorn.workers.UvicornWorker openslides.asgi:application" + # command: "gunicorn -w 8 --preload -b 0.0.0.0:8000 openslides.wsgi" # # Uncomment the following line to use daphne instead of gunicorn: - # command: "daphne -b 0.0.0.0 -p 8000 openslides.asgi:application" + # command: "daphne -b 0.0.0.0 -p 8000 openslides.wsgi" depends_on: - server-setup environment: @@ -127,17 +144,24 @@ services: - pgbouncer - redis - redis-slave - - redis-channels client: image: FRONTEND_IMAGE restart: always - depends_on: - - server networks: - - front - ports: - - "127.0.0.1:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):80" + - back + + autoupdate: + image: AUTOUPDATE_IMAGE + restart: always + depends_on: + - redis + - server + environment: + MESSAGE_BUS_HOST: redis + WORKER_HOST: server + networks: + - back pgnode1: << : *default-pgnode @@ -189,9 +213,7 @@ ifelse(read_env(`PGNODE_3_ENABLED'), 1, `' image: redis:alpine restart: always networks: - back: - aliases: - - rediscache + - back redis-slave: image: redis:alpine restart: always @@ -199,18 +221,11 @@ ifelse(read_env(`PGNODE_3_ENABLED'), 1, `' depends_on: - redis networks: - back: - aliases: - - rediscache-slave + - back ifelse(read_env(`REDIS_RO_SERVICE_REPLICAS'),,,deploy: replicas: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 1)) - redis-channels: - image: redis:alpine - restart: always - networks: - back: media: - image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media-service:latest + image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media:latest environment: - CHECK_REQUEST_URL=server:8000/check-media/ - CACHE_SIZE=ifenvelse(`CACHE_SIZE', 10) @@ -218,8 +233,7 @@ ifelse(read_env(`PGNODE_3_ENABLED'), 1, `' - CACHE_DATA_MAX_SIZE_KB=ifenvelse(`CACHE_DATA_MAX_SIZE_KB', 10240) restart: always networks: - front: - back: + - back # Override command to run more workers per task # command: ["gunicorn", "-w", "4", "--preload", "-b", # "0.0.0.0:8000", "src.mediaserver:app"] diff --git a/docker/docker-stack.yml.m4 b/docker/docker-stack.yml.m4 index da87417a6..0d83990cb 100644 --- a/docker/docker-stack.yml.m4 +++ b/docker/docker-stack.yml.m4 @@ -13,12 +13,22 @@ define(`read_env', `esyscmd(`printf "\`%s'" "$$1"')') dnl return env variable if set; otherwise, return given alternative value define(`ifenvelse', `ifelse(read_env(`$1'),, `$2', read_env(`$1'))') +define(`HAPROXY_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_HAPROXY_NAME', openslides-haproxy):dnl +ifenvelse(`DOCKER_OPENSLIDES_HAPROXY_TAG', latest)) define(`BACKEND_IMAGE', -ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides/openslides-server):dnl +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_BACKEND_NAME', openslides-server):dnl ifenvelse(`DOCKER_OPENSLIDES_BACKEND_TAG', latest)) define(`FRONTEND_IMAGE', -ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides/openslides-client):dnl +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_NAME', openslides-client):dnl ifenvelse(`DOCKER_OPENSLIDES_FRONTEND_TAG', latest)) +define(`AUTOUPDATE_IMAGE', +ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_NAME', openslides-autoupdate):dnl +ifenvelse(`DOCKER_OPENSLIDES_AUTOUPDATE_TAG', latest)) define(`PRIMARY_DB', `ifenvelse(`PGNODE_REPMGR_PRIMARY', pgnode1)') @@ -41,10 +51,8 @@ x-osserver: &default-osserver image: BACKEND_IMAGE networks: - - front - back x-osserver-env: &default-osserver-env - AMOUNT_REPLICAS: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3) AUTOUPDATE_DELAY: ifenvelse(`AUTOUPDATE_DELAY', 1) DEMO_USERS: "ifenvelse(`DEMO_USERS',)" CONNECTION_POOL_LIMIT: ifenvelse(`CONNECTION_POOL_LIMIT', 100) @@ -53,7 +61,6 @@ x-osserver-env: &default-osserver-env DATABASE_PORT: ifenvelse(`DATABASE_PORT', 5432) DATABASE_USER: "ifenvelse(`DATABASE_USER', openslides)" DEFAULT_FROM_EMAIL: "ifenvelse(`DEFAULT_FROM_EMAIL', noreply@example.com)" - DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)" EMAIL_HOST: "ifenvelse(`EMAIL_HOST', postfix)" EMAIL_HOST_PASSWORD: "ifenvelse(`EMAIL_HOST_PASSWORD',)" EMAIL_HOST_USER: "ifenvelse(`EMAIL_HOST_USER',)" @@ -68,13 +75,11 @@ x-osserver-env: &default-osserver-env JITSI_ROOM_PASSWORD: "ifenvelse(`JITSI_ROOM_PASSWORD',)" JITSI_ROOM_NAME: "ifenvelse(`JITSI_ROOM_NAME',)" OPENSLIDES_LOG_LEVEL: "ifenvelse(`OPENSLIDES_LOG_LEVEL', INFO)" - REDIS_CHANNLES_HOST: "ifenvelse(`REDIS_CHANNLES_HOST', redis-channels)" - REDIS_CHANNLES_PORT: ifenvelse(`REDIS_CHANNLES_PORT', 6379) + DJANGO_LOG_LEVEL: "ifenvelse(`DJANGO_LOG_LEVEL', INFO)" REDIS_HOST: "ifenvelse(`REDIS_HOST', redis)" REDIS_PORT: ifenvelse(`REDIS_PORT', 6379) REDIS_SLAVE_HOST: "ifenvelse(`REDIS_SLAVE_HOST', redis-slave)" REDIS_SLAVE_PORT: ifenvelse(`REDIS_SLAVE_PORT', 6379) - REDIS_SLAVE_WAIT_TIMEOUT: ifenvelse(`REDIS_SLAVE_WAIT_TIMEOUT', 10000) RESET_PASSWORD_VERBOSE_ERRORS: "ifenvelse(`RESET_PASSWORD_VERBOSE_ERRORS', False)" x-pgnode: &default-pgnode image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-repmgr:latest @@ -90,6 +95,18 @@ x-pgnode-env: &default-pgnode-env REPMGR_WAL_ARCHIVE: "ifenvelse(`PGNODE_WAL_ARCHIVING', on)" services: + haproxy: + image: HAPROXY_IMAGE + networks: + - front + - back + ports: + - "0.0.0.0:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):8000" + deploy: + restart_policy: + condition: on-failure + delay: 5s + server: << : *default-osserver # Below is the default command. You can uncomment it to override the @@ -128,15 +145,26 @@ services: client: image: FRONTEND_IMAGE networks: - - front - ports: - - "0.0.0.0:ifenvelse(`EXTERNAL_HTTP_PORT', 8000):80" + - back deploy: replicas: ifenvelse(`OPENSLIDES_FRONTEND_SERVICE_REPLICAS', 1) restart_policy: condition: on-failure delay: 5s + autoupdate: + image: AUTOUPDATE_IMAGE + environment: + MESSAGE_BUS_HOST: redis + WORKER_HOST: server + networks: + - back + deploy: + replicas: ifenvelse(`OPENSLIDES_AUTOUPDATE_SERVICE_REPLICAS', 1) + restart_policy: + condition: on-failure + delay: 5s + pgnode1: << : *default-pgnode environment: @@ -206,9 +234,7 @@ ifelse(read_env(`PGNODE_3_ENABLED'), 1, `' redis: image: redis:alpine networks: - back: - aliases: - - rediscache + - back deploy: replicas: 1 restart_policy: @@ -218,38 +244,26 @@ ifelse(read_env(`PGNODE_3_ENABLED'), 1, `' image: redis:alpine command: ["redis-server", "--save", "", "--slaveof", "redis", "6379"] networks: - back: - aliases: - - rediscache-slave + - back deploy: - replicas: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 3) - restart_policy: - condition: on-failure - delay: 5s - redis-channels: - image: redis:alpine - networks: - back: - deploy: - replicas: 1 + replicas: ifenvelse(`REDIS_RO_SERVICE_REPLICAS', 1) restart_policy: condition: on-failure delay: 5s media: - image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media-service:latest + image: ifenvelse(`DEFAULT_DOCKER_REGISTRY', openslides)/openslides-media:latest environment: - CHECK_REQUEST_URL=server:8000/check-media/ - CACHE_SIZE=ifenvelse(`CACHE_SIZE', 10) - CACHE_DATA_MIN_SIZE_KB=ifenvelse(`CACHE_DATA_MIN_SIZE_KB', 0) - CACHE_DATA_MAX_SIZE_KB=ifenvelse(`CACHE_DATA_MAX_SIZE_KB', 10240) deploy: - replicas: ifenvelse(`MEDIA_SERVICE_REPLICAS', 8) + replicas: ifenvelse(`MEDIA_SERVICE_REPLICAS', 2) restart_policy: condition: on-failure delay: 10s networks: - front: - back: + - back # Override command to run more workers per task # command: ["gunicorn", "-w", "4", "--preload", "-b", # "0.0.0.0:8000", "src.mediaserver:app"] diff --git a/haproxy/Dockerfile b/haproxy/Dockerfile new file mode 100644 index 000000000..2505e8fb5 --- /dev/null +++ b/haproxy/Dockerfile @@ -0,0 +1,5 @@ +FROM haproxy:2.0-alpine +COPY src/haproxy.common.cfg /usr/local/etc/haproxy/haproxy.common.cfg +COPY src/haproxy.prod.cfg /usr/local/etc/haproxy/haproxy.prod.cfg +COPY src/combined.pem /usr/local/etc/haproxy/combined.pem +CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.common.cfg", "-f", "/usr/local/etc/haproxy/haproxy.prod.cfg"] \ No newline at end of file diff --git a/haproxy/Dockerfile.dev b/haproxy/Dockerfile.dev new file mode 100644 index 000000000..bdd6e35b2 --- /dev/null +++ b/haproxy/Dockerfile.dev @@ -0,0 +1,5 @@ +FROM haproxy:2.0-alpine +COPY src/haproxy.common.cfg /usr/local/etc/haproxy/haproxy.common.cfg +COPY src/haproxy.dev.cfg /usr/local/etc/haproxy/haproxy.dev.cfg +COPY src/combined.pem /usr/local/etc/haproxy/combined.pem +CMD ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.common.cfg", "-f", "/usr/local/etc/haproxy/haproxy.dev.cfg"] \ No newline at end of file diff --git a/haproxy/Makefile b/haproxy/Makefile new file mode 100644 index 000000000..08107a1a5 --- /dev/null +++ b/haproxy/Makefile @@ -0,0 +1,3 @@ +build-dev: + ./prepare-cert.sh + docker build -t os3-haproxy-dev -f Dockerfile.dev . diff --git a/haproxy/build.sh b/haproxy/build.sh new file mode 100755 index 000000000..1a73eb1dc --- /dev/null +++ b/haproxy/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" +./prepare-cert.sh +docker build --tag "${img:-openslides/openslides-${service_name}:latest}" \ + --pull "${OPTIONS[@]}" . diff --git a/haproxy/prepare-cert.sh b/haproxy/prepare-cert.sh new file mode 100755 index 000000000..62d6f2037 --- /dev/null +++ b/haproxy/prepare-cert.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e +cd "$(dirname "$0")" + +# check, if we already generated a cert +combined="src/combined.pem" + +if [[ ! -f $combined ]]; then + echo "Creating certificates..." + cd src + if type 2>&1 >/dev/null openssl ; then + echo "Using openssl to generate a certificate." + echo "You will need to accept an security exception for the" + echo "generated certificate in your browser manually." + openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \ + -subj "/C=DE/O=Selfsigned Test/CN=localhost" \ + -keyout localhost-key.pem -out localhost.pem + else + echo >&2 "FATAL: No valid certificate generation tool found!" + exit -1 + fi + cat localhost.pem localhost-key.pem > combined.pem + echo "done" +else + echo "Certificate exists." +fi \ No newline at end of file diff --git a/haproxy/src/haproxy.common.cfg b/haproxy/src/haproxy.common.cfg new file mode 100644 index 000000000..6e764bd5f --- /dev/null +++ b/haproxy/src/haproxy.common.cfg @@ -0,0 +1,31 @@ +global + log stdout format raw local0 debug + +defaults + option http-use-htx + option dontlognull + timeout connect 3s + timeout client 10s + timeout client-fin 10s + timeout server 10s + timeout server-fin 10s + timeout check 2s + timeout tunnel 10s + timeout queue 2s + log global + option httplog + +resolvers docker_resolver + nameserver dns 127.0.0.11:53 + +backend backend_server + mode http + # Do not pass the auth-header from /stats to OS. It confuses the server... + http-request del-header authorization + timeout server 4m + server server server:8000 resolvers docker_resolver check alpn http/1.1 + +backend backend_autoupdate + mode http + timeout server 1h + server autoupdate autoupdate:8002 resolvers docker_resolver check ssl verify none alpn h2 diff --git a/haproxy/src/haproxy.dev.cfg b/haproxy/src/haproxy.dev.cfg new file mode 100644 index 000000000..f8c2e19d4 --- /dev/null +++ b/haproxy/src/haproxy.dev.cfg @@ -0,0 +1,20 @@ +frontend https + mode http + bind *:8000 ssl crt /usr/local/etc/haproxy/combined.pem alpn h2,http/1.1 + default_backend backend_client + + acl autoupdate path_beg -i /system + use_backend backend_autoupdate if autoupdate + + acl server path_beg -i /apps /media/ /rest /server-version.txt + use_backend backend_server if server + + stats enable + stats uri /stats + stats refresh 10s + stats auth admin:admin + +backend backend_client + mode http + timeout tunnel 1h + server client client:4200 resolvers docker_resolver no-check diff --git a/haproxy/src/haproxy.prod.cfg b/haproxy/src/haproxy.prod.cfg new file mode 100644 index 000000000..599c800f7 --- /dev/null +++ b/haproxy/src/haproxy.prod.cfg @@ -0,0 +1,26 @@ +frontend https + mode http + bind *:8000 ssl crt /usr/local/etc/haproxy/combined.pem alpn h2,http/1.1 + default_backend backend_client + + acl autoupdate path_beg -i /system + use_backend backend_autoupdate if autoupdate + + acl server path_beg -i /apps /rest /server-version.txt + use_backend backend_server if server + + acl media path_beg -i /media/ + use_backend backend_media if media + + stats enable + stats uri /stats + stats refresh 10s + stats auth admin:admin + +backend backend_client + mode http + server client client:80 resolvers docker_resolver check + +backend backend_media + mode http + server media media:8000 resolvers docker_resolver check diff --git a/server/.dockerignore b/server/.dockerignore index 3c9b9fc6f..cc9c6fd8f 100644 --- a/server/.dockerignore +++ b/server/.dockerignore @@ -3,3 +3,6 @@ **/.venv tests/ personal_data/ +**/.mypy_cache +**/.pytest_cache +**/tests/file diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 7a5f266dd..5fd29b856 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -70,5 +70,4 @@ COPY manage.py /app/ COPY openslides /app/openslides COPY docker/server-version.txt /app/openslides/core/static/server-version.txt ENTRYPOINT ["/usr/local/sbin/entrypoint"] -CMD ["gunicorn", "-w", "8", "--preload", "-b", "0.0.0.0:8000", "-k", \ - "uvicorn.workers.UvicornWorker", "openslides.asgi:application"] +CMD ["gunicorn", "-w", "8", "--preload", "-t", "240", "-b", "0.0.0.0:8000", "openslides.wsgi"] diff --git a/server/docker/Dockerfile.dev b/server/docker/Dockerfile.dev new file mode 100644 index 000000000..b5318d528 --- /dev/null +++ b/server/docker/Dockerfile.dev @@ -0,0 +1,30 @@ +FROM python:3.7-slim AS base + +# Variables relevant for CMD +ENV DJANGO_SETTINGS_MODULE settings +ENV PYTHONPATH personal_data/var/ + +WORKDIR /app +RUN apt-get -y update && apt-get install --no-install-recommends -y \ + postgresql-client \ + wait-for-it \ + gcc \ + git \ + libxml2-dev \ + libxmlsec1-dev \ + libxmlsec1-openssl \ + pkg-config + +RUN rm -rf /var/lib/apt/lists/* + +COPY requirements /app/requirements +COPY requirements.txt /app/requirements.txt + +RUN pip install -r requirements.txt -r requirements/saml.txt && \ + rm -rf /root/.cache/pip + +EXPOSE 8000 +COPY docker/entrypoint-dev /usr/local/sbin/ +COPY . . +ENTRYPOINT ["/usr/local/sbin/entrypoint-dev"] +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] diff --git a/server/docker/entrypoint-db-setup b/server/docker/entrypoint-db-setup index 5f96c6a59..155c2c703 100755 --- a/server/docker/entrypoint-db-setup +++ b/server/docker/entrypoint-db-setup @@ -37,7 +37,6 @@ done # Wait for redis wait-for-it redis:6379 wait-for-it redis-slave:6379 -wait-for-it redis-channels:6379 echo 'running migrations' python manage.py migrate diff --git a/server/docker/entrypoint-dev b/server/docker/entrypoint-dev new file mode 100755 index 000000000..94833dd72 --- /dev/null +++ b/server/docker/entrypoint-dev @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +wait-for-it -t 0 redis:6379 + +if [[ ! -f "/app/personal_data/var/settings.py" ]]; then + echo "Create settings" + python manage.py createsettings +fi + +python -u manage.py migrate + +exec "$@" diff --git a/server/docker/settings.py b/server/docker/settings.py index 722982eab..b69274321 100644 --- a/server/docker/settings.py +++ b/server/docker/settings.py @@ -54,7 +54,7 @@ DEBUG = False RESET_PASSWORD_VERBOSE_ERRORS = get_env("RESET_PASSWORD_VERBOSE_ERRORS", True, bool) # OpenSlides specific settings -AUTOUPDATE_DELAY = get_env("AUTOUPDATE_DELAY", 1, int) +AUTOUPDATE_DELAY = get_env("AUTOUPDATE_DELAY", 1, float) DEMO_USERS = get_env("DEMO_USERS", default=None) DEMO_USERS = json.loads(DEMO_USERS) if DEMO_USERS else None @@ -99,25 +99,10 @@ REDIS_HOST = get_env("REDIS_HOST", "redis") REDIS_PORT = get_env("REDIS_PORT", 6379, int) REDIS_SLAVE_HOST = get_env("REDIS_SLAVE_HOST", "redis-slave") REDIS_SLAVE_PORT = get_env("REDIS_SLAVE_PORT", 6379, int) -REDIS_CHANNLES_HOST = get_env("REDIS_CHANNLES_HOST", "redis-channels") -REDIS_CHANNLES_PORT = get_env("REDIS_CHANNLES_PORT", 6379, int) -REDIS_SLAVE_WAIT_TIMEOUT = get_env("REDIS_SLAVE_WAIT_TIMEOUT", 10000, int) - -# Django Channels -CHANNEL_LAYERS = { - "default": { - "BACKEND": "channels_redis.core.RedisChannelLayer", - "CONFIG": { - "hosts": [(REDIS_CHANNLES_HOST, REDIS_CHANNLES_PORT)], - "capacity": 10000, - }, - }, -} # Collection Cache REDIS_ADDRESS = f"redis://{REDIS_HOST}:{REDIS_PORT}/0" REDIS_READ_ONLY_ADDRESS = f"redis://{REDIS_SLAVE_HOST}:{REDIS_SLAVE_PORT}/0" -AMOUNT_REPLICAS = get_env("AMOUNT_REPLICAS", 1, int) CONNECTION_POOL_LIMIT = get_env("CONNECTION_POOL_LIMIT", 100, int) # Session backend @@ -135,8 +120,6 @@ ENABLE_SAML = get_env("ENABLE_SAML", False, bool) if ENABLE_SAML: INSTALLED_APPS += ["openslides.saml"] -# TODO: More saml stuff... - # Controls if electronic voting (means non-analog polls) are enabled. ENABLE_ELECTRONIC_VOTING = get_env("ENABLE_ELECTRONIC_VOTING", False, bool) diff --git a/server/openslides/agenda/apps.py b/server/openslides/agenda/apps.py index 00d358838..a7813ebdb 100644 --- a/server/openslides/agenda/apps.py +++ b/server/openslides/agenda/apps.py @@ -15,7 +15,6 @@ class AgendaAppConfig(AppConfig): from ..utils.access_permissions import required_user from ..utils.rest_api import router from . import serializers # noqa - from .projector import register_projector_slides from .signals import ( get_permission_change_data, listen_to_related_object_post_delete, @@ -23,9 +22,6 @@ class AgendaAppConfig(AppConfig): ) from .views import ItemViewSet, ListOfSpeakersViewSet - # Define projector elements. - register_projector_slides() - # Connect signals. post_save.connect( listen_to_related_object_post_save, diff --git a/server/openslides/agenda/projector.py b/server/openslides/agenda/projector.py deleted file mode 100644 index c7ffa0f7d..000000000 --- a/server/openslides/agenda/projector.py +++ /dev/null @@ -1,309 +0,0 @@ -from collections import defaultdict -from typing import Any, Dict, List, Union - -from ..users.projector import get_user_name -from ..utils.projector import ( - ProjectorAllDataProvider, - ProjectorElementException, - get_config, - get_model, - register_projector_slide, -) - - -# Important: All functions have to be prune. This means, that thay can only -# access the data, that they get as argument and do not have any -# side effects. - - -async def get_sorted_agenda_items( - agenda_items: Dict[int, Dict[str, Any]] -) -> List[Dict[str, Any]]: - """ - Returns all sorted agenda items by id first and then weight, resulting in - ordered items, if some have the same weight. - """ - return sorted( - sorted(agenda_items.values(), key=lambda item: item["id"]), - key=lambda item: item["weight"], - ) - - -async def get_flat_tree( - agenda_items: Dict[int, Dict[str, Any]], parent_id: int = 0 -) -> List[Dict[str, Any]]: - """ - Build the item tree from all_data_provider. - - Only build the tree from elements unterneath parent_id. - - Returns a list of two element tuples where the first element is the item title - and the second a List with children as two element tuples. - """ - - # Build a dict from an item_id to all its children - children: Dict[int, List[int]] = defaultdict(list) - - for item in await get_sorted_agenda_items(agenda_items): - if item["type"] == 1: # only normal items - children[item["parent_id"] or 0].append(item["id"]) - - tree = [] - - def build_tree(item_ids: List[int], depth: int) -> None: - for item_id in item_ids: - item = agenda_items[item_id] - title_information = item["title_information"] - title_information["_agenda_item_number"] = item["item_number"] - tree.append( - { - "title_information": title_information, - "collection": item["content_object"]["collection"], - "depth": depth, - } - ) - build_tree(children[item_id], depth + 1) - - build_tree(children[parent_id], 0) - return tree - - -async def item_list_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Item list slide. - - Returns all root items or all children of an item. - """ - # fetch all items, so they are cached: - all_agenda_items = await all_data_provider.get_collection("agenda/item") - - only_main_items = element.get("only_main_items", True) - if only_main_items: - agenda_items = [] - for item in await get_sorted_agenda_items(all_agenda_items): - if item["parent_id"] is None and item["type"] == 1: - title_information = item["title_information"] - title_information["_agenda_item_number"] = item["item_number"] - agenda_items.append( - { - "title_information": title_information, - "collection": item["content_object"]["collection"], - } - ) - else: - agenda_items = await get_flat_tree(all_agenda_items) - - return {"items": agenda_items} - - -async def list_of_speakers_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - List of speakers slide. - - Returns all usernames, that are on the list of speaker of a slide. - """ - list_of_speakers_id = element.get("id") - - if list_of_speakers_id is None: - raise ProjectorElementException("id is required for list of speakers slide") - - return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id) - - -async def get_list_of_speakers_slide_data( - all_data_provider: ProjectorAllDataProvider, list_of_speakers_id: int -) -> Dict[str, Any]: - list_of_speakers = await get_model( - all_data_provider, "agenda/list-of-speakers", list_of_speakers_id - ) - - title_information = list_of_speakers["title_information"] - # try to get the agenda item for the content object (which must not exist) - content_object = await get_model( - all_data_provider, - list_of_speakers["content_object"]["collection"], - list_of_speakers["content_object"]["id"], - ) - agenda_item_id = content_object.get("agenda_item_id") - if agenda_item_id is not None: - agenda_item = await all_data_provider.get("agenda/item", agenda_item_id) - if agenda_item is not None: - title_information["_agenda_item_number"] = agenda_item["item_number"] - - # Partition speaker objects to waiting, current and finished - speakers_waiting = [] - speakers_finished = [] - current_speaker = None - for speaker in list_of_speakers["speakers"]: - user = await get_user_name(all_data_provider, speaker["user_id"]) - formatted_speaker = { - "user": user, - "marked": speaker["marked"], - "point_of_order": speaker["point_of_order"], - "weight": speaker["weight"], - "end_time": speaker["end_time"], - } - - if speaker["begin_time"] is None and speaker["end_time"] is None: - speakers_waiting.append(formatted_speaker) - elif speaker["begin_time"] is not None and speaker["end_time"] is None: - current_speaker = formatted_speaker - else: - speakers_finished.append(formatted_speaker) - - # sort speakers - speakers_waiting = sorted(speakers_waiting, key=lambda s: s["weight"]) - speakers_finished = sorted(speakers_finished, key=lambda s: s["end_time"]) - - number_of_last_speakers = await get_config( - all_data_provider, "agenda_show_last_speakers" - ) - number_of_next_speakers = await get_config( - all_data_provider, "agenda_show_next_speakers" - ) - - if number_of_last_speakers == 0: - speakers_finished = [] - else: - # Take the last speakers - speakers_finished = speakers_finished[-number_of_last_speakers:] - - if number_of_next_speakers != -1: - speakers_waiting = speakers_waiting[:number_of_next_speakers] - - return { - "waiting": speakers_waiting, - "current": current_speaker, - "finished": speakers_finished, - "content_object_collection": list_of_speakers["content_object"]["collection"], - "title_information": title_information, - "closed": list_of_speakers["closed"], - } - - -async def get_current_list_of_speakers_id_for_projector( - all_data_provider: ProjectorAllDataProvider, projector: Dict[str, Any] -) -> Union[int, None]: - """ - Search for elements, that do have a list of speakers: - Try to get a model by the collection and id in the element. This - model needs to have a 'list_of_speakers_id'. This list of speakers - must exist. The first matching element is taken. - """ - elements = projector["elements"] - list_of_speakers_id = None - for element in elements: - if "id" not in element: - continue - collection = element["name"] - id = element["id"] - model = await all_data_provider.get(collection, id) - if model is None: - continue - - if "list_of_speakers_id" not in model: - continue - - list_of_speakers_id = model["list_of_speakers_id"] - los_exists = await all_data_provider.exists( - "agenda/list-of-speakers", list_of_speakers_id - ) - if not los_exists: - continue - - break - - return list_of_speakers_id - - -async def get_reference_projector( - all_data_provider: ProjectorAllDataProvider, projector_id: int -) -> Dict[str, Any]: - """ - Returns the reference projector to the given projector (by id) - """ - this_projector = await get_model(all_data_provider, "core/projector", projector_id) - - reference_projector_id = this_projector["reference_projector_id"] or projector_id - return await get_model(all_data_provider, "core/projector", reference_projector_id) - - -async def current_list_of_speakers_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - The current list of speakers slide. Creates the data for the given projector. - """ - reference_projector = await get_reference_projector(all_data_provider, projector_id) - list_of_speakers_id = await get_current_list_of_speakers_id_for_projector( - all_data_provider, reference_projector - ) - if list_of_speakers_id is None: # no element found - return {} - - return await get_list_of_speakers_slide_data(all_data_provider, list_of_speakers_id) - - -async def current_speaker_chyron_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Returns the username for the current speaker. - """ - # get projector for color information - projector = await get_model(all_data_provider, "core/projector", projector_id) - - slide_data = { - "background_color": projector["chyron_background_color"], - "font_color": projector["chyron_font_color"], - } - - reference_projector = await get_reference_projector(all_data_provider, projector_id) - list_of_speakers_id = await get_current_list_of_speakers_id_for_projector( - all_data_provider, reference_projector - ) - if list_of_speakers_id is None: # no element found - return slide_data - - # get list of speakers to search current speaker - list_of_speakers = await get_model( - all_data_provider, "agenda/list-of-speakers", list_of_speakers_id - ) - - # find current speaker - current_speaker = None - for speaker in list_of_speakers["speakers"]: - if speaker["begin_time"] is not None and speaker["end_time"] is None: - current_speaker = await get_user_name(all_data_provider, speaker["user_id"]) - break - - if current_speaker is not None: - slide_data["current_speaker"] = current_speaker - - return slide_data - - -def register_projector_slides() -> None: - register_projector_slide("agenda/item-list", item_list_slide) - register_projector_slide("agenda/list-of-speakers", list_of_speakers_slide) - register_projector_slide( - "agenda/current-list-of-speakers", current_list_of_speakers_slide - ) - register_projector_slide( - "agenda/current-list-of-speakers-overlay", current_list_of_speakers_slide - ) - register_projector_slide( - "agenda/current-speaker-chyron", current_speaker_chyron_slide - ) diff --git a/server/openslides/assignments/apps.py b/server/openslides/assignments/apps.py index 3560d08e3..2d6da0506 100644 --- a/server/openslides/assignments/apps.py +++ b/server/openslides/assignments/apps.py @@ -13,7 +13,6 @@ class AssignmentsAppConfig(AppConfig): from ..utils.access_permissions import required_user from ..utils.rest_api import router from . import serializers # noqa - from .projector import register_projector_slides from .signals import get_permission_change_data from .views import ( AssignmentOptionViewSet, @@ -22,9 +21,6 @@ class AssignmentsAppConfig(AppConfig): AssignmentVoteViewSet, ) - # Define projector elements. - register_projector_slides() - # Connect signals. permission_change.connect( get_permission_change_data, diff --git a/server/openslides/assignments/projector.py b/server/openslides/assignments/projector.py deleted file mode 100644 index a07e4d73d..000000000 --- a/server/openslides/assignments/projector.py +++ /dev/null @@ -1,113 +0,0 @@ -from typing import Any, Dict, List - -from ..users.projector import get_user_name -from ..utils.projector import ( - ProjectorAllDataProvider, - get_model, - get_models, - register_projector_slide, -) -from .models import AssignmentPoll - - -async def assignment_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Assignment slide. - """ - assignment = await get_model( - all_data_provider, "assignments/assignment", element.get("id") - ) - - assignment_related_users: List[Dict[str, Any]] = [ - {"user": await get_user_name(all_data_provider, aru["user_id"])} - for aru in sorted( - assignment["assignment_related_users"], key=lambda aru: aru["weight"] - ) - ] - - return { - "title": assignment["title"], - "phase": assignment["phase"], - "open_posts": assignment["open_posts"], - "description": assignment["description"], - "assignment_related_users": assignment_related_users, - "number_poll_candidates": assignment["number_poll_candidates"], - } - - -async def assignment_poll_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Poll slide. - """ - poll = await get_model( - all_data_provider, "assignments/assignment-poll", element.get("id") - ) - assignment = await get_model( - all_data_provider, "assignments/assignment", poll["assignment_id"] - ) - - poll_data = { - key: poll[key] - for key in ( - "title", - "type", - "pollmethod", - "min_votes_amount", - "max_votes_amount", - "description", - "state", - "onehundred_percent_base", - "majority_method", - ) - } - - # Add options: - poll_data["options"] = [] - options = await get_models( - all_data_provider, "assignments/assignment-option", poll["options_id"] - ) - for option in sorted(options, key=lambda option: option["weight"]): - option_data: Dict[str, Any] = { - "user": { - "short_name": await get_user_name(all_data_provider, option["user_id"]) - } - } - if poll["state"] == AssignmentPoll.STATE_PUBLISHED: - option_data["yes"] = float(option["yes"]) - option_data["no"] = float(option["no"]) - option_data["abstain"] = float(option["abstain"]) - poll_data["options"].append(option_data) - - if poll["state"] == AssignmentPoll.STATE_PUBLISHED: - poll_data["amount_global_yes"] = ( - float(poll["amount_global_yes"]) if poll["amount_global_yes"] else None - ) - poll_data["amount_global_no"] = ( - float(poll["amount_global_no"]) if poll["amount_global_no"] else None - ) - poll_data["amount_global_abstain"] = ( - float(poll["amount_global_abstain"]) - if poll["amount_global_abstain"] - else None - ) - poll_data["votesvalid"] = float(poll["votesvalid"]) - poll_data["votesinvalid"] = float(poll["votesinvalid"]) - poll_data["votescast"] = float(poll["votescast"]) - - return { - "assignment": {"title": assignment["title"]}, - "poll": poll_data, - } - - -def register_projector_slides() -> None: - register_projector_slide("assignments/assignment", assignment_slide) - register_projector_slide("assignments/assignment-poll", assignment_poll_slide) diff --git a/server/openslides/core/apps.py b/server/openslides/core/apps.py index 835b21335..1c3ba6a84 100644 --- a/server/openslides/core/apps.py +++ b/server/openslides/core/apps.py @@ -16,11 +16,9 @@ class CoreAppConfig(AppConfig): def ready(self): # Import all required stuff. # Let all client websocket message register - from ..utils import websocket_client_messages # noqa from ..utils.rest_api import router from . import serializers # noqa from .config import config - from .projector import register_projector_slides from .signals import ( autoupdate_for_many_to_many_relations, cleanup_unused_permissions, @@ -41,9 +39,6 @@ class CoreAppConfig(AppConfig): # Collect all config variables before getting the constants. config.collect_config_variables_from_apps() - # Define projector elements. - register_projector_slides() - # Connect signals. post_permission_creation.connect( delete_django_app_permissions, dispatch_uid="delete_django_app_permissions" @@ -126,6 +121,7 @@ class CoreAppConfig(AppConfig): # Client settings client_settings_keys = [ + "AUTOUPDATE_DELAY", "PRIORITIZED_GROUP_IDS", "PING_INTERVAL", "PING_TIMEOUT", diff --git a/server/openslides/core/projector.py b/server/openslides/core/projector.py deleted file mode 100644 index 3c9f2ed82..000000000 --- a/server/openslides/core/projector.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import Any, Dict - -from ..utils.projector import ( - ProjectorAllDataProvider, - get_config, - get_model, - register_projector_slide, -) - - -async def countdown_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Countdown slide. - - Returns the full_data of the countdown element. - - element = { - name: 'core/countdown', - id: 5, # Countdown ID - } - """ - countdown = await get_model(all_data_provider, "core/countdown", element.get("id")) - return { - "description": countdown["description"], - "running": countdown["running"], - "countdown_time": countdown["countdown_time"], - "warning_time": await get_config( - all_data_provider, "agenda_countdown_warning_time" - ), - } - - -async def message_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Message slide. - - Returns the full_data of the message element. - - element = { - name: 'core/projector-message', - id: 5, # ProjectorMessage ID - } - """ - return await get_model( - all_data_provider, "core/projector-message", element.get("id") - ) - - -async def clock_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - return {} - - -def register_projector_slides() -> None: - register_projector_slide("core/countdown", countdown_slide) - register_projector_slide("core/projector-message", message_slide) - register_projector_slide("core/clock", clock_slide) diff --git a/server/openslides/core/serializers.py b/server/openslides/core/serializers.py index 2f935af67..062627699 100644 --- a/server/openslides/core/serializers.py +++ b/server/openslides/core/serializers.py @@ -1,7 +1,6 @@ from typing import Any from ..core.config import config -from ..utils.projector import projector_slides from ..utils.rest_api import ( Field, IdPrimaryKeyRelatedField, @@ -62,10 +61,6 @@ def elements_validator(value: Any) -> None: raise ValidationError( {"detail": "Every dictionary must have a key 'name'."} ) - if element["name"] not in projector_slides: - raise ValidationError( - {"detail": "Unknown projector element {0}.", "args": [element["name"]]} - ) def elements_array_validator(value: Any) -> None: diff --git a/server/openslides/core/urls.py b/server/openslides/core/urls.py index 6275c582c..564d1ae07 100644 --- a/server/openslides/core/urls.py +++ b/server/openslides/core/urls.py @@ -4,7 +4,8 @@ from . import views urlpatterns = [ - url(r"^servertime/$", views.ServerTime.as_view(), name="core_servertime"), + url(r"^servertime/$", views.ServertimeView.as_view(), name="core_servertime"), + url(r"^constants/$", views.ConstantsView.as_view(), name="core_constants"), url(r"^version/$", views.VersionView.as_view(), name="core_version"), url( r"^history/information/$", diff --git a/server/openslides/core/views.py b/server/openslides/core/views.py index 6389adff7..1a0e8f160 100644 --- a/server/openslides/core/views.py +++ b/server/openslides/core/views.py @@ -22,6 +22,7 @@ from ..utils.arguments import arguments from ..utils.auth import GROUP_ADMIN_PK, anonymous_is_enabled, has_perm, in_some_groups from ..utils.autoupdate import inform_changed_data from ..utils.cache import element_cache +from ..utils.constants import get_constants from ..utils.plugins import ( get_plugin_description, get_plugin_license, @@ -569,7 +570,7 @@ class CountdownViewSet(ModelViewSet): # Special API views -class ServerTime(utils_views.APIView): +class ServertimeView(utils_views.APIView): """ Returns the server time as UNIX timestamp. """ @@ -580,6 +581,19 @@ class ServerTime(utils_views.APIView): return now().timestamp() +class ConstantsView(utils_views.APIView): + """ + Returns the server time as UNIX timestamp. + """ + + http_method_names = ["get"] + + def get_context_data(self, **context): + if not self.request.user.is_authenticated and not anonymous_is_enabled(): + self.permission_denied(self.request) + return get_constants() + + class VersionView(utils_views.APIView): """ Returns a dictionary with the OpenSlides version and the version of all diff --git a/server/openslides/global_settings.py b/server/openslides/global_settings.py index bc1a201cc..1130593d0 100644 --- a/server/openslides/global_settings.py +++ b/server/openslides/global_settings.py @@ -16,7 +16,6 @@ INSTALLED_APPS = [ "django.contrib.sessions", "django.contrib.staticfiles", "rest_framework", - "channels", "openslides.agenda", "openslides.topics", "openslides.motions", @@ -122,13 +121,5 @@ PASSWORD_HASHERS = [ MEDIA_URL = "/media/" -# Django Channels -# http://channels.readthedocs.io/en/latest/ - -ASGI_APPLICATION = "openslides.routing.application" - -CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} - - # Enable updating the last_login field for users on every login. ENABLE_LAST_LOGIN_FIELD = False diff --git a/server/openslides/mediafiles/apps.py b/server/openslides/mediafiles/apps.py index ce557ae42..19d408145 100644 --- a/server/openslides/mediafiles/apps.py +++ b/server/openslides/mediafiles/apps.py @@ -11,9 +11,7 @@ class MediafilesAppConfig(AppConfig): # Import all required stuff. from openslides.core.signals import permission_change from openslides.utils.rest_api import router - from . import serializers # noqa - from .projector import register_projector_slides from .signals import get_permission_change_data from .views import MediafileViewSet @@ -25,9 +23,6 @@ class MediafilesAppConfig(AppConfig): "The MEDIA_URL setting must start and end with a slash" ) - # Define projector elements. - register_projector_slides() - # Connect signals. permission_change.connect( get_permission_change_data, diff --git a/server/openslides/mediafiles/projector.py b/server/openslides/mediafiles/projector.py deleted file mode 100644 index b996c823f..000000000 --- a/server/openslides/mediafiles/projector.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import Any, Dict - -from ..utils.projector import ( - ProjectorAllDataProvider, - get_model, - register_projector_slide, -) - - -async def mediafile_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Slide for Mediafile. - """ - mediafile = await get_model( - all_data_provider, "mediafiles/mediafile", element.get("id") - ) - return { - "path": mediafile["path"], - "mimetype": mediafile["mimetype"], - "media_url_prefix": mediafile["media_url_prefix"], - } - - -def register_projector_slides() -> None: - register_projector_slide("mediafiles/mediafile", mediafile_slide) diff --git a/server/openslides/motions/apps.py b/server/openslides/motions/apps.py index e5a22a149..9a79a9758 100644 --- a/server/openslides/motions/apps.py +++ b/server/openslides/motions/apps.py @@ -12,10 +12,8 @@ class MotionsAppConfig(AppConfig): # Import all required stuff. from openslides.core.signals import permission_change from openslides.utils.rest_api import router - from ..utils.access_permissions import required_user from . import serializers # noqa - from .projector import register_projector_slides from .signals import create_builtin_workflows, get_permission_change_data from .views import ( CategoryViewSet, @@ -31,9 +29,6 @@ class MotionsAppConfig(AppConfig): WorkflowViewSet, ) - # Define projector elements. - register_projector_slides() - # Connect signals. post_migrate.connect( create_builtin_workflows, dispatch_uid="motion_create_builtin_workflows" diff --git a/server/openslides/motions/projector.py b/server/openslides/motions/projector.py deleted file mode 100644 index 45fdca983..000000000 --- a/server/openslides/motions/projector.py +++ /dev/null @@ -1,437 +0,0 @@ -import re -from typing import Any, Dict, List, Optional - -from ..users.projector import get_user_name -from ..utils.projector import ( - ProjectorAllDataProvider, - ProjectorElementException, - get_config, - get_model, - register_projector_slide, -) -from .models import MotionPoll - - -motion_placeholder_regex = re.compile(r"\[motion:(\d+)\]") - - -async def get_state( - all_data_provider: ProjectorAllDataProvider, - motion: Dict[str, Any], - state_id_key: str, -) -> Dict[str, Any]: - """ - Returns a state element from one motion. Raises an error if the state does not exist. - """ - state = await all_data_provider.get("motions/state", motion[state_id_key]) - if state is None: - raise ProjectorElementException( - f"motion {motion['id']} can not be on the state with id {motion[state_id_key]}" - ) - return state - - -async def get_amendment_merge_into_motion_diff(all_data_provider, amendment): - """ - HINT: This implementation should be consistent to showInDiffView() in ViewMotionAmendedParagraph.ts - """ - if amendment["state_id"] is None: - return 0 - - state = await get_state(all_data_provider, amendment, "state_id") - if state["merge_amendment_into_final"] == -1: - return 0 - if state["merge_amendment_into_final"] == 1: - return 1 - - if amendment["recommendation_id"] is None: - return 0 - recommendation = await get_state(all_data_provider, amendment, "recommendation_id") - if recommendation["merge_amendment_into_final"] == 1: - return 1 - - return 0 - - -async def get_amendment_merge_into_motion_final(all_data_provider, amendment): - """ - HINT: This implementation should be consistent to showInFinalView() in ViewMotionAmendedParagraph.ts - """ - if amendment["state_id"] is None: - return 0 - - state = await get_state(all_data_provider, amendment, "state_id") - if state["merge_amendment_into_final"] == 1: - return 1 - - return 0 - - -async def get_amendments_for_motion(motion, all_data_provider): - amendment_data = [] - for amendment_id in motion["amendments_id"]: - amendment = await all_data_provider.get("motions/motion", amendment_id) - - merge_amendment_into_final = await get_amendment_merge_into_motion_final( - all_data_provider, amendment - ) - merge_amendment_into_diff = await get_amendment_merge_into_motion_diff( - all_data_provider, amendment - ) - - # Add change recommendations to the amendments: - change_recommendations = [] # type: ignore - for change_recommendation_id in amendment["change_recommendations_id"]: - cr = await get_model( - all_data_provider, - "motions/motion-change-recommendation", - change_recommendation_id, - ) - if cr is not None and not cr["internal"] and not cr["rejected"]: - change_recommendations.append(cr) - - amendment_data.append( - { - "id": amendment["id"], - "identifier": amendment["identifier"], - "title": amendment["title"], - "amendment_paragraphs": amendment["amendment_paragraphs"], - "change_recommendations": change_recommendations, - "merge_amendment_into_diff": merge_amendment_into_diff, - "merge_amendment_into_final": merge_amendment_into_final, - } - ) - return amendment_data - - -async def get_amendment_base_motion(amendment, all_data_provider): - motion = await get_model( - all_data_provider, "motions/motion", amendment.get("parent_id") - ) - - return { - "identifier": motion["identifier"], - "title": motion["title"], - "text": motion["text"], - } - - -async def get_amendment_base_statute(amendment, all_data_provider): - statute = await get_model( - all_data_provider, - "motions/statute-paragraph", - amendment.get("statute_paragraph_id"), - ) - return {"title": statute["title"], "text": statute["text"]} - - -async def extend_reference_motion_dict( - all_data_provider: ProjectorAllDataProvider, - recommendation: Optional[str], - referenced_motions: Dict[int, Dict[str, str]], -) -> None: - """ - Extends a dict of motion ids mapped to their title information. - The client can replace the placeholders in the recommendation correctly. - """ - if recommendation is None: - return - - # Collect all meantioned motions via [motion:] - referenced_ids = [ - int(id) for id in motion_placeholder_regex.findall(recommendation) - ] - for id in referenced_ids: - # Put every referenced motion into the referenced_motions dict - referenced_motion = await all_data_provider.get("motions/motion", id) - if id not in referenced_motions and referenced_motion is not None: - referenced_motions[id] = { - "title": referenced_motion["title"], - "identifier": referenced_motion["identifier"], - } - - -async def motion_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Motion slide. - - The returned dict can contain the following fields: - * identifier - * title - * text - * submitters - * amendment_paragraphs - * is_child - * show_meta_box - * show_referring_motions - * reason - * modified_final_version - * recommendation - * recommendation_extension - * recommender - * change_recommendations - """ - # Get motion - mode = element.get( - "mode", await get_config(all_data_provider, "motions_recommendation_text_mode") - ) - - # populate cache: - - motion = await get_model(all_data_provider, "motions/motion", element.get("id")) - - # Add submitters - submitters = [ - await get_user_name(all_data_provider, submitter["user_id"]) - for submitter in sorted( - motion["submitters"], key=lambda submitter: submitter["weight"] - ) - ] - - # Get some needed config values - show_meta_box = not await get_config( - all_data_provider, "motions_disable_sidebox_on_projector" - ) - show_referring_motions = not await get_config( - all_data_provider, "motions_hide_referring_motions" - ) - line_length = await get_config(all_data_provider, "motions_line_length") - line_numbering_mode = await get_config( - all_data_provider, "motions_default_line_numbering" - ) - motions_preamble = await get_config(all_data_provider, "motions_preamble") - - # Query all change-recommendation and amendment related things. - amendments = [] # type: ignore - base_motion = None - base_statute = None - if motion["statute_paragraph_id"]: - base_statute = await get_amendment_base_statute(motion, all_data_provider) - elif motion["parent_id"] is not None and motion["amendment_paragraphs"]: - base_motion = await get_amendment_base_motion(motion, all_data_provider) - else: - amendments = await get_amendments_for_motion(motion, all_data_provider) - - change_recommendations = [] # type: ignore - for change_recommendation_id in motion["change_recommendations_id"]: - cr = await get_model( - all_data_provider, - "motions/motion-change-recommendation", - change_recommendation_id, - ) - if cr is not None and not cr["internal"]: - change_recommendations.append(cr) - - # The base return value. More fields will get added below. - return_value = { - "identifier": motion["identifier"], - "title": motion["title"], - "submitters": submitters, - "preamble": motions_preamble, - "amendment_paragraphs": motion["amendment_paragraphs"], - "base_motion": base_motion, - "base_statute": base_statute, - "is_child": bool(motion["parent_id"]), - "show_meta_box": show_meta_box, - "show_referring_motions": show_referring_motions, - "change_recommendations": change_recommendations, - "amendments": amendments, - "line_length": line_length, - "line_numbering_mode": line_numbering_mode, - } - - if not await get_config(all_data_provider, "motions_disable_text_on_projector"): - return_value["text"] = motion["text"] - - if not await get_config(all_data_provider, "motions_disable_reason_on_projector"): - return_value["reason"] = motion["reason"] - - if mode == "final": - return_value["modified_final_version"] = motion["modified_final_version"] - - # Add recommendation, if enabled in config (and the motion has one) - if ( - not await get_config( - all_data_provider, "motions_disable_recommendation_on_projector" - ) - and motion["recommendation_id"] - ): - recommendation_state = await get_state( - all_data_provider, motion, "recommendation_id" - ) - return_value["recommendation"] = recommendation_state["recommendation_label"] - if recommendation_state["show_recommendation_extension_field"]: - recommendation_extension = motion["recommendation_extension"] - # All title information for referenced motions in the recommendation - referenced_motions: Dict[int, Dict[str, str]] = {} - await extend_reference_motion_dict( - all_data_provider, recommendation_extension, referenced_motions - ) - return_value["recommendation_extension"] = recommendation_extension - return_value["referenced_motions"] = referenced_motions - if motion["statute_paragraph_id"]: - return_value["recommender"] = await get_config( - all_data_provider, "motions_statute_recommendations_by" - ) - else: - return_value["recommender"] = await get_config( - all_data_provider, "motions_recommendations_by" - ) - - if show_referring_motions: - # Add recommendation-referencing motions - return_value[ - "recommendation_referencing_motions" - ] = await get_recommendation_referencing_motions( - all_data_provider, motion["id"] - ) - - return return_value - - -async def get_recommendation_referencing_motions( - all_data_provider: ProjectorAllDataProvider, motion_id: int -) -> Optional[List[Dict[str, Any]]]: - """ - Returns all title information for motions, that are referencing - the given motion (by id) in their recommendation. If there are no - motions, None is returned (instead of []). - """ - recommendation_referencing_motions = [] - all_motions = await all_data_provider.get_collection("motions/motion") - for motion in all_motions.values(): - # Motion must have a recommendation and a recommendaiton extension - if not motion["recommendation_id"] or not motion["recommendation_extension"]: - continue - - # The recommendation must allow the extension field (there might be left-overs - # in a motions recommendation extension..) - recommendation = await get_state(all_data_provider, motion, "recommendation_id") - if not recommendation["show_recommendation_extension_field"]: - continue - - # Find referenced motion ids - referenced_ids = [ - int(id) - for id in motion_placeholder_regex.findall( - motion["recommendation_extension"] - ) - ] - - # if one of the referenced ids is the given motion, add the current motion. - if motion_id in referenced_ids: - recommendation_referencing_motions.append( - {"title": motion["title"], "identifier": motion["identifier"]} - ) - return recommendation_referencing_motions or None - - -async def motion_block_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Motion block slide. - """ - motion_block = await get_model( - all_data_provider, "motions/motion-block", element.get("id") - ) - - # All motions in this motion block - motions = [] - - # All title information for referenced motions in the recommendation - referenced_motions: Dict[int, Dict[str, str]] = {} - - # iterate motions. - for motion_id in motion_block["motions_id"]: - motion = await all_data_provider.get("motions/motion", motion_id) - # primarily to please mypy, should theoretically not happen - if motion is None: - raise RuntimeError( - f"motion {motion_id} of block {element.get('id')} could not be found" - ) - - motion_object = { - "title": motion["title"], - "identifier": motion["identifier"], - } - - recommendation_id = motion["recommendation_id"] - if recommendation_id is not None: - recommendation = await get_state( - all_data_provider, motion, "recommendation_id" - ) - motion_object["recommendation"] = { - "name": recommendation["recommendation_label"], - "css_class": recommendation["css_class"], - } - if recommendation["show_recommendation_extension_field"]: - recommendation_extension = motion["recommendation_extension"] - await extend_reference_motion_dict( - all_data_provider, recommendation_extension, referenced_motions - ) - motion_object["recommendation_extension"] = recommendation_extension - - motions.append(motion_object) - - return { - "title": motion_block["title"], - "motions": motions, - "referenced_motions": referenced_motions, - } - - -async def motion_poll_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Poll slide. - """ - poll = await get_model(all_data_provider, "motions/motion-poll", element.get("id")) - motion = await get_model(all_data_provider, "motions/motion", poll["motion_id"]) - - poll_data = { - key: poll[key] - for key in ( - "title", - "type", - "pollmethod", - "state", - "onehundred_percent_base", - "majority_method", - ) - } - - if poll["state"] == MotionPoll.STATE_PUBLISHED: - option = await get_model( - all_data_provider, "motions/motion-option", poll["options_id"][0] - ) # there can only be exactly one option - poll_data["options"] = [ - { - "yes": float(option["yes"]), - "no": float(option["no"]), - "abstain": float(option["abstain"]), - } - ] - poll_data["votesvalid"] = poll["votesvalid"] - poll_data["votesinvalid"] = poll["votesinvalid"] - poll_data["votescast"] = poll["votescast"] - - return { - "motion": {"title": motion["title"], "identifier": motion["identifier"]}, - "poll": poll_data, - } - - -def register_projector_slides() -> None: - register_projector_slide("motions/motion", motion_slide) - register_projector_slide("motions/motion-block", motion_block_slide) - register_projector_slide("motions/motion-poll", motion_poll_slide) diff --git a/server/openslides/routing.py b/server/openslides/routing.py deleted file mode 100644 index af20cfea9..000000000 --- a/server/openslides/routing.py +++ /dev/null @@ -1,15 +0,0 @@ -from channels.routing import ProtocolTypeRouter, URLRouter -from django.conf.urls import url - -from openslides.utils.consumers import CloseConsumer, SiteConsumer -from openslides.utils.middleware import AuthMiddlewareStack - - -application = ProtocolTypeRouter( - { - # WebSocket chat handler - "websocket": AuthMiddlewareStack( - URLRouter([url(r"^ws/$", SiteConsumer), url(".*", CloseConsumer)]) - ) - } -) diff --git a/server/openslides/topics/apps.py b/server/openslides/topics/apps.py index 0424f8562..4d16668a1 100644 --- a/server/openslides/topics/apps.py +++ b/server/openslides/topics/apps.py @@ -11,13 +11,9 @@ class TopicsAppConfig(AppConfig): from ..utils.rest_api import router from . import serializers # noqa - from .projector import register_projector_slides from .signals import get_permission_change_data from .views import TopicViewSet - # Define projector elements. - register_projector_slides() - # Connect signals. permission_change.connect( get_permission_change_data, dispatch_uid="topics_get_permission_change_data" diff --git a/server/openslides/topics/projector.py b/server/openslides/topics/projector.py deleted file mode 100644 index 3687c0f31..000000000 --- a/server/openslides/topics/projector.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import Any, Dict - -from ..utils.projector import ( - ProjectorAllDataProvider, - get_model, - register_projector_slide, -) - - -async def topic_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - Topic slide. - - The returned dict can contain the following fields: - * title - * text - """ - topic = await get_model(all_data_provider, "topics/topic", element.get("id")) - item = await get_model(all_data_provider, "agenda/item", topic["agenda_item_id"]) - return { - "title": topic["title"], - "text": topic["text"], - "item_number": item["item_number"], - } - - -def register_projector_slides() -> None: - register_projector_slide("topics/topic", topic_slide) diff --git a/server/openslides/users/apps.py b/server/openslides/users/apps.py index 71e18d7cd..b588c3a1e 100644 --- a/server/openslides/users/apps.py +++ b/server/openslides/users/apps.py @@ -15,13 +15,9 @@ class UsersAppConfig(AppConfig): from ..core.signals import permission_change, post_permission_creation from ..utils.rest_api import router from . import serializers # noqa - from .projector import register_projector_slides from .signals import create_builtin_groups_and_admin, get_permission_change_data from .views import GroupViewSet, PersonalNoteViewSet, UserViewSet - # Define projector elements. - register_projector_slides() - # Connect signals. post_permission_creation.connect( create_builtin_groups_and_admin, diff --git a/server/openslides/users/projector.py b/server/openslides/users/projector.py deleted file mode 100644 index f5c27e4c6..000000000 --- a/server/openslides/users/projector.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Any, Dict, List, Optional - -from ..utils.projector import ( - ProjectorAllDataProvider, - get_model, - register_projector_slide, -) - - -async def user_slide( - all_data_provider: ProjectorAllDataProvider, - element: Dict[str, Any], - projector_id: int, -) -> Dict[str, Any]: - """ - User slide. - - The returned dict can contain the following fields: - * user - """ - return {"user": await get_user_name(all_data_provider, element.get("id"))} - - -async def get_user_name( - all_data_provider: ProjectorAllDataProvider, user_id: Optional[int] -) -> str: - """ - Returns the short name for an user_id. - """ - user = await get_model(all_data_provider, "users/user", user_id) - - name_parts: List[str] = [] - for name_part in ("title", "first_name", "last_name"): - if user[name_part]: - name_parts.append(user[name_part]) - if not name_parts: - name_parts.append(user["username"]) - if user["structure_level"]: - name_parts.append(f"({user['structure_level']})") - return " ".join(name_parts) - - -def register_projector_slides() -> None: - register_projector_slide("users/user", user_slide) diff --git a/server/openslides/utils/autoupdate.py b/server/openslides/utils/autoupdate.py index b77ba101f..6c5d6ae8e 100644 --- a/server/openslides/utils/autoupdate.py +++ b/server/openslides/utils/autoupdate.py @@ -4,13 +4,12 @@ from collections import defaultdict from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from asgiref.sync import async_to_sync -from channels.layers import get_channel_layer from django.db.models import Model from mypy_extensions import TypedDict from .auth import UserDoesNotExist from .cache import ChangeIdTooLowError, element_cache, get_element_id -from .projector import get_projector_data +from .stream import stream from .timing import Timing from .utils import get_model_from_collection_string, is_iterable, split_element_id @@ -112,8 +111,7 @@ class AutoupdateBundle: save_history(self.element_iterator) # Update cache and send autoupdate using async code. - change_id = async_to_sync(self.dispatch_autoupdate)() - return change_id + return async_to_sync(self.dispatch_autoupdate)() @property def element_iterator(self) -> Iterable[AutoupdateElement]: @@ -121,7 +119,7 @@ class AutoupdateBundle: for elements in self.autoupdate_elements.values(): yield from elements.values() - async def update_cache(self) -> int: + async def get_data_for_cache(self) -> Dict[str, Optional[Dict[str, Any]]]: """ Async helper function to update the cache. @@ -136,7 +134,7 @@ class AutoupdateBundle: "no_delete_on_restriction", False ) cache_elements[element_id] = full_data - return await element_cache.change_elements(cache_elements) + return cache_elements async def dispatch_autoupdate(self) -> int: """ @@ -145,25 +143,12 @@ class AutoupdateBundle: Return the change_id """ # Update cache - change_id = await self.update_cache() + cache_elements = await self.get_data_for_cache() + change_id = await element_cache.change_elements(cache_elements) # Send autoupdate - channel_layer = get_channel_layer() - await channel_layer.group_send( - "autoupdate", {"type": "msg_new_change_id", "change_id": change_id} - ) - - # Send projector - projector_data = await get_projector_data() - channel_layer = get_channel_layer() - await channel_layer.group_send( - "projector", - { - "type": "msg_projector_data", - "data": projector_data, - "change_id": change_id, - }, - ) + autoupdate_payload = {"elements": cache_elements, "change_id": change_id} + await stream.send("autoupdate", autoupdate_payload) return change_id @@ -286,14 +271,12 @@ class AutoupdateBundleMiddleware: if status_ok or status_redirect: change_id = bundle.done() - # inject the autoupdate, if there is an autoupdate and the status is + # inject the change id, if there was an autoupdate and the response status is # ok (and not redirect; redirects do not have a useful content) if change_id is not None and status_ok: - user_id = request.user.pk or 0 # Inject the autoupdate in the response. # The complete response body will be overwritten! - _, autoupdate = async_to_sync(get_autoupdate_data)(change_id, user_id) - content = {"autoupdate": autoupdate, "data": response.data} + content = {"change_id": change_id, "data": response.data} # Note: autoupdate may be none on skipped ones (which should not happen # since the user has made the request....) response.content = json.dumps(content) diff --git a/server/openslides/utils/cache.py b/server/openslides/utils/cache.py index 31828a7b7..7696f3a07 100644 --- a/server/openslides/utils/cache.py +++ b/server/openslides/utils/cache.py @@ -163,6 +163,8 @@ class ElementCache: logger.info("Saving cache data into the cache...") await self.cache_provider.add_to_full_data(mapping) logger.info("Done saving the cache data.") + await self.cache_provider.set_cache_ready() + logger.info("Done: Cache is ready now.") def _build_cache_get_elementid_model_mapping( self, config_only: bool = False diff --git a/server/openslides/utils/cache_providers.py b/server/openslides/utils/cache_providers.py index 53676d011..329c6377f 100644 --- a/server/openslides/utils/cache_providers.py +++ b/server/openslides/utils/cache_providers.py @@ -8,11 +8,7 @@ from django.core.exceptions import ImproperlyConfigured from typing_extensions import Protocol from . import logging -from .redis import ( - read_only_redis_amount_replicas, - read_only_redis_wait_timeout, - use_redis, -) +from .redis import use_redis from .schema_version import SchemaVersion from .utils import split_element_id, str_dict_to_bytes @@ -54,6 +50,9 @@ class ElementCacheProvider(Protocol): async def data_exists(self) -> bool: ... + async def set_cache_ready(self) -> None: + ... + async def get_all_data(self) -> Dict[bytes, bytes]: ... @@ -127,6 +126,7 @@ class RedisCacheProvider: full_data_cache_key: str = "full_data" change_id_cache_key: str = "change_id" schema_cache_key: str = "schema" + cache_ready_key: str = "cache_ready" # All lua-scripts used by this provider. Every entry is a Tuple (str, bool) with the # script and an ensure_cache-indicator. If the indicator is True, a short ensure_cache-script @@ -325,6 +325,7 @@ class RedisCacheProvider: """ async with get_connection() as redis: tr = redis.multi_exec() + tr.delete(self.cache_ready_key) tr.delete(self.change_id_cache_key) tr.delete(self.full_data_cache_key) tr.hmset_dict(self.full_data_cache_key, data) @@ -342,11 +343,16 @@ class RedisCacheProvider: Returns True, when there is data in the cache. """ async with get_connection(read_only=True) as redis: - return await redis.exists(self.full_data_cache_key) and bool( - await redis.zrangebyscore( - self.change_id_cache_key, withscores=True, count=1, offset=0 - ) - ) + return (await redis.get(self.cache_ready_key)) is not None + # return await redis.exists(self.full_data_cache_key) and bool( + # await redis.zrangebyscore( + # self.change_id_cache_key, withscores=True, count=1, offset=0 + # ) + # ) + + async def set_cache_ready(self) -> None: + async with get_connection(read_only=False) as redis: + await redis.set(self.cache_ready_key, "ok") @ensure_cache_wrapper() async def get_all_data(self) -> Dict[bytes, bytes]: @@ -495,7 +501,11 @@ class RedisCacheProvider: async def get_schema_version(self) -> Optional[SchemaVersion]: """ Retrieves the schema version of the cache or None, if not existent """ async with get_connection(read_only=True) as redis: - schema_version = await redis.hgetall(self.schema_cache_key) + try: + schema_version = await redis.hgetall(self.schema_cache_key) + except aioredis.errors.ReplyError: + await redis.delete(self.schema_cache_key) + return None if not schema_version: return None @@ -543,15 +553,6 @@ class RedisCacheProvider: raise CacheReset() else: raise e - if not read_only and read_only_redis_amount_replicas is not None: - reported_amount = await redis.wait( - read_only_redis_amount_replicas, read_only_redis_wait_timeout - ) - if reported_amount != read_only_redis_amount_replicas: - logger.warn( - f"WAIT reported {reported_amount} replicas of {read_only_redis_amount_replicas} " - + f"requested after {read_only_redis_wait_timeout} ms!" - ) return result async def _eval( @@ -584,6 +585,7 @@ class MemoryCacheProvider: self.set_data_dicts() def set_data_dicts(self) -> None: + self.ready = False self.full_data: Dict[str, str] = {} self.change_id_data: Dict[int, Set[str]] = {} self.locks: Dict[str, str] = {} @@ -594,6 +596,7 @@ class MemoryCacheProvider: async def clear_cache(self) -> None: self.set_data_dicts() + self.ready = False async def reset_full_cache( self, data: Dict[str, str], default_change_id: int @@ -606,7 +609,11 @@ class MemoryCacheProvider: self.full_data.update(data) async def data_exists(self) -> bool: - return bool(self.full_data) and self.default_change_id >= 0 + return self.ready + # return bool(self.full_data) and self.default_change_id >= 0 + + async def set_cache_ready(self) -> None: + self.ready = True async def get_all_data(self) -> Dict[bytes, bytes]: return str_dict_to_bytes(self.full_data) diff --git a/server/openslides/utils/consumer_autoupdate_strategy.py b/server/openslides/utils/consumer_autoupdate_strategy.py deleted file mode 100644 index 63bde7021..000000000 --- a/server/openslides/utils/consumer_autoupdate_strategy.py +++ /dev/null @@ -1,102 +0,0 @@ -import asyncio -from asyncio import Task -from typing import Optional, cast - -from django.conf import settings - -from .autoupdate import get_autoupdate_data -from .cache import element_cache -from .websocket import ChangeIdTooHighException, ProtocollAsyncJsonWebsocketConsumer - - -AUTOUPDATE_DELAY = getattr(settings, "AUTOUPDATE_DELAY", None) - - -class ConsumerAutoupdateStrategy: - def __init__(self, consumer: ProtocollAsyncJsonWebsocketConsumer) -> None: - self.consumer = consumer - # client_change_id = None: unknown -> set on first autoupdate or request_change_id - # client_change_id is int: the change_id, the client knows about, so the next - # update must be from client_change_id+1 .. - self.client_change_id: Optional[int] = None - self.next_send_time = None - self.timer_task_handle: Optional[Task[None]] = None - self.lock = asyncio.Lock() - - async def request_change_id( - self, change_id: int, in_response: Optional[str] = None - ) -> None: - """ - The change id is not inclusive, so the client is on change_id and wants - data from change_id+1 .. now - """ - # This resets the server side tracking of the client's change id. - async with self.lock: - await self.stop_timer() - - max_change_id = await element_cache.get_current_change_id() - self.client_change_id = change_id - - if self.client_change_id == max_change_id: - # The client is up-to-date, so nothing will be done - return None - - if self.client_change_id > max_change_id: - message = ( - f"Requested change_id {self.client_change_id} is higher than the " - + f"highest change_id {max_change_id}." - ) - raise ChangeIdTooHighException(message, in_response=in_response) - - await self.send_autoupdate(in_response=in_response) - - async def new_change_id(self, change_id: int) -> None: - async with self.lock: - if self.client_change_id is None: - # The -1 is to send this autoupdate as the first one to he client. - # Remember: the client_change_id is the change_id the client knows about - self.client_change_id = change_id - 1 - - if AUTOUPDATE_DELAY is None: # feature deactivated, send directly - await self.send_autoupdate() - elif self.timer_task_handle is None: - await self.start_timer() - - async def get_running_loop(self) -> asyncio.AbstractEventLoop: - if hasattr(asyncio, "get_running_loop"): - return asyncio.get_running_loop() # type: ignore - else: - return asyncio.get_event_loop() - - async def start_timer(self) -> None: - loop = await self.get_running_loop() - self.timer_task_handle = loop.create_task(self.timer_task()) - - async def stop_timer(self) -> None: - if self.timer_task_handle is not None: - self.timer_task_handle.cancel() - self.timer_task_handle = None - - async def timer_task(self) -> None: - try: - await asyncio.sleep(AUTOUPDATE_DELAY) - except asyncio.CancelledError: - return - - async with self.lock: - await self.send_autoupdate() - self.timer_task_handle = None - - async def send_autoupdate(self, in_response: Optional[str] = None) -> None: - # here, 1 is added to the change_id, because the client_change_id is the id the client - # *knows* about -> the client needs client_change_id+1 since get_autoupdate_data is - # inclusive [change_id .. max_change_id]. - max_change_id, autoupdate = await get_autoupdate_data( - cast(int, self.client_change_id) + 1, self.consumer.user_id - ) - if autoupdate is not None: - self.client_change_id = max_change_id - # It will be send, so we can set the client_change_id - await self.consumer.send_json( - type="autoupdate", content=autoupdate, in_response=in_response - ) diff --git a/server/openslides/utils/consumers.py b/server/openslides/utils/consumers.py deleted file mode 100644 index 7432423f9..000000000 --- a/server/openslides/utils/consumers.py +++ /dev/null @@ -1,175 +0,0 @@ -import time -from typing import Any, Dict, List, Optional, cast -from urllib.parse import parse_qs - -from channels.generic.websocket import AsyncWebsocketConsumer - -from . import logging -from .auth import UserDoesNotExist, async_anonymous_is_enabled -from .cache import element_cache -from .consumer_autoupdate_strategy import ConsumerAutoupdateStrategy -from .utils import get_worker_id -from .websocket import BaseWebsocketException, ProtocollAsyncJsonWebsocketConsumer - - -logger = logging.getLogger("openslides.websocket") - - -class SiteConsumer(ProtocollAsyncJsonWebsocketConsumer): - """ - Websocket Consumer for the site. - """ - - groups = ["site"] - - ID_COUNTER = 0 - """ - ID counter for assigning each instance of this class an unique id. - """ - - def __init__(self, *args: Any, **kwargs: Any) -> None: - self.projector_hash: Dict[int, int] = {} - SiteConsumer.ID_COUNTER += 1 - self._id = get_worker_id() + "-" + str(SiteConsumer.ID_COUNTER) - self.autoupdate_strategy = ConsumerAutoupdateStrategy(self) - super().__init__(*args, **kwargs) - - async def connect(self) -> None: - """ - A user connects to the site. - - If it is an anonymous user and anonymous is disabled, the connection is closed. - - Sends the startup data to the user. - """ - self.user_id = self.scope["user"]["id"] - - self.connect_time = time.time() - # self.scope['user'] is the full_data dict of the user. For an - # anonymous user is it the dict {'id': 0} - change_id = None - if not await async_anonymous_is_enabled() and not self.user_id: - await self.accept() # workaround for #4009 - await self.close() - logger.debug(f"connect: denied ({self._id})") - return - - query_string = cast( - Dict[bytes, List[bytes]], parse_qs(self.scope["query_string"]) - ) - if b"change_id" in query_string: - try: - change_id = int(query_string[b"change_id"][0]) - except ValueError: - await self.accept() # workaround for #4009 - await self.close() - logger.debug(f"connect: wrong change id ({self._id})") - return - - await self.accept() - - if change_id is not None: - logger.debug(f"connect: change id {change_id} ({self._id})") - try: - await self.request_autoupdate(change_id) - except BaseWebsocketException as e: - await self.send_exception(e) - else: - logger.debug(f"connect: no change id ({self._id})") - - await self.channel_layer.group_add("autoupdate", self.channel_name) - - async def disconnect(self, close_code: int) -> None: - """ - A user disconnects. Remove it from autoupdate. - """ - await self.channel_layer.group_discard("autoupdate", self.channel_name) - active_seconds = int(time.time() - self.connect_time) - logger.debug( - f"disconnect code={close_code} active_secs={active_seconds} ({self._id})" - ) - - async def msg_new_change_id(self, event: Dict[str, Any]) -> None: - """ - Send changed or deleted elements to the user. - """ - change_id = event["change_id"] - try: - await self.autoupdate_strategy.new_change_id(change_id) - except UserDoesNotExist: - # Maybe the user was deleted, but a websocket connection is still open to the user. - # So we can close this connection and return. - await self.close() - - async def msg_projector_data(self, event: Dict[str, Any]) -> None: - """ - The projector has changed. - """ - all_projector_data = event["data"] - change_id = event["change_id"] - - projector_data: Dict[int, Dict[str, Any]] = {} - for projector_id in self.listen_projector_ids: - data = all_projector_data.get(projector_id, []) - new_hash = hash(str(data)) - if new_hash != self.projector_hash.get(projector_id): - projector_data[projector_id] = data - self.projector_hash[projector_id] = new_hash - - if projector_data: - await self.send_projector_data(projector_data, change_id=change_id) - - async def msg_notify(self, event: Dict[str, Any]) -> None: - """ - Send a notify message to the user. - """ - item = event["incomming"] - - users = item.get("users") - reply_channels = item.get("replyChannels") - if ( - (isinstance(users, bool) and users) - or (isinstance(users, list) and self.user_id in users) - or ( - isinstance(reply_channels, list) and self.channel_name in reply_channels - ) - or (users is None and reply_channels is None) - ): - item["senderChannelName"] = event["senderChannelName"] - item["senderUserId"] = event["senderUserId"] - await self.send_json(type="notify", content=item) - - async def request_autoupdate( - self, change_id: int, in_response: Optional[str] = None - ) -> None: - await self.autoupdate_strategy.request_change_id( - change_id, in_response=in_response - ) - - async def send_projector_data( - self, - data: Dict[int, Dict[str, Any]], - change_id: Optional[int] = None, - in_response: Optional[str] = None, - ) -> None: - """ - Sends projector data to the consumer. - """ - if change_id is None: - change_id = await element_cache.get_current_change_id() - - content = {"change_id": change_id, "data": data} - await self.send_json(type="projector", content=content, in_response=in_response) - - -class CloseConsumer(AsyncWebsocketConsumer): - """ Auto-closes the connection """ - - groups: List[str] = [] - - def __init__(self, args: Dict[str, Any], **kwargs: Any) -> None: - logger.info(f'Closing connection to unknown websocket url {args["path"]}') - - async def connect(self) -> None: - await self.accept() - await self.close() diff --git a/server/openslides/utils/middleware.py b/server/openslides/utils/middleware.py deleted file mode 100644 index 513755988..000000000 --- a/server/openslides/utils/middleware.py +++ /dev/null @@ -1,73 +0,0 @@ -from typing import Any, Dict, Optional - -from channels.auth import ( - AuthMiddleware, - CookieMiddleware, - SessionMiddleware, - _get_user_session_key, -) -from django.conf import settings -from django.contrib.auth import BACKEND_SESSION_KEY, HASH_SESSION_KEY -from django.utils.crypto import constant_time_compare - -from .cache import element_cache - - -class CollectionAuthMiddleware(AuthMiddleware): - """ - Like the channels AuthMiddleware but returns a user dict id instead of - a django Model as user. - """ - - def populate_scope(self, scope: Dict[str, Any]) -> None: - # Make sure we have a session - if "session" not in scope: - raise ValueError( - "AuthMiddleware cannot find session in scope. SessionMiddleware must be above it." - ) - # Add it to the scope if it's not there already - if "user" not in scope: - scope["user"] = {} - - async def resolve_scope(self, scope: Dict[str, Any]) -> None: - scope["user"].update(await get_user(scope)) - - -async def get_user(scope: Dict[str, Any]) -> Dict[str, Any]: - """ - Returns a user id from a channels-scope-session. - - If no user is retrieved, return {'id': 0}. - """ - # This code is basicly from channels.auth: - # https://github.com/django/channels/blob/d5e81a78e96770127da79248349808b6ee6ec2a7/channels/auth.py#L16 - if "session" not in scope: - raise ValueError( - "Cannot find session in scope. You should wrap your consumer in SessionMiddleware." - ) - session = scope["session"] - user: Optional[Dict[str, Any]] = None - try: - user_id = _get_user_session_key(session) - backend_path = session[BACKEND_SESSION_KEY] - except KeyError: - pass - else: - if backend_path in settings.AUTHENTICATION_BACKENDS: - user = await element_cache.get_element_data("users/user", user_id) - if user: - # Verify the session - session_hash = session.get(HASH_SESSION_KEY) - session_hash_verified = session_hash and constant_time_compare( - session_hash, user["session_auth_hash"] - ) - if not session_hash_verified: - session.flush() - user = None - return user or {"id": 0} - - -# Handy shortcut for applying all three layers at once -AuthMiddlewareStack = lambda inner: CookieMiddleware( # noqa - SessionMiddleware(CollectionAuthMiddleware(inner)) -) diff --git a/server/openslides/utils/projector.py b/server/openslides/utils/projector.py deleted file mode 100644 index 3a8f1cc00..000000000 --- a/server/openslides/utils/projector.py +++ /dev/null @@ -1,172 +0,0 @@ -""" -General projector code. - -Functions that handel the registration of projector elements and the rendering -of the data to present it on the projector. -""" - -from collections import defaultdict -from typing import Any, Awaitable, Callable, Dict, List, Optional - -from . import logging -from .cache import element_cache - - -logger = logging.getLogger(__name__) - - -class ProjectorElementException(Exception): - """ - Exception for errors in one element on the projector. - """ - - -class ProjectorAllDataProvider: - NON_EXISTENT_MARKER = object() - - def __init__(self) -> None: - self.cache: Any = defaultdict(dict) # fuu you mypy - self.fetched_collection: Dict[str, bool] = {} - - async def get(self, collection: str, id: int) -> Optional[Dict[str, Any]]: - cache_data = self.cache[collection].get(id) - if cache_data is None: - data: Any = await element_cache.get_element_data(collection, id) - if data is None: - data = ProjectorAllDataProvider.NON_EXISTENT_MARKER - self.cache[collection][id] = data - - cache_data = self.cache[collection][id] - if cache_data == ProjectorAllDataProvider.NON_EXISTENT_MARKER: - return None - return cache_data - - async def get_collection(self, collection: str) -> Dict[int, Dict[str, Any]]: - if not self.fetched_collection.get(collection, False): - collection_data = await element_cache.get_collection_data(collection) - self.cache[collection] = collection_data - self.fetched_collection[collection] = True - return self.cache[collection] - - async def exists(self, collection: str, id: int) -> bool: - model = await self.get(collection, id) - return model is not None - - -ProjectorSlide = Callable[ - [ProjectorAllDataProvider, Dict[str, Any], int], Awaitable[Dict[str, Any]] -] - - -projector_slides: Dict[str, ProjectorSlide] = {} - - -def register_projector_slide(name: str, slide: ProjectorSlide) -> None: - """ - Registers a projector slide. - - Has to be called in the app.ready method. - """ - projector_slides[name] = slide - - -async def get_projector_data( - projector_ids: List[int] = None, -) -> Dict[int, List[Dict[str, Any]]]: - """ - Calculates and returns the data for one or all projectors. - - The keys of the returned data are the projector ids as int. When converted - to json, the numbers will changed to strings like "1". - - The data for each projector is a list of elements. - - Each element is a dict where the keys are "elements", "data". "elements" - contains the projector elements. It is the same as the projector elements in - the database. "data" contains all necessary data to render the projector - element. The key can also be "error" if there is a generall error for the - slide. In this case the values "elements" and "data" are optional. - - The returned value looks like this: - - projector_data = { - 1: [ - { - "element": { - "name": "agenda/item-list", - }, - "data": { - "items": [] - }, - }, - ], - } - """ - if projector_ids is None: - projector_ids = [] - - projector_data: Dict[int, List[Dict[str, Any]]] = {} - all_data_provider = ProjectorAllDataProvider() - projectors = await all_data_provider.get_collection("core/projector") - - for projector_id, projector in projectors.items(): - if projector_ids and projector_id not in projector_ids: - # only render the projector in question. - continue - - if not projector["elements"]: - # Skip empty elements. - continue - - projector_data[projector_id] = [] - for element in projector["elements"]: - projector_slide = projector_slides[element["name"]] - try: - data = await projector_slide(all_data_provider, element, projector_id) - except ProjectorElementException as err: - data = {"error": str(err)} - projector_data[projector_id].append({"data": data, "element": element}) - - return projector_data - - -async def get_config(all_data_provider: ProjectorAllDataProvider, key: str) -> Any: - """ - Returns a config value from all_data_provider. - Triggers the cache early: It access `get_colelction` instead of `get`. It - allows for all successive queries for configs to be cached. - """ - from ..core.config import config - - config_id = (await config.async_get_key_to_id())[key] - - configs = await all_data_provider.get_collection(config.get_collection_string()) - return configs[config_id]["value"] - - -async def get_model( - all_data_provider: ProjectorAllDataProvider, collection: str, id: Any -) -> Dict[str, Any]: - """ - Tries to get the model identified by the collection and id. - If the id is invalid or the model not found, ProjectorElementExceptions will be raised. - """ - if id is None: - raise ProjectorElementException(f"id is required for {collection} slide") - - model = await all_data_provider.get(collection, id) - if model is None: - raise ProjectorElementException(f"{collection} with id {id} does not exist") - return model - - -async def get_models( - all_data_provider: ProjectorAllDataProvider, collection: str, ids: List[Any] -) -> List[Dict[str, Any]]: - """ - Tries to fetch all given models. Models are required to be all of the collection `collection`. - """ - logger.info( - f"Note: a call to `get_models` with {collection}/{ids}. This might be cache-intensive" - ) - return [await get_model(all_data_provider, collection, id) for id in ids] diff --git a/server/openslides/utils/redis.py b/server/openslides/utils/redis.py index 23cbfffff..279e533b8 100644 --- a/server/openslides/utils/redis.py +++ b/server/openslides/utils/redis.py @@ -10,15 +10,13 @@ logger = logging.getLogger(__name__) # Defaults use_redis = False use_read_only_redis = False -read_only_redis_amount_replicas = None -read_only_redis_wait_timeout = None try: import aioredis except ImportError: pass else: - from .redis_connection_pool import ConnectionPool + from .redis_connection_pool import ConnectionPool # type: ignore # set use_redis to true, if there is a value for REDIS_ADDRESS in the settings redis_address = getattr(settings, "REDIS_ADDRESS", "") @@ -33,13 +31,6 @@ else: if use_read_only_redis: logger.info(f"Redis read only address {redis_read_only_address}") read_only_pool = ConnectionPool({"address": redis_read_only_address}) - - read_only_redis_amount_replicas = getattr(settings, "AMOUNT_REPLICAS", 1) - logger.info(f"AMOUNT_REPLICAS={read_only_redis_amount_replicas}") - read_only_redis_wait_timeout = getattr( - settings, "REDIS_SLAVE_WAIT_TIMEOUT", 1000 - ) - logger.info(f"REDIS_SLAVE_WAIT_TIMEOUT={read_only_redis_wait_timeout}") else: logger.info("Redis is not configured.") diff --git a/server/openslides/utils/redis_connection_pool.py b/server/openslides/utils/redis_connection_pool.py index 51e8a2e88..1f71b1e34 100644 --- a/server/openslides/utils/redis_connection_pool.py +++ b/server/openslides/utils/redis_connection_pool.py @@ -1,8 +1,10 @@ +# type: ignore import asyncio +import sys +import types from typing import Any, Dict, List, Optional import aioredis -from channels_redis.core import ConnectionPool as ChannelRedisConnectionPool from django.conf import settings from . import logging @@ -13,6 +15,128 @@ connection_pool_limit = getattr(settings, "CONNECTION_POOL_LIMIT", 100) logger.info(f"CONNECTION_POOL_LIMIT={connection_pool_limit}") +# Copied from https://github.com/django/channels_redis/blob/master/channels_redis/core.py +# and renamed.. + +AIOREDIS_VERSION = tuple(map(int, aioredis.__version__.split("."))) + + +def _wrap_close(loop, pool): + """ + Decorate an event loop's close method with our own. + """ + original_impl = loop.close + + def _wrapper(self, *args, **kwargs): + # If the event loop was closed, there's nothing we can do anymore. + if not self.is_closed(): + self.run_until_complete(pool.close_loop(self)) + # Restore the original close() implementation after we're done. + self.close = original_impl + return self.close(*args, **kwargs) + + loop.close = types.MethodType(_wrapper, loop) + + +class ChannelRedisConnectionPool: + """ + Connection pool manager for the channel layer. + It manages a set of connections for the given host specification and + taking into account asyncio event loops. + """ + + def __init__(self, host): + self.host = host + self.conn_map = {} + self.in_use = {} + + def _ensure_loop(self, loop): + """ + Get connection list for the specified loop. + """ + if loop is None: + loop = asyncio.get_event_loop() + + if loop not in self.conn_map: + # Swap the loop's close method with our own so we get + # a chance to do some cleanup. + _wrap_close(loop, self) + self.conn_map[loop] = [] + + return self.conn_map[loop], loop + + async def pop(self, loop=None): + """ + Get a connection for the given identifier and loop. + """ + conns, loop = self._ensure_loop(loop) + if not conns: + if sys.version_info >= (3, 8, 0) and AIOREDIS_VERSION >= (1, 3, 1): + conn = await aioredis.create_redis(**self.host) + else: + conn = await aioredis.create_redis(**self.host, loop=loop) + conns.append(conn) + conn = conns.pop() + if conn.closed: + conn = await self.pop(loop=loop) + return conn + self.in_use[conn] = loop + return conn + + def push(self, conn): + """ + Return a connection to the pool. + """ + loop = self.in_use[conn] + del self.in_use[conn] + if loop is not None: + conns, _ = self._ensure_loop(loop) + conns.append(conn) + + def conn_error(self, conn): + """ + Handle a connection that produced an error. + """ + conn.close() + del self.in_use[conn] + + def reset(self): + """ + Clear all connections from the pool. + """ + self.conn_map = {} + self.in_use = {} + + async def close_loop(self, loop): + """ + Close all connections owned by the pool on the given loop. + """ + if loop in self.conn_map: + for conn in self.conn_map[loop]: + conn.close() + await conn.wait_closed() + del self.conn_map[loop] + + for k, v in self.in_use.items(): + if v is loop: + self.in_use[k] = None + + async def close(self): + """ + Close all connections owned by the pool. + """ + conn_map = self.conn_map + in_use = self.in_use + self.reset() + for conns in conn_map.values(): + for conn in conns: + conn.close() + await conn.wait_closed() + for conn in in_use: + conn.close() + await conn.wait_closed() + + class InvalidConnection(Exception): pass diff --git a/server/openslides/utils/settings.py.tpl b/server/openslides/utils/settings.py.tpl index 071e67ea7..834e9238a 100644 --- a/server/openslides/utils/settings.py.tpl +++ b/server/openslides/utils/settings.py.tpl @@ -83,45 +83,20 @@ DATABASES = { } -# Set use_redis to True to activate redis as cache-, asgi- and session backend. -use_redis = False +# Collection Cache +REDIS_ADDRESS = "redis://redis:6379/0" -if use_redis: - # Django Channels - - # https://channels.readthedocs.io/en/latest/topics/channel_layers.html#configuration - CHANNEL_LAYERS = { - "default": { - "BACKEND": "channels_redis.core.RedisChannelLayer", - "CONFIG": { - "hosts": [("localhost", 6379)], - "capacity": 100000, - }, - }, - } - # Collection Cache - - # Can be: - # a Redis URI — "redis://host:6379/0?encoding=utf-8"; - # a (host, port) tuple — ('localhost', 6379); - # or a unix domain socket path string — "/path/to/redis.sock". - REDIS_ADDRESS = "redis://127.0.0.1" - # REDIS_READ_ONLY_ADDRESS - AMOUNT_REPLICAS = 1 - - # Session backend - - # Redis configuration for django-redis-sessions. - # https://github.com/martinrusev/django-redis-sessions - - SESSION_ENGINE = 'redis_sessions.session' - SESSION_REDIS = { - 'host': '127.0.0.1', - 'port': 6379, - 'db': 0, - 'prefix': 'session', - 'socket_timeout': 2 - } +# Session backend +# Redis configuration for django-redis-sessions. +# https://github.com/martinrusev/django-redis-sessions +SESSION_ENGINE = 'redis_sessions.session' +SESSION_REDIS = { + 'host': 'redis', + 'port': 6379, + 'db': 0, + 'prefix': 'session', + 'socket_timeout': 2 +} # SAML integration # Please read https://github.com/OpenSlides/OpenSlides/blob/master/openslides/saml/README.md @@ -144,21 +119,17 @@ ENABLE_ELECTRONIC_VOTING = False # Internationalization # https://docs.djangoproject.com/en/1.10/topics/i18n/ - TIME_ZONE = 'Europe/Berlin' # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ - STATICFILES_DIRS = [os.path.join(OPENSLIDES_USER_DATA_DIR, 'static')] + STATICFILES_DIRS - STATIC_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'collected-static') # Files # https://docs.djangoproject.com/en/1.10/topics/files/ - MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'media', '') @@ -169,7 +140,6 @@ MEDIA_ROOT = os.path.join(OPENSLIDES_USER_DATA_DIR, 'media', '') # Logging # see https://docs.djangoproject.com/en/2.2/topics/logging/ - LOGGING = { 'version': 1, 'disable_existing_loggers': False, diff --git a/server/openslides/utils/stream.py b/server/openslides/utils/stream.py new file mode 100644 index 000000000..4488c384b --- /dev/null +++ b/server/openslides/utils/stream.py @@ -0,0 +1,50 @@ +import json +from typing import Any + +from django.conf import settings +from typing_extensions import Protocol + +from . import logging +from .redis import use_redis + + +logger = logging.getLogger(__name__) + +REDIS_STREAM_MAXLEN = getattr(settings, "REDIS_STREAM_MAXLEN", None) +logger.info(f"Redis stream maxlen {REDIS_STREAM_MAXLEN}") + +if use_redis: + from .redis import get_connection + + +class Stream(Protocol): + def __init__(self) -> None: + ... + + async def send(self, stream_name: str, payload: Any) -> None: + ... + + +class RedisStream: + async def send(self, stream_name: str, payload: Any) -> None: + fields = { + "content": json.dumps(payload, separators=(",", ":")), + } + async with get_connection() as redis: + await redis.xadd(stream_name, fields, max_len=REDIS_STREAM_MAXLEN) + + +class NoopStream: + async def send(self, stream_name: str, payload: Any) -> None: + pass + + +def load_stream() -> Stream: + if use_redis: + return RedisStream() + else: + logger.error("You have to configure redis to let OpenSlides work properly!") + return NoopStream() + + +stream = load_stream() diff --git a/server/openslides/utils/websocket.py b/server/openslides/utils/websocket.py deleted file mode 100644 index 4157d7ce2..000000000 --- a/server/openslides/utils/websocket.py +++ /dev/null @@ -1,267 +0,0 @@ -import json -from typing import Any, Dict, Optional - -import jsonschema -import lz4.frame -from channels.generic.websocket import AsyncWebsocketConsumer -from django.conf import settings -from websockets.exceptions import ConnectionClosed - -from .stats import WebsocketThroughputLogger - - -# Custom Websocket error codes (not to be confused with the websocket *connection* -# status codes like 1000 or 1006. These are custom ones for OpenSlides to give a -# machine parseable response, so the client can react on errors. - -WEBSOCKET_NOT_AUTHORIZED = 100 -# E.g. if a user does not have the right permission(s) for a message. - -WEBSOCKET_CHANGE_ID_TOO_HIGH = 101 -# If data is requested and the given change id is higher than the highest change id -# from the element_cache. - -WEBSOCKET_WRONG_FORMAT = 102 -# If the recieved data has not the expected format. - - -class BaseWebsocketException(Exception): - code: int - - def __init__(self, message: str, in_response: Optional[str] = None) -> None: - self.message = message - self.in_response = in_response - - -class NotAuthorizedException(BaseWebsocketException): - code = WEBSOCKET_NOT_AUTHORIZED - - -class ChangeIdTooHighException(BaseWebsocketException): - code = WEBSOCKET_CHANGE_ID_TOO_HIGH - - -class WrongFormatException(BaseWebsocketException): - code = WEBSOCKET_WRONG_FORMAT - - -class AsyncCompressedJsonWebsocketConsumer(AsyncWebsocketConsumer): - async def receive( - self, - text_data: Optional[str] = None, - bytes_data: Optional[bytes] = None, - **kwargs: Dict[str, Any], - ) -> None: - if bytes_data: - uncompressed_data = lz4.frame.decompress(bytes_data) - text_data = uncompressed_data.decode("utf-8") - - recv_len = len(bytes_data) - uncompressed_len = len(uncompressed_data) - await WebsocketThroughputLogger.receive(uncompressed_len, recv_len) - elif text_data: - uncompressed_len = len(text_data.encode("utf-8")) - await WebsocketThroughputLogger.receive(uncompressed_len) - - if text_data: - await self.receive_json(json.loads(text_data), **kwargs) - - async def send_json(self, content: Any, close: bool = False) -> None: - text_data = json.dumps(content) - bytes_data = None # type: ignore - - b_text_data = text_data.encode("utf-8") - uncompressed_len = len(b_text_data) - - if getattr(settings, "COMPRESSION", True): - compressed_data = lz4.frame.compress(b_text_data) - ratio = len(b_text_data) / len(compressed_data) - if ratio > 1: - bytes_data = compressed_data - text_data = None # type: ignore - await WebsocketThroughputLogger.send(uncompressed_len, len(bytes_data)) - - if not bytes_data: - await WebsocketThroughputLogger.send(uncompressed_len) - - await self.send(text_data=text_data, bytes_data=bytes_data, close=close) - - async def receive_json(self, content: str, **kwargs: Dict[str, Any]) -> None: - pass - - -class ProtocollAsyncJsonWebsocketConsumer(AsyncCompressedJsonWebsocketConsumer): - """ - Mixin for JSONWebsocketConsumers, that speaks the a special protocol. - """ - - async def send_json( # type: ignore - self, - type: str, - content: Any, - id: Optional[str] = None, - in_response: Optional[str] = None, - silence_errors: Optional[bool] = True, - ) -> None: - """ - Sends the data with the type. - If silence_errors is True (default), all ConnectionClosed - and runtime errors during sending will be ignored. - """ - out = {"type": type, "content": content} - if id: - out["id"] = id - if in_response: - out["in_response"] = in_response - try: - await super().send_json(out) - except (ConnectionClosed, RuntimeError) as e: - # The ConnectionClosed error is thrown by the websocket lib: websocket/protocol.py in ensure_open - # `websockets.exceptions.ConnectionClosed: WebSocket connection is closed: code = 1005 - # (no status code [internal]), no reason` (Also with other codes) - # The RuntimeError is thrown by uvicorn: uvicorn/protocols/websockets/websockets_impl.py in asgi_send - # `RuntimeError: Unexpected ASGI message 'websocket.send', after sending 'websocket.close'` - if not silence_errors: - raise e - - async def send_error( - self, - code: int, - message: str, - in_response: Optional[str] = None, - silence_errors: Optional[bool] = True, - ) -> None: - """ - Send generic error messages with a custom status code (see above) and a text message. - """ - await self.send_json( - "error", - {"code": code, "message": message}, - None, - in_response=in_response, - silence_errors=silence_errors, - ) - - async def send_exception( - self, e: BaseWebsocketException, silence_errors: Optional[bool] = True - ) -> None: - """ - Send generic error messages with a custom status code (see above) and a text message. - """ - await self.send_json( - "error", - {"code": e.code, "message": e.message}, - None, - in_response=e.in_response, - silence_errors=silence_errors, - ) - - async def receive_json(self, content: Any) -> None: # type: ignore - """ - Receives the json data, parses it and calls receive_content. - """ - try: - jsonschema.validate(content, schema) - except jsonschema.ValidationError as err: - try: - in_response = content["id"] - except (TypeError, KeyError): - # content is not a dict (TypeError) or has not the key id (KeyError) - in_response = None - - await self.send_error( - code=WEBSOCKET_WRONG_FORMAT, message=str(err), in_response=in_response - ) - return - - try: - await websocket_client_messages[content["type"]].receive_content( - self, content["content"], id=content["id"] - ) - except BaseWebsocketException as e: - await self.send_exception(e) - - -schema: Dict[str, Any] = { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "OpenSlidesWebsocketProtocol", - "description": "The packages that OpenSlides sends between the server and the client.", - "type": "object", - "properties": { - "type": { - "description": "Defines what kind of packages is packed.", - "type": "string", - }, - "content": {"description": "The content of the package."}, - "id": {"description": "An identifier of the package.", "type": "string"}, - "in_response": { - "description": "The id of another package that the other part sent before.", - "type": "string", - }, - }, - "required": ["type", "content", "id"], - "anyOf": [], # This will be filled in register_client_message() -} - - -class BaseWebsocketClientMessage: - schema: Dict[str, object] = {} - """ - Optional schema. - - If schema is not set, any value in content is accepted. - """ - - identifier: str = "" - """ - A unique identifier for the websocket message. - - This is used as value in the 'type' property in the websocket message. - """ - - content_required = True - """ - Desiedes, if the content property is required. - """ - - async def receive_content( - self, consumer: "ProtocollAsyncJsonWebsocketConsumer", message: Any, id: str - ) -> None: - raise NotImplementedError( - "WebsocketClientMessage needs the method receive_content()." - ) - - -websocket_client_messages: Dict[str, BaseWebsocketClientMessage] = {} -""" -Saves all websocket client message object ordered by there identifier. -""" - - -def register_client_message( - websocket_client_message: BaseWebsocketClientMessage, -) -> None: - """ - Registers one websocket client message class. - """ - if ( - not websocket_client_message.identifier - or websocket_client_message.identifier in websocket_client_messages - ): - raise NotImplementedError("WebsocketClientMessage needs a unique identifier.") - - websocket_client_messages[ - websocket_client_message.identifier - ] = websocket_client_message - - # Add the message schema to the schema - message_schema: Dict[str, Any] = { - "properties": { - "type": {"const": websocket_client_message.identifier}, - "content": websocket_client_message.schema, - } - } - if websocket_client_message.content_required: - message_schema["required"] = ["content"] - - schema["anyOf"].append(message_schema) diff --git a/server/openslides/utils/websocket_client_messages.py b/server/openslides/utils/websocket_client_messages.py deleted file mode 100644 index ce8c62759..000000000 --- a/server/openslides/utils/websocket_client_messages.py +++ /dev/null @@ -1,215 +0,0 @@ -from typing import Any, Dict, Optional - -from . import logging -from .auth import async_has_perm -from .constants import get_constants -from .projector import get_projector_data -from .stats import WebsocketLatencyLogger -from .websocket import ( - BaseWebsocketClientMessage, - NotAuthorizedException, - ProtocollAsyncJsonWebsocketConsumer, - register_client_message, -) - - -logger = logging.getLogger(__name__) - - -class Notify(BaseWebsocketClientMessage): - """ - Websocket message from a client to send a message to other clients. - """ - - identifier = "notify" - schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Notify element.", - "description": "Element that one client can send to one or many other clients.", - "type": "object", - "properties": { - "name": {"description": "The name of the notify message", "type": "string"}, - "content": {"description": "The actual content of this message."}, - "reply_channels": { - "description": "A list of channels to send this message to.", - "type": "array", - "items": {"type": "string"}, - }, - "users": { - "anyOf": [ - { - "description": "A list of user ids to send this message to.", - "type": "array", - "items": {"type": "integer"}, - }, - { - "description": "This flag indicates, that this message should be send to all users.", - "enum": [True], - }, - ] - }, - }, - "required": ["name", "content"], - } - # Define a required permission for a notify message here. If the emitting user does not - # have this permission, he will get an error message in response. - notify_permissions: Dict[str, str] = {"swCheckForUpdate": "superadmin"} - - async def receive_content( - self, consumer: "ProtocollAsyncJsonWebsocketConsumer", content: Any, id: str - ) -> None: - # Check if the user is allowed to send this notify message - perm = self.notify_permissions.get(content["name"]) - if perm is not None and not await async_has_perm(consumer.user_id, perm): - raise NotAuthorizedException( - f"You need '{perm}' to send this message.", in_response=id - ) - else: - # Some logging - name = content.get("name", "") - users = content.get("users", []) - if users is True: - users = "all" - else: - users = ", ".join(str(user) for user in users) - reply_channels = ", ".join(content.get("replyChannels", [])) - logger.info( - f"Got notify '{name}' from {consumer.channel_name} users={users} reply_channels={reply_channels}" - ) - - # Forward to all other active site consumers to handle the notify message. - await consumer.channel_layer.group_send( - "site", - { - "type": "msg_notify", - "incomming": content, - "senderChannelName": consumer.channel_name, - "senderUserId": consumer.user_id, - }, - ) - - -register_client_message(Notify()) - - -class Constants(BaseWebsocketClientMessage): - """ - The Client requests the constants. - """ - - identifier = "constants" - content_required = False - - async def receive_content( - self, consumer: "ProtocollAsyncJsonWebsocketConsumer", content: Any, id: str - ) -> None: - # Return all constants to the client. - await consumer.send_json( - type="constants", content=get_constants(), in_response=id - ) - - -register_client_message(Constants()) - - -class GetElements(BaseWebsocketClientMessage): - """ - The Client request database elements. - """ - - identifier = "getElements" - schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "titel": "getElement request", - "description": "Request from the client to server to get elements.", - "type": "object", - "properties": { - # change_id is not required - "change_id": {"type": "integer"} - }, - } - - async def receive_content( - self, consumer: "ProtocollAsyncJsonWebsocketConsumer", content: Any, id: str - ) -> None: - requested_change_id = content.get("change_id", 0) - await consumer.request_autoupdate(requested_change_id, in_response=id) - - -register_client_message(GetElements()) - - -class ListenToProjectors(BaseWebsocketClientMessage): - """ - The client tells, to which projector it listens. - - Therefor it sends a list of projector ids. If it sends an empty list, it does - not want to get any projector information. - """ - - identifier = "listenToProjectors" - schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "titel": "Listen to projectors", - "description": "Listen to zero, one or more projectors..", - "type": "object", - "properties": { - "projector_ids": { - "type": "array", - "items": {"type": "integer"}, - "uniqueItems": True, - } - }, - "required": ["projector_ids"], - } - - async def receive_content( - self, consumer: "ProtocollAsyncJsonWebsocketConsumer", content: Any, id: str - ) -> None: - consumer.listen_projector_ids = content["projector_ids"] - if consumer.listen_projector_ids: - # listen to projector group - await consumer.channel_layer.group_add("projector", consumer.channel_name) - else: - # do not listen to projector group - await consumer.channel_layer.group_discard( - "projector", consumer.channel_name - ) - - # Send projector data - if consumer.listen_projector_ids: - projector_data = await get_projector_data(consumer.listen_projector_ids) - for projector_id, data in projector_data.items(): - consumer.projector_hash[projector_id] = hash(str(data)) - - await consumer.send_projector_data(projector_data, in_response=id) - - -register_client_message(ListenToProjectors()) - - -class PingPong(BaseWebsocketClientMessage): - """ - Responds to pings from the client. - """ - - identifier = "ping" - schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "titel": "PingPong", - "description": "Does a ping pong handshake", - "anyOf": [{"type": "number"}, {"type": "null"}], - } - - async def receive_content( - self, - consumer: "ProtocollAsyncJsonWebsocketConsumer", - latency: Optional[int], - id: str, - ) -> None: - await consumer.send_json(type="pong", content=latency, in_response=id) - if latency is not None: - await WebsocketLatencyLogger.add_latency(latency) - - -register_client_message(PingPong()) diff --git a/server/openslides/asgi.py b/server/openslides/wsgi.py similarity index 70% rename from server/openslides/asgi.py rename to server/openslides/wsgi.py index f1ff19b8b..7a72c9030 100644 --- a/server/openslides/asgi.py +++ b/server/openslides/wsgi.py @@ -3,8 +3,7 @@ ASGI entrypoint. Configures Django and then runs the application defined in the ASGI_APPLICATION setting. """ -import django -from channels.routing import get_default_application +from django.core.wsgi import get_wsgi_application from .utils.main import setup_django_settings_module from .utils.startup import run_startup_hooks @@ -12,7 +11,6 @@ from .utils.startup import run_startup_hooks # Loads the openslides setting. You can use your own settings by setting the # environment variable DJANGO_SETTINGS_MODULE -setup_django_settings_module() -django.setup() +setup_django_settings_module(local_installation=True) +application = get_wsgi_application() run_startup_hooks() -application = get_default_application() diff --git a/server/requirements/big_mode.txt b/server/requirements/big_mode.txt index a7cd9e291..fd8ead856 100644 --- a/server/requirements/big_mode.txt +++ b/server/requirements/big_mode.txt @@ -1,9 +1,11 @@ # Requirements for Redis and PostgreSQL support -channels-redis>=2.2,<2.5 django-redis-sessions>=0.6.1,<0.7 psycopg2-binary>=2.7.3.2,<2.9 aioredis>=1.1.0,<1.3 # Requirements for fast asgi server -gunicorn>=19.9.0,<20 +#gunicorn>=19.9.0,<20 + +# https://github.com/benoitc/gunicorn/issues/1913 +git+https://github.com/FinnStutzenstein/gunicorn.git@fix uvicorn[standard]>=0.9,<1.0 diff --git a/server/requirements/production.txt b/server/requirements/production.txt index 8739d7432..d622807d8 100644 --- a/server/requirements/production.txt +++ b/server/requirements/production.txt @@ -7,14 +7,12 @@ asgiref>=3.2.9 # Requirements for OpenSlides in production in alphabetical order bleach>=1.5.0,<3.2 -channels>=2.1.2,<2.4 daphne>=2.2,<2.5 Django>=2.1,<2.3 djangorestframework>=3.9.4,<3.10 jsonfield2>=3.0,<3.1 attrs>=19.2.0 jsonschema>=3.0,<3.1 -lz4>=2.1.6 mypy_extensions>=0.4,<0.5 PyPDF2>=1.26,<1.27 roman>=2.0,<3.2 diff --git a/server/tests/integration/utils/__init__.py b/server/tests/integration/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/server/tests/integration/utils/test_consumers.py b/server/tests/integration/utils/test_consumers.py deleted file mode 100644 index 764bb143e..000000000 --- a/server/tests/integration/utils/test_consumers.py +++ /dev/null @@ -1,594 +0,0 @@ -from importlib import import_module -from typing import Optional, Tuple -from unittest.mock import patch - -import pytest -from asgiref.sync import sync_to_async -from django.conf import settings -from django.contrib.auth import BACKEND_SESSION_KEY, HASH_SESSION_KEY, SESSION_KEY - -from openslides.asgi import application -from openslides.core.config import config -from openslides.utils.autoupdate import ( - AutoupdateElement, - inform_deleted_data, - inform_elements, -) -from openslides.utils.cache import element_cache -from openslides.utils.websocket import ( - WEBSOCKET_CHANGE_ID_TOO_HIGH, - WEBSOCKET_WRONG_FORMAT, -) - -from ...unit.utils.cache_provider import ( - Collection1, - Collection2, - PersonalizedCollection, - get_cachable_provider, -) -from ..helpers import TConfig, TProjector, TUser -from ..websocket import WebsocketCommunicator - - -@pytest.fixture(autouse=True) -async def prepare_element_cache(settings): - """ - Resets the element cache. - - Uses a cacheable_provider for tests with example data. - """ - await element_cache.cache_provider.clear_cache() - orig_cachable_provider = element_cache.cachable_provider - element_cache.cachable_provider = get_cachable_provider( - [ - Collection1(), - Collection2(), - PersonalizedCollection(), - TConfig(), - TUser(), - TProjector(), - ] - ) - element_cache._cachables = None - await element_cache.async_ensure_cache(default_change_id=10) - yield - # Reset the cachable_provider - element_cache.cachable_provider = orig_cachable_provider - element_cache._cachables = None - await element_cache.cache_provider.clear_cache() - - -@pytest.fixture -async def get_communicator(): - communicator: Optional[WebsocketCommunicator] = None - - def get_communicator(query_string="", headers=None): - nonlocal communicator # use the outer communicator variable - if query_string: - query_string = f"?{query_string}" - communicator = WebsocketCommunicator( - application, f"/ws/{query_string}", headers=headers - ) - return communicator - - yield get_communicator - if communicator: - await communicator.disconnect() - - -@pytest.fixture -async def communicator(get_communicator): - yield get_communicator() - - -@pytest.fixture -async def set_config(): - """ - Set a config variable in the element_cache without hitting the database. - """ - - async def _set_config(key, value): - with patch("openslides.utils.autoupdate.save_history"): - collection_string = config.get_collection_string() - config_id = config.key_to_id[key] # type: ignore - full_data = {"id": config_id, "key": key, "value": value} - await sync_to_async(inform_elements)( - [ - AutoupdateElement( - id=config_id, - collection_string=collection_string, - full_data=full_data, - disable_history=True, - ) - ] - ) - - return _set_config - - -@pytest.mark.asyncio -async def test_normal_connection(get_communicator, set_config): - await set_config("general_system_enable_anonymous", True) - connected, __ = await get_communicator().connect() - assert connected - - -@pytest.mark.asyncio -async def test_connection_with_change_id(get_communicator, set_config): - await set_config("general_system_enable_anonymous", True) - communicator = get_communicator("change_id=0") - await communicator.connect() - - response = await communicator.receive_json_from() - - type = response.get("type") - content = response.get("content") - assert type == "autoupdate" - assert "changed" in content - assert "deleted" in content - assert "from_change_id" in content - assert "to_change_id" in content - assert Collection1().get_collection_string() in content["changed"] - assert Collection2().get_collection_string() in content["changed"] - assert PersonalizedCollection().get_collection_string() in content["changed"] - assert TConfig().get_collection_string() in content["changed"] - assert TUser().get_collection_string() in content["changed"] - - -@pytest.mark.xfail # This will fail until a proper solution in #4009 -@pytest.mark.asyncio -async def test_connection_with_invalid_change_id(get_communicator, set_config): - await set_config("general_system_enable_anonymous", True) - communicator = get_communicator("change_id=invalid") - connected, __ = await communicator.connect() - - assert connected is False - - -@pytest.mark.asyncio -async def test_connection_with_too_big_change_id(get_communicator, set_config): - await set_config("general_system_enable_anonymous", True) - communicator = get_communicator("change_id=100") - - connected, __ = await communicator.connect() - - assert connected is True - await communicator.assert_receive_error(code=WEBSOCKET_CHANGE_ID_TOO_HIGH) - - -@pytest.mark.asyncio -async def test_changed_data_autoupdate(get_communicator, set_config): - await set_config("general_system_enable_anonymous", True) - communicator = get_communicator() - await communicator.connect() - - # Change a config value - await set_config("general_event_name", "Test Event") - response = await communicator.receive_json_from() - - id = config.get_key_to_id()["general_event_name"] - type = response.get("type") - content = response.get("content") - assert type == "autoupdate" - assert content["changed"] == { - "core/config": [{"id": id, "key": "general_event_name", "value": "Test Event"}] - } - - -@pytest.mark.xfail # This will fail until a proper solution in #4009 -@pytest.mark.asyncio -async def test_anonymous_disabled(communicator): - connected, __ = await communicator.connect() - - assert not connected - - -async def create_user_session_cookie(user_id: int) -> Tuple[bytes, bytes]: - # login user with id 1 - engine = import_module(settings.SESSION_ENGINE) - session = engine.SessionStore() # type: ignore - session[SESSION_KEY] = str(user_id) - session[ - HASH_SESSION_KEY - ] = "362d4f2de1463293cb3aaba7727c967c35de43ee" # see helpers.TUser - session[BACKEND_SESSION_KEY] = "django.contrib.auth.backends.ModelBackend" - session.save() - scn = settings.SESSION_COOKIE_NAME - cookie_header = (b"cookie", f"{scn}={session.session_key}".encode()) - return cookie_header - - -@pytest.mark.asyncio -async def test_with_user(get_communicator): - cookie_header = await create_user_session_cookie(1) - communicator = get_communicator(headers=[cookie_header]) - - connected, __ = await communicator.connect() - - assert connected - - -@pytest.mark.asyncio -async def test_skipping_autoupdate(set_config, get_communicator): - cookie_header = await create_user_session_cookie(1) - communicator = get_communicator(headers=[cookie_header]) - - await communicator.connect() - - with patch("openslides.utils.autoupdate.save_history"): - await sync_to_async(inform_elements)( - [ - AutoupdateElement( - id=2, - collection_string=PersonalizedCollection().get_collection_string(), - full_data={"id": 2, "value": "new value 1", "user_id": 2}, - disable_history=True, - ) - ] - ) - await sync_to_async(inform_elements)( - [ - AutoupdateElement( - id=2, - collection_string=PersonalizedCollection().get_collection_string(), - full_data={"id": 2, "value": "new value 2", "user_id": 2}, - disable_history=True, - ) - ] - ) - assert await communicator.receive_nothing() - - # Trigger autoupdate - await set_config("general_event_name", "Test Event") - response = await communicator.receive_json_from() - content = response["content"] - - assert PersonalizedCollection().get_collection_string() not in content["deleted"] - assert PersonalizedCollection().get_collection_string() not in content["changed"] - assert config.get_collection_string() in content["changed"] - assert ( - content["to_change_id"] - content["from_change_id"] - ) == 2 # Skipped two autoupdates - - -@pytest.mark.asyncio -async def test_receive_deleted_data(get_communicator, set_config): - await set_config("general_system_enable_anonymous", True) - communicator = get_communicator() - await communicator.connect() - - # Delete test element - with patch("openslides.utils.autoupdate.save_history"): - await sync_to_async(inform_deleted_data)( - [(Collection1().get_collection_string(), 1)] - ) - response = await communicator.receive_json_from() - - type = response.get("type") - content = response.get("content") - assert type == "autoupdate" - assert content["deleted"] == {Collection1().get_collection_string(): [1]} - - -@pytest.mark.asyncio -async def test_send_notify(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - { - "type": "notify", - "content": {"content": "foobar, what else.", "name": "message_name"}, - "id": "test", - } - ) - response = await communicator.receive_json_from() - - content = response["content"] - assert isinstance(content, dict) - assert content["content"] == "foobar, what else." - assert content["name"] == "message_name" - assert "senderChannelName" in content - assert content["senderUserId"] == 0 - - -@pytest.mark.asyncio -async def test_invalid_websocket_message_type(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to([]) - - await communicator.assert_receive_error(code=WEBSOCKET_WRONG_FORMAT) - - -@pytest.mark.asyncio -async def test_invalid_websocket_message_no_id(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to({"type": "test", "content": "foobar"}) - - await communicator.assert_receive_error(code=WEBSOCKET_WRONG_FORMAT) - - -@pytest.mark.asyncio -async def test_send_unknown_type(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - { - "type": "if_you_add_this_type_to_openslides_I_will_be_sad", - "content": True, - "id": "test_id", - } - ) - - await communicator.assert_receive_error( - code=WEBSOCKET_WRONG_FORMAT, in_response="test_id" - ) - - -@pytest.mark.asyncio -async def test_request_constants(communicator, settings, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - {"type": "constants", "content": "", "id": "test_id"} - ) - - response = await communicator.receive_json_from() - assert response["type"] == "constants" - # See conftest.py for the content of 'content' - assert response["content"] == {"constant1": "value1", "constant2": "value2"} - - -@pytest.mark.asyncio -async def test_send_get_elements(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - {"type": "getElements", "content": {}, "id": "test_id"} - ) - response = await communicator.receive_json_from() - - type = response.get("type") - content = response.get("content") - assert type == "autoupdate" - assert "changed" in content - assert "deleted" in content - assert "from_change_id" in content - assert "to_change_id" in content - assert Collection1().get_collection_string() in content["changed"] - assert Collection2().get_collection_string() in content["changed"] - assert PersonalizedCollection().get_collection_string() in content["changed"] - assert TConfig().get_collection_string() in content["changed"] - assert TUser().get_collection_string() in content["changed"] - - -@pytest.mark.asyncio -async def test_send_get_elements_too_big_change_id(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - {"type": "getElements", "content": {"change_id": 100}, "id": "test_id"} - ) - await communicator.assert_receive_error( - code=WEBSOCKET_CHANGE_ID_TOO_HIGH, in_response="test_id" - ) - - -@pytest.mark.asyncio -async def test_send_get_elements_too_small_change_id(communicator, set_config): - # Note: this test depends on the default_change_id set in prepare_element_cache - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - {"type": "getElements", "content": {"change_id": 1}, "id": "test_id"} - ) - response = await communicator.receive_json_from() - - type = response.get("type") - assert type == "autoupdate" - assert response.get("in_response") == "test_id" - assert response.get("content")["all_data"] - - -@pytest.mark.asyncio -async def test_send_connect_up_to_date(communicator, set_config): - """ - Test, that a second request with change_id+1 from the first request does not - send anything, becuase the client is up to date. - """ - await set_config("general_system_enable_anonymous", True) - element_cache.cache_provider.change_id_data = {} # type: ignore - await communicator.connect() - await communicator.send_json_to( - {"type": "getElements", "content": {"change_id": 0}, "id": "test_id"} - ) - response1 = await communicator.receive_json_from() - max_change_id = response1.get("content")["to_change_id"] - - await communicator.send_json_to( - { - "type": "getElements", - "content": {"change_id": max_change_id}, - "id": "test_id", - } - ) - assert await communicator.receive_nothing() - - -@pytest.mark.xfail # This test is broken -@pytest.mark.asyncio -async def test_send_connect_twice_with_clear_change_id_cache_same_change_id_then_first_request( - communicator, set_config -): - """ - Test, that a second request with the change_id from the first request, returns - all data. - - A client should not do this but request for change_id+1 - """ - await set_config("general_system_enable_anonymous", True) - await element_cache.cache_provider.clear_cache() - await communicator.connect() - await communicator.send_json_to( - {"type": "getElements", "content": {"change_id": 0}, "id": "test_id"} - ) - response1 = await communicator.receive_json_from() - first_change_id = response1.get("content")["to_change_id"] - - await communicator.send_json_to( - { - "type": "getElements", - "content": {"change_id": first_change_id}, - "id": "test_id", - } - ) - response2 = await communicator.receive_json_from() - - assert response2["type"] == "autoupdate" - assert response2.get("content")["all_data"] - - -@pytest.mark.asyncio -async def test_request_changed_elements_no_douple_elements(communicator, set_config): - """ - Test, that when an elements is changed twice, it is only returned - onces when ask a range of change ids. - - Test when all_data is false - """ - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - # Change element twice - await set_config("general_event_name", "Test Event") - await set_config("general_event_name", "Other value") - # Ask for all elements - await communicator.send_json_to( - {"type": "getElements", "content": {"change_id": 2}, "id": "test_id"} - ) - - response = await communicator.receive_json_from() - type = response.get("type") - content = response.get("content") - assert type == "autoupdate" - assert not response.get("content")["all_data"] - config_ids = [e["id"] for e in content["changed"]["core/config"]] - # test that config_ids are unique - assert len(config_ids) == len(set(config_ids)) - - -@pytest.mark.asyncio -async def test_send_invalid_get_elements(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - {"type": "getElements", "content": {"change_id": "some value"}, "id": "test_id"} - ) - response = await communicator.receive_json_from() - - type = response.get("type") - assert type == "error" - assert response.get("in_response") == "test_id" - - -@pytest.mark.asyncio -async def test_listen_to_projector(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - - await communicator.send_json_to( - { - "type": "listenToProjectors", - "content": {"projector_ids": [1]}, - "id": "test_id", - } - ) - response = await communicator.receive_json_from() - - type = response.get("type") - content = response.get("content") - assert type == "projector" - assert content == { - "change_id": 11, - "data": { - "1": [ - { - "data": {"name": "slide1", "event_name": "OpenSlides"}, - "element": {"id": 1, "name": "test/slide1"}, - } - ] - }, - } - - -@pytest.mark.asyncio -async def test_update_projector(communicator, set_config): - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - await communicator.send_json_to( - { - "type": "listenToProjectors", - "content": {"projector_ids": [1]}, - "id": "test_id", - } - ) - await communicator.receive_json_from() # recieve initial projector data - - # Change a config value - await set_config("general_event_name", "Test Event") - - # We need two messages: The autoupdate and the projector data in this order - response = await communicator.receive_json_from() - assert response.get("type") == "autoupdate" - - response = await communicator.receive_json_from() - - type = response.get("type") - content = response.get("content") - assert type == "projector" - assert content == { - "change_id": 12, - "data": { - "1": [ - { - "data": {"name": "slide1", "event_name": "Test Event"}, - "element": {"id": 1, "name": "test/slide1"}, - } - ] - }, - } - - -@pytest.mark.asyncio -async def test_update_projector_to_current_value(communicator, set_config): - """ - When a value does not change, the projector should not be updated. - """ - await set_config("general_system_enable_anonymous", True) - await communicator.connect() - await communicator.send_json_to( - { - "type": "listenToProjectors", - "content": {"projector_ids": [1]}, - "id": "test_id", - } - ) - await communicator.receive_json_from() - - # Change a config value to current_value - await set_config("general_event_name", "OpenSlides") - - # We await an autoupdate, bot no projector data - response = await communicator.receive_json_from() - assert response.get("type") == "autoupdate" - - assert await communicator.receive_nothing() diff --git a/server/tests/integration/websocket.py b/server/tests/integration/websocket.py deleted file mode 100644 index a528eeee9..000000000 --- a/server/tests/integration/websocket.py +++ /dev/null @@ -1,38 +0,0 @@ -import json - -import lz4.frame -from channels.testing import WebsocketCommunicator as ChannelsWebsocketCommunicator - - -class WebsocketCommunicator(ChannelsWebsocketCommunicator): - """ - Implements decompression when receiving JSON data. - """ - - async def receive_json_from(self, timeout=1): - """ - Receives a JSON text frame or a compressed JSON bytes object, decompresses and decodes it - """ - payload = await self.receive_from(timeout) - if isinstance(payload, bytes): - # try to decompress the message - uncompressed_data = lz4.frame.decompress(payload) - text_data = uncompressed_data.decode("utf-8") - else: - text_data = payload - - assert isinstance(text_data, str), "JSON data is not a text frame" - return json.loads(text_data) - - async def assert_receive_error(self, timeout=1, in_response=None, **kwargs): - response = await self.receive_json_from(timeout) - assert response["type"] == "error" - - content = response.get("content") - if kwargs: - assert content - for key, value in kwargs.items(): - assert content.get(key) == value - - if in_response: - assert response["in_response"] == in_response diff --git a/server/tests/unit/agenda/test_projector.py b/server/tests/unit/agenda/test_projector.py deleted file mode 100644 index 442be4079..000000000 --- a/server/tests/unit/agenda/test_projector.py +++ /dev/null @@ -1,134 +0,0 @@ -from typing import Any, Dict - -import pytest - -from openslides.agenda import projector - -from ...integration.helpers import get_all_data_provider - - -@pytest.fixture -def all_data_provider(): - data = { - "agenda/item": { - 1: { - "id": 1, - "item_number": "", - "title_information": {"title": "item1"}, - "comment": None, - "closed": False, - "type": 1, - "is_internal": False, - "is_hidden": False, - "duration": None, - "speakers": [], - "speaker_list_closed": False, - "content_object": {"collection": "topics/topic", "id": 1}, - "weight": 10, - "parent_id": None, - }, - 2: { - "id": 2, - "item_number": "", - "title": "item2", - "title_with_type": "item2", - "title_information": {"title": "item2"}, - "comment": None, - "closed": False, - "type": 1, - "is_internal": False, - "is_hidden": False, - "duration": None, - "speakers": [], - "speaker_list_closed": False, - "content_object": {"collection": "topics/topic", "id": 1}, - "weight": 20, - "parent_id": None, - }, - # hidden item - 3: { - "id": 3, - "item_number": "", - "title": "item3", - "title_with_type": "item3", - "title_information": {"title": "item3"}, - "comment": None, - "closed": True, - "type": 2, - "is_internal": False, - "is_hidden": True, - "duration": None, - "speakers": [], - "speaker_list_closed": False, - "content_object": {"collection": "topics/topic", "id": 1}, - "weight": 30, - "parent_id": None, - }, - # Child of item 1 - 4: { - "id": 4, - "item_number": "", - "title_information": {"title": "item4"}, - "comment": None, - "closed": True, - "type": 1, - "is_internal": False, - "is_hidden": False, - "duration": None, - "speakers": [], - "speaker_list_closed": False, - "content_object": {"collection": "topics/topic", "id": 1}, - "weight": 0, - "parent_id": 1, - }, - } - } - - return get_all_data_provider(data) - - -@pytest.mark.asyncio -async def test_main_items(all_data_provider): - element: Dict[str, Any] = {} - - data = await projector.item_list_slide(all_data_provider, element, 1) - - assert data == { - "items": [ - { - "collection": "topics/topic", - "title_information": {"title": "item1", "_agenda_item_number": ""}, - }, - { - "collection": "topics/topic", - "title_information": {"title": "item2", "_agenda_item_number": ""}, - }, - ] - } - - -@pytest.mark.asyncio -async def test_all_items(all_data_provider): - element: Dict[str, Any] = {"only_main_items": False} - - data = await projector.item_list_slide(all_data_provider, element, 1) - - assert data == { - "items": [ - { - "collection": "topics/topic", - "depth": 0, - "title_information": {"title": "item1", "_agenda_item_number": ""}, - }, - { - "collection": "topics/topic", - "depth": 1, - "title_information": {"title": "item4", "_agenda_item_number": ""}, - }, - { - "collection": "topics/topic", - "depth": 0, - "title_information": {"title": "item2", "_agenda_item_number": ""}, - }, - ] - } diff --git a/server/tests/unit/core/__init__.py b/server/tests/unit/core/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/server/tests/unit/core/test_websocket.py b/server/tests/unit/core/test_websocket.py deleted file mode 100644 index cdc5279dc..000000000 --- a/server/tests/unit/core/test_websocket.py +++ /dev/null @@ -1,44 +0,0 @@ -import jsonschema -import pytest - -from openslides.utils.websocket import schema - - -def test_notify_schema_validation(): - # This raises a validaten error if it fails - message = { - "id": "test-message", - "type": "notify", - "content": {"name": "testname", "content": ["some content"]}, - } - jsonschema.validate(message, schema) - - -def test_notify_schema_invalid_str_in_list(): - message = { - "type": "notify", - "content": [{}, "testmessage"], - "id": "test_send_invalid_notify_str_in_list", - } - with pytest.raises(jsonschema.ValidationError): - jsonschema.validate(message, schema) - - -def test_notify_schema_invalid_no_elements(): - message = { - "type": "notify", - "content": [], - "id": "test_send_invalid_notify_str_in_list", - } - with pytest.raises(jsonschema.ValidationError): - jsonschema.validate(message, schema) - - -def test_notify_schema_invalid_not_a_list(): - message = { - "type": "notify", - "content": {"testmessage": "foobar, what else."}, - "id": "test_send_invalid_notify_str_in_list", - } - with pytest.raises(jsonschema.ValidationError): - jsonschema.validate(message, schema) diff --git a/server/tests/unit/motions/test_projector.py b/server/tests/unit/motions/test_projector.py deleted file mode 100644 index 7315af2ee..000000000 --- a/server/tests/unit/motions/test_projector.py +++ /dev/null @@ -1,362 +0,0 @@ -from typing import Any, Dict - -import pytest - -from openslides.motions import projector - -from ...integration.helpers import get_all_data_provider - - -@pytest.fixture -def all_data_provider(): - data = {} - data["motions/motion"] = { - 1: { - "id": 1, - "identifier": "4", - "title": "12345", - "text": "motion text", - "amendment_paragraphs": None, - "modified_final_version": "", - "reason": "", - "parent_id": None, - "category_id": None, - "comments": [], - "motion_block_id": None, - "origin": "", - "submitters": [{"id": 4, "user_id": 1, "motion_id": 1, "weight": 1}], - "supporters_id": [], - "state_id": 1, - "state_extension": None, - "state_restriction": [], - "statute_paragraph_id": None, - "workflow_id": 1, - "recommendation_id": None, - "recommendation_extension": None, - "tags_id": [], - "attachments_id": [], - "polls": [ - { - "id": 1, - "motion_id": 4, - "yes": "10.000000", - "no": "-1.000000", - "abstain": "20.000000", - "votesvalid": "11.000000", - "votesinvalid": "2.000000", - "votescast": "30.000000", - "has_votes": True, - } - ], - "agenda_item_id": 4, - "log_messages": [ - { - "message_list": "['Vote updated']", - "person_id": 1, - "time": "2019-01-19T22:15:53.291123+01:00", - "message": "Jan. 19, 2019, 10:15 p.m. Vote updated by Administrator", - }, - { - "message_list": "['Vote created']", - "person_id": 1, - "time": "2019-01-19T22:15:37.446262+01:00", - "message": "Jan. 19, 2019, 10:15 p.m. Vote created by Administrator", - }, - { - "message_list": "['Motion created']", - "person_id": 1, - "time": "2019-01-19T18:37:34.833749+01:00", - "message": "Jan. 19, 2019, 6:37 p.m. Motion created by Administrator", - }, - ], - "sort_parent_id": None, - "weight": 10000, - "created": "2019-01-19T18:37:34.741336+01:00", - "last_modified": "2019-01-19T18:37:34.741368+01:00", - "change_recommendations_id": [1, 2], - "amendments_id": [2], - }, - 2: { - "id": 2, - "identifier": "Ä1", - "title": "Amendment for 12345", - "text": "", - "amendment_paragraphs": ["New motion text"], - "modified_final_version": "", - "reason": "", - "parent_id": 1, - "category_id": None, - "comments": [], - "motion_block_id": None, - "origin": "", - "submitters": [{"id": 4, "user_id": 1, "motion_id": 1, "weight": 1}], - "supporters_id": [], - "state_id": 1, - "state_extension": None, - "state_restriction": [], - "statute_paragraph_id": None, - "workflow_id": 1, - "recommendation_id": None, - "recommendation_extension": None, - "tags_id": [], - "attachments_id": [], - "polls": [], - "agenda_item_id": 4, - "log_messages": [], - "sort_parent_id": None, - "weight": 10000, - "created": "2019-01-19T18:37:34.741336+01:00", - "last_modified": "2019-01-19T18:37:34.741368+01:00", - "change_recommendations": [], - "change_recommendations_id": [], - "amendments_id": [], - }, - 3: { - "id": 3, - "identifier": None, - "title": "Statute amendment for §1 Preamble", - "text": "

Some other preamble text

", - "amendment_paragraphs": None, - "modified_final_version": "", - "reason": "", - "parent_id": None, - "category_id": None, - "comments": [], - "motion_block_id": None, - "origin": "", - "submitters": [{"id": 4, "user_id": 1, "motion_id": 1, "weight": 1}], - "supporters_id": [], - "state_id": 1, - "state_extension": None, - "state_restriction": [], - "statute_paragraph_id": 1, - "workflow_id": 1, - "recommendation_id": None, - "recommendation_extension": None, - "tags_id": [], - "attachments_id": [], - "polls": [], - "agenda_item_id": 4, - "log_messages": [], - "sort_parent_id": None, - "weight": 10000, - "created": "2019-01-19T18:37:34.741336+01:00", - "last_modified": "2019-01-19T18:37:34.741368+01:00", - "change_recommendations": [], - "change_recommendations_id": [], - "amendments_id": [], - }, - } - data["motions/workflow"] = { - 1: { - "id": 1, - "name": "Simple Workflow", - "states": [1, 2, 3, 4], - "first_state_id": 1, - } - } - data["motions/state"] = { - 1: { - "id": 1, - "name": "submitted", - "recommendation_label": None, - "css_class": "lightblue", - "restriction": [], - "allow_support": True, - "allow_create_poll": True, - "allow_submitter_edit": True, - "dont_set_identifier": False, - "show_state_extension_field": False, - "merge_amendment_into_final": 0, - "show_recommendation_extension_field": False, - "next_states_id": [2, 3, 4], - "workflow_id": 1, - }, - 2: { - "id": 2, - "name": "accepted", - "recommendation_label": "Acceptance", - "css_class": "green", - "restriction": [], - "allow_support": False, - "allow_create_poll": False, - "allow_submitter_edit": False, - "dont_set_identifier": False, - "show_state_extension_field": False, - "merge_amendment_into_final": 1, - "show_recommendation_extension_field": False, - "next_states_id": [], - "workflow_id": 1, - }, - 3: { - "id": 3, - "name": "rejected", - "recommendation_label": "Rejection", - "css_class": "red", - "restriction": [], - "allow_support": False, - "allow_create_poll": False, - "allow_submitter_edit": False, - "dont_set_identifier": False, - "show_state_extension_field": False, - "merge_amendment_into_final": -1, - "show_recommendation_extension_field": False, - "next_states_id": [], - "workflow_id": 1, - }, - 4: { - "id": 4, - "name": "not decided", - "recommendation_label": "No decision", - "css_class": "grey", - "restriction": [], - "allow_support": False, - "allow_create_poll": False, - "allow_submitter_edit": False, - "dont_set_identifier": False, - "show_state_extension_field": False, - "merge_amendment_into_final": -1, - "show_recommendation_extension_field": False, - "next_states_id": [], - "workflow_id": 1, - }, - } - data["motions/statute-paragraph"] = { - 1: { - "id": 1, - "title": "§1 Preamble", - "text": "

Some preamble text

", - "weight": 10000, - } - } - data["motions/motion-change-recommendation"] = { - 1: { - "id": 1, - "motion_id": 1, - "rejected": False, - "internal": True, - "type": 0, - "other_description": "", - "line_from": 1, - "line_to": 2, - "text": "internal new motion text", - "creation_time": "2019-02-09T09:54:06.256378+01:00", - }, - 2: { - "id": 2, - "motion_id": 1, - "rejected": False, - "internal": False, - "type": 0, - "other_description": "", - "line_from": 1, - "line_to": 2, - "text": "public new motion text", - "creation_time": "2019-02-09T09:54:06.256378+01:00", - }, - } - return get_all_data_provider(data) - - -@pytest.mark.asyncio -async def test_motion_slide(all_data_provider): - element: Dict[str, Any] = {"id": 1} - - data = await projector.motion_slide(all_data_provider, element, 1) - - assert data == { - "identifier": "4", - "title": "12345", - "text": "motion text", - "amendments": [ - { - "id": 2, - "title": "Amendment for 12345", - "amendment_paragraphs": ["New motion text"], - "identifier": "Ä1", - "merge_amendment_into_final": 0, - "merge_amendment_into_diff": 0, - "change_recommendations": [], - } - ], - "amendment_paragraphs": None, - "change_recommendations": [ - { - "id": 2, - "motion_id": 1, - "rejected": False, - "internal": False, - "type": 0, - "other_description": "", - "line_from": 1, - "line_to": 2, - "text": "public new motion text", - "creation_time": "2019-02-09T09:54:06.256378+01:00", - } - ], - "base_motion": None, - "base_statute": None, - "is_child": False, - "show_meta_box": False, - "show_referring_motions": True, - "reason": "", - "submitters": ["Administrator"], - "line_length": 85, - "line_numbering_mode": "outside", - "preamble": "The assembly may decide:", - "recommendation_referencing_motions": None, - } - - -@pytest.mark.asyncio -async def test_amendment_slide(all_data_provider): - element: Dict[str, Any] = {"id": 2} - - data = await projector.motion_slide(all_data_provider, element, 1) - - assert data == { - "identifier": "Ä1", - "title": "Amendment for 12345", - "text": "", - "amendments": [], - "amendment_paragraphs": ["New motion text"], - "change_recommendations": [], - "base_motion": {"identifier": "4", "text": "motion text", "title": "12345"}, - "base_statute": None, - "is_child": True, - "show_meta_box": False, - "show_referring_motions": True, - "reason": "", - "submitters": ["Administrator"], - "line_length": 85, - "line_numbering_mode": "outside", - "preamble": "The assembly may decide:", - "recommendation_referencing_motions": None, - } - - -@pytest.mark.asyncio -async def test_statute_amendment_slide(all_data_provider): - element: Dict[str, Any] = {"id": 3} - - data = await projector.motion_slide(all_data_provider, element, 1) - - assert data == { - "identifier": None, - "title": "Statute amendment for §1 Preamble", - "text": "

Some other preamble text

", - "amendments": [], - "amendment_paragraphs": None, - "change_recommendations": [], - "base_motion": None, - "base_statute": {"title": "§1 Preamble", "text": "

Some preamble text

"}, - "is_child": False, - "show_meta_box": False, - "show_referring_motions": True, - "reason": "", - "submitters": ["Administrator"], - "line_length": 85, - "line_numbering_mode": "outside", - "preamble": "The assembly may decide:", - "recommendation_referencing_motions": None, - }