new autoupdate format

This commit is contained in:
Oskar Hahn 2018-10-14 08:26:51 +02:00 committed by FinnStutzenstein
parent 1ba3af968d
commit 5b5d0e395a
10 changed files with 96 additions and 210 deletions

View File

@ -18,7 +18,7 @@ matrix:
- flake8 openslides tests
- isort --check-only --diff --recursive openslides tests
- python -m mypy openslides/
- pytest tests/old/ tests/integration/ tests/unit/ --cov --cov-fail-under=75
- python -W ignore -m pytest --cov --cov-fail-under=75
- language: python
cache:
@ -35,7 +35,7 @@ matrix:
- flake8 openslides tests
- isort --check-only --diff --recursive openslides tests
- python -m mypy openslides/
- pytest tests/old/ tests/integration/ tests/unit/ --cov --cov-fail-under=75
- python -W ignore -m pytest --cov --cov-fail-under=75
- language: node_js
node_js:

View File

@ -16,6 +16,7 @@ Core:
- Changed URL schema [#3798].
- Enabled docs for using OpenSlides with Gunicorn and Uvicorn in big
mode [#3799, #3817].
- Changed format for elements send via autoupdate [#3926].
Motions:
- Option to customly sort motions [#3894].

View File

@ -6,6 +6,27 @@ import { WebsocketService } from './websocket.service';
import { CollectionStringModelMapperService } from './collectionStringModelMapper.service';
import { DataStoreService } from './data-store.service';
interface AutoupdateFormat {
/**
* All changed (and created) items as their full/restricted data grouped by their collection.
*/
changed: {
[collectionString: string]: object[];
};
/**
* All deleted items (by id) grouped by their collection.
*/
deleted: {
[collectionString: string]: number[];
};
/**
* The current change id for this autoupdate
*/
change_id: number;
}
/**
* Handles the initial update and automatic updates using the {@link WebsocketService}
* Incoming objects, usually BaseModels, will be saved in the dataStore (`this.DS`)
@ -27,7 +48,7 @@ export class AutoupdateService extends OpenSlidesComponent {
private modelMapper: CollectionStringModelMapperService
) {
super();
websocketService.getOberservable<any>('autoupdate').subscribe(response => {
websocketService.getOberservable<AutoupdateFormat>('autoupdate').subscribe(response => {
this.storeResponse(response);
});
}
@ -42,36 +63,19 @@ export class AutoupdateService extends OpenSlidesComponent {
*
* Saves models in DataStore.
*/
public storeResponse(socketResponse: any): void {
// Reorganize the autoupdate: groupy by action, then by collection. The final
// entries are the single autoupdate objects.
const autoupdate = {
changed: {},
deleted: {}
};
// Reorganize them.
socketResponse.forEach(obj => {
if (!autoupdate[obj.action][obj.collection]) {
autoupdate[obj.action][obj.collection] = [];
}
autoupdate[obj.action][obj.collection].push(obj);
});
public storeResponse(autoupdate: AutoupdateFormat): void {
// Delete the removed objects from the DataStore
Object.keys(autoupdate.deleted).forEach(collection => {
this.DS.remove(collection, ...autoupdate.deleted[collection].map(_obj => _obj.id));
this.DS.remove(collection, autoupdate.deleted[collection], autoupdate.change_id);
});
// Add the objects to the DataStore.
Object.keys(autoupdate.changed).forEach(collection => {
const targetClass = this.modelMapper.getModelConstructor(collection);
if (!targetClass) {
// TODO: throw an error later..
/*throw new Error*/ console.log(`Unregistered resource ${collection}`);
return;
throw new Error(`Unregistered resource ${collection}`);
}
this.DS.add(...autoupdate.changed[collection].map(_obj => new targetClass(_obj.data)));
this.DS.add(autoupdate.changed[collection].map(model => new targetClass(model)), autoupdate.change_id);
});
}

View File

@ -261,20 +261,15 @@ export class DataStoreService {
/**
* Add one or multiple models to dataStore.
*
* @param ...models The model(s) that shall be add use spread operator ("...")
* @example this.DS.add(new User(1))
* @example this.DS.add((new User(2), new User(3)))
* @example this.DS.add(...arrayWithUsers)
* @param models BaseModels to add to the store
* @param changeId The changeId of this update
* @example this.DS.add([new User(1)], changeId)
* @example this.DS.add([new User(2), new User(3)], changeId)
* @example this.DS.add(arrayWithUsers, changeId)
*/
public add(...models: BaseModel[]): void {
const maxChangeId = 0;
public add(models: BaseModel[], changeId: number): void {
models.forEach(model => {
const collectionString = model.collectionString;
if (!model.id) {
throw new Error('The model must have an id!');
} else if (collectionString === 'invalid-collection-string') {
throw new Error('Cannot save a BaseModel');
}
if (this.modelStore[collectionString] === undefined) {
this.modelStore[collectionString] = {};
}
@ -284,25 +279,22 @@ export class DataStoreService {
this.JsonStore[collectionString] = {};
}
this.JsonStore[collectionString][model.id] = JSON.stringify(model);
// if (model.changeId > maxChangeId) {maxChangeId = model.maxChangeId;}
this.changedSubject.next(model);
});
this.storeToCache(maxChangeId);
this.storeToCache(changeId);
}
/**
* removes one or multiple models from dataStore.
*
* @param Type The desired BaseModel type to be read from the dataStore
* @param ...ids An or multiple IDs or a list of IDs of BaseModels. use spread operator ("...") for arrays
* @example this.DS.remove('users/user', myUser.id, 3, 4)
* @param Type The desired BaseModel type to be read from the datastore
* @param ids A list of IDs of BaseModels to remove from the datastore
* @param changeId The changeId of this update
* @example this.DS.remove('users/user', [myUser.id, 3, 4], 38213)
*/
public remove(collectionString: string, ...ids: number[]): void {
const maxChangeId = 0;
public remove(collectionString: string, ids: number[], changeId: number): void {
ids.forEach(id => {
if (this.modelStore[collectionString]) {
// get changeId from store
// if (model.changeId > maxChangeId) {maxChangeId = model.maxChangeId;}
delete this.modelStore[collectionString][id];
}
if (this.JsonStore[collectionString]) {
@ -313,18 +305,18 @@ export class DataStoreService {
id: id
});
});
this.storeToCache(maxChangeId);
this.storeToCache(changeId);
}
/**
* Updates the cache by inserting the serialized DataStore. Also changes the chageId, if it's larger
* @param maxChangeId
* @param changeId The changeId from the update. If it's the highest change id seen, it will be set into the cache.
*/
private storeToCache(maxChangeId: number): void {
private storeToCache(changeId: number): void {
this.cacheService.set(DataStoreService.cachePrefix + 'DS', this.JsonStore);
if (maxChangeId > this._maxChangeId) {
this._maxChangeId = maxChangeId;
this.cacheService.set(DataStoreService.cachePrefix + 'maxChangeId', maxChangeId);
if (changeId > this._maxChangeId) {
this._maxChangeId = changeId;
this.cacheService.set(DataStoreService.cachePrefix + 'maxChangeId', changeId);
}
}
@ -334,5 +326,6 @@ export class DataStoreService {
*/
public printWhole(): void {
console.log('Everything in DataStore: ', this.modelStore);
console.log('changeId', this.maxChangeId);
}
}

View File

@ -10,8 +10,5 @@
<button mat-button (click)="TranslateTest()">Translate in console</button>
<br />
<button mat-button (click)="giveDataStore()">print the dataStore</button>
<br />
<input matInput #motionNumber placeholder="Number of Motions to add" value="100">
<button mat-button (click)="createMotions(motionNumber.value)">Add Random Motions</button>
</div>
</mat-card>

View File

@ -6,8 +6,6 @@ import { TranslateService } from '@ngx-translate/core'; // showcase
// for testing the DS and BaseModel
import { Config } from '../../../../shared/models/core/config';
import { Motion } from '../../../../shared/models/motions/motion';
import { MotionSubmitter } from '../../../../shared/models/motions/motion-submitter';
import { DataStoreService } from '../../../../core/services/data-store.service';
@Component({
@ -87,60 +85,4 @@ export class StartComponent extends BaseComponent implements OnInit {
console.log('lets translate the word "motion" in the current in the current lang');
console.log('Motions in ' + this.translate.currentLang + ' is ' + this.translate.instant('Motions'));
}
/**
* Adds random generated motions
*/
public createMotions(requiredMotions: number): void {
console.log('adding ' + requiredMotions + ' Motions.');
const newMotionsArray = [];
const longMotionText = `
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.
Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.
Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat.
Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis.
At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, At accusam aliquyam diam diam dolore dolores duo eirmod eos erat, et nonumy sed tempor et et invidunt justo labore Stet clita ea et gubergren, kasd magna no rebum. sanctus sea sed takimata ut vero voluptua. est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat.
Consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus.
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.
Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.
Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo
`;
for (let i = 1; i <= requiredMotions; ++i) {
// submitter
const newMotionSubmitter = new MotionSubmitter({
id: 1,
user_id: 1,
motion_id: 200 + i,
weight: 0
});
// motion
const newMotion = new Motion({
id: 200 + i,
identifier: 'GenMo ' + i,
title: 'title',
text: longMotionText,
reason: longMotionText,
origin: 'Generated',
submitters: [newMotionSubmitter],
state_id: 1
});
newMotionsArray.push(newMotion);
}
this.DS.add(...newMotionsArray);
console.log('Done adding motions');
}
}

View File

@ -25,6 +25,16 @@ if TYPE_CHECKING:
AutoupdateFormat = TypedDict(
'AutoupdateFormat',
{
'changed': Dict[str, List[Dict[str, Any]]],
'deleted': Dict[str, List[int]],
'change_id': int,
},
)
AutoupdateFormatOld = TypedDict(
'AutoupdateFormatOld',
{
'collection': str,
'id': int,
@ -116,24 +126,7 @@ class CollectionElement:
return (self.collection_string == collection_element.collection_string and
self.id == collection_element.id)
def as_autoupdate_for_user(self, user: Optional['CollectionElement']) -> AutoupdateFormat:
"""
Returns a dict that can be sent through the autoupdate system for a site
user.
"""
if not self.is_deleted():
restricted_data = self.get_access_permissions().get_restricted_data([self.get_full_data()], user)
data = restricted_data[0] if restricted_data else None
else:
data = None
return format_for_autoupdate(
collection_string=self.collection_string,
id=self.id,
action='deleted' if self.is_deleted() else 'changed',
data=data)
def as_autoupdate_for_projector(self) -> AutoupdateFormat:
def as_autoupdate_for_projector(self) -> AutoupdateFormatOld:
"""
Returns a dict that can be sent through the autoupdate system for the
projector.
@ -144,7 +137,7 @@ class CollectionElement:
else:
data = None
return format_for_autoupdate(
return format_for_autoupdate_old(
collection_string=self.collection_string,
id=self.id,
action='deleted' if self.is_deleted() else 'changed',
@ -337,10 +330,12 @@ def get_model_from_collection_string(collection_string: str) -> Type[Model]:
return model
def format_for_autoupdate(
collection_string: str, id: int, action: str, data: Dict[str, Any] = None) -> AutoupdateFormat:
def format_for_autoupdate_old(
collection_string: str, id: int, action: str, data: Dict[str, Any] = None) -> AutoupdateFormatOld:
"""
Returns a dict that can be used for autoupdate.
This is depricated. Use format_for_autoupdate.
"""
if data is None:
# If the data is None then the action has to be deleted,
@ -348,7 +343,7 @@ def format_for_autoupdate(
# deleted, but the user has no permission to see it.
action = 'deleted'
output = AutoupdateFormat(
output = AutoupdateFormatOld(
collection=collection_string,
id=id,
action=action,

View File

@ -1,3 +1,4 @@
from collections import defaultdict
from typing import Any, Dict, List, Optional
import jsonschema
@ -10,9 +11,10 @@ from ..core.models import Projector
from .auth import async_anonymous_is_enabled, has_perm
from .cache import element_cache, split_element_id
from .collection import (
AutoupdateFormat,
Collection,
CollectionElement,
format_for_autoupdate,
format_for_autoupdate_old,
from_channel_message,
)
from .constants import get_constants
@ -169,22 +171,13 @@ class SiteConsumer(ProtocollAsyncJsonWebsocketConsumer):
Send changed or deleted elements to the user.
"""
change_id = event['change_id']
output = []
changed_elements, deleted_elements = await element_cache.get_restricted_data(self.scope['user'], change_id, max_change_id=change_id)
for collection_string, elements in changed_elements.items():
for element in elements:
output.append(format_for_autoupdate(
collection_string=collection_string,
id=element['id'],
action='changed',
data=element))
for element_id in deleted_elements:
changed_elements, deleted_elements_ids = await element_cache.get_restricted_data(self.scope['user'], change_id, max_change_id=change_id)
deleted_elements: Dict[str, List[int]] = defaultdict(list)
for element_id in deleted_elements_ids:
collection_string, id = split_element_id(element_id)
output.append(format_for_autoupdate(
collection_string=collection_string,
id=id,
action='deleted'))
await self.send_json(type='autoupdate', content=output)
deleted_elements[collection_string].append(id)
await self.send_json(type='autoupdate', content=AutoupdateFormat(changed=changed_elements, deleted=deleted_elements, change_id=change_id))
class ProjectorConsumer(ProtocollAsyncJsonWebsocketConsumer):
@ -274,23 +267,21 @@ class ProjectorConsumer(ProtocollAsyncJsonWebsocketConsumer):
await self.send_json(type='autoupdate', content=output)
async def startup_data(user: Optional[CollectionElement], change_id: int = 0) -> List[Any]:
async def startup_data(user: Optional[CollectionElement], change_id: int = 0) -> AutoupdateFormat:
"""
Returns all data for startup.
"""
# TODO: use the change_id argument
output = []
restricted_data = await element_cache.get_all_restricted_data(user)
for collection_string, elements in restricted_data.items():
for element in elements:
formatted_data = format_for_autoupdate(
collection_string=collection_string,
id=element['id'],
action='changed',
data=element)
# TODO: This two calls have to be atomic
changed_elements, deleted_element_ids = await element_cache.get_restricted_data(user)
current_change_id = await element_cache.get_current_change_id()
output.append(formatted_data)
return output
deleted_elements: Dict[str, List[int]] = defaultdict(list)
for element_id in deleted_element_ids:
collection_string, id = split_element_id(element_id)
deleted_elements[collection_string].append(id)
return AutoupdateFormat(changed=changed_elements, deleted=deleted_elements, change_id=current_change_id)
def projector_startup_data(projector_id: int) -> Any:
@ -318,7 +309,7 @@ def projector_startup_data(projector_id: int) -> Any:
projector_data = (config_collection.get_access_permissions()
.get_projector_data(config_collection.get_full_data()))
for data in projector_data:
output.append(format_for_autoupdate(
output.append(format_for_autoupdate_old(
config_collection.collection_string,
data['id'],
'changed',

View File

@ -59,8 +59,13 @@ async def test_normal_connection(communicator):
type = response.get('type')
content = response.get('content')
assert type == 'autoupdate'
# Test, that both example objects are returned
assert len(content) > 10
assert 'changed' in content
assert 'deleted' in content
assert 'change_id' in content
assert Collection1().get_collection_string() in content['changed']
assert Collection2().get_collection_string() in content['changed']
assert TConfig().get_collection_string() in content['changed']
assert TUser().get_collection_string() in content['changed']
@pytest.mark.asyncio
@ -77,11 +82,8 @@ async def test_receive_changed_data(communicator):
type = response.get('type')
content = response.get('content')
assert type == 'autoupdate'
assert content == [
{'action': 'changed',
'collection': 'core/config',
'data': {'id': id, 'key': 'general_event_name', 'value': 'Test Event'},
'id': id}]
assert content['changed'] == {
'core/config': [{'id': id, 'key': 'general_event_name', 'value': 'Test Event'}]}
@pytest.mark.asyncio
@ -124,7 +126,7 @@ async def test_receive_deleted_data(communicator):
type = response.get('type')
content = response.get('content')
assert type == 'autoupdate'
assert content == [{'action': 'deleted', 'collection': Collection1().get_collection_string(), 'id': 1}]
assert content['deleted'] == {Collection1().get_collection_string(): [1]}
@pytest.mark.asyncio

View File

@ -1,5 +1,5 @@
from unittest import TestCase
from unittest.mock import MagicMock, patch
from unittest.mock import patch
from openslides.core.models import Projector
from openslides.utils import collection
@ -43,45 +43,6 @@ class TestCollectionElement(TestCase):
self.assertEqual(created_collection_element.full_data, {'data': 'value'})
self.assertEqual(created_collection_element.information, {'some': 'information'})
def test_as_autoupdate_for_user(self):
with patch.object(collection.CollectionElement, 'get_full_data'):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
fake_user = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_restricted_data.return_value = ['restricted_data']
collection_element.get_full_data = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'changed',
'data': 'restricted_data'})
def test_as_autoupdate_for_user_no_permission(self):
with patch.object(collection.CollectionElement, 'get_full_data'):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42)
fake_user = MagicMock()
collection_element.get_access_permissions = MagicMock()
collection_element.get_access_permissions().get_restricted_data.return_value = None
collection_element.get_full_data = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'deleted'})
def test_as_autoupdate_for_user_deleted(self):
collection_element = collection.CollectionElement.from_values('testmodule/model', 42, deleted=True)
fake_user = MagicMock()
self.assertEqual(
collection_element.as_autoupdate_for_user(fake_user),
{'collection': 'testmodule/model',
'id': 42,
'action': 'deleted'})
@patch.object(collection.CollectionElement, 'get_full_data')
def test_equal(self, mock_get_full_data):
self.assertEqual(