Merge pull request #4874 from FinnStutzenstein/history

Improve history

client/src/app/core/core-services/time-travel.service.ts

@@ -11,17 +11,8 @@ import { OpenSlidesStatusService } from './openslides-status.service';
 import { OpenSlidesService } from './openslides.service';
 import { WebsocketService } from './websocket.service';
 
-/**
- * Interface for full history data objects.
- * The are not too different from the history-objects,
- * but contain full-data and a timestamp in contrast to a date
- */
 interface HistoryData {
-    element_id: string;
+    [collection: string]: BaseModel[];
-    full_data: BaseModel;
-    information: string;
-    timestamp: number;
-    user_id: number;
 }
 
 /**
@@ -65,19 +56,16 @@ export class TimeTravelService {
         const updateSlot = await this.DSUpdateManager.getNewUpdateSlot(this.DS);
 
         await this.stopTime(history);
-        const fullDataHistory: HistoryData[] = await this.getHistoryData(history);
+        const historyData: HistoryData = await this.getHistoryData(history);
-        for (const historyObject of fullDataHistory) {
-            let collectionString: string;
-            let id: string;
-            [collectionString, id] = historyObject.element_id.split(':');
 
-            if (historyObject.full_data) {
+        const allModels = [];
-                const targetClass = this.modelMapperService.getModelConstructor(collectionString);
+        Object.keys(historyData).forEach(collection => {
-                await this.DS.add([new targetClass(historyObject.full_data)]);
+            const targetClass = this.modelMapperService.getModelConstructor(collection);
-            } else {
+            historyData[collection].forEach(model => {
-                await this.DS.remove(collectionString, [+id]);
+                allModels.push(new targetClass(model));
-            }
+            });
-        }
+        });
+        await this.DS.set(allModels, 0);
 
         this.DSUpdateManager.commit(updateSlot);
     }
@@ -99,9 +87,13 @@ export class TimeTravelService {
      * @param date the Date object
      * @returns the full history on the given date
      */
-    private async getHistoryData(history: History): Promise<HistoryData[]> {
+    private async getHistoryData(history: History): Promise<HistoryData> {
         const queryParams = { timestamp: Math.ceil(history.timestamp) };
-        return this.httpService.get<HistoryData[]>(`${environment.urlPrefix}/core/history/data/`, null, queryParams);
+        return await this.httpService.get<HistoryData>(
+            `${environment.urlPrefix}/core/history/data/`,
+            null,
+            queryParams
+        );
     }
 
     /**

client/src/app/core/ui-services/base-filter-list.service.ts

@@ -3,6 +3,7 @@ import { BehaviorSubject, Observable, Subscription } from 'rxjs';
 import { BaseModel } from 'app/shared/models/base/base-model';
 import { BaseRepository } from '../repositories/base-repository';
 import { BaseViewModel, TitleInformation } from '../../site/base/base-view-model';
+import { OpenSlidesStatusService } from '../core-services/openslides-status.service';
 import { StorageService } from '../core-services/storage.service';
 
 /**
@@ -149,7 +150,11 @@ export abstract class BaseFilterListService<V extends BaseViewModel> {
      * @param name the name of the filter service
      * @param store storage service, to read saved filter variables
      */
-    public constructor(protected name: string, private store: StorageService) {}
+    public constructor(
+        protected name: string,
+        private store: StorageService,
+        private OSStatus: OpenSlidesStatusService
+    ) {}
 
     /**
      * Initializes the filterService.
@@ -157,7 +162,10 @@ export abstract class BaseFilterListService<V extends BaseViewModel> {
      * @param inputData Observable array with ViewModels
      */
     public async initFilters(inputData: Observable<V[]>): Promise<void> {
-        const storedFilter = await this.store.get<OsFilter[]>('filter_' + this.name);
+        let storedFilter: OsFilter[] = null;
+        if (!this.OSStatus.isInHistoryMode) {
+            storedFilter = await this.store.get<OsFilter[]>('filter_' + this.name);
+        }
 
         if (storedFilter && this.isOsFilter(storedFilter)) {
             this.filterDefinitions = storedFilter;
@@ -221,39 +229,42 @@ export abstract class BaseFilterListService<V extends BaseViewModel> {
      * Takes the filter definition from children and using {@link getFilterDefinitions}
      * and sets/updates {@link filterDefinitions}
      */
-    public setFilterDefinitions(): void {
+    public async setFilterDefinitions(): Promise<void> {
         if (this.filterDefinitions) {
             const newDefinitions = this.getFilterDefinitions();
 
-            this.store.get<OsFilter[]>('filter_' + this.name).then(storedFilter => {
+            let storedFilter = null;
-                if (!!storedFilter) {
+            if (!this.OSStatus.isInHistoryMode) {
-                    for (const newDef of newDefinitions) {
+                storedFilter = await this.store.get<OsFilter[]>('filter_' + this.name);
-                        let count = 0;
+            }
-                        const matchingExistingFilter = storedFilter.find(oldDef => oldDef.property === newDef.property);
+
-                        for (const option of newDef.options) {
+            if (!!storedFilter) {
-                            if (typeof option === 'object') {
+                for (const newDef of newDefinitions) {
-                                if (matchingExistingFilter && matchingExistingFilter.options) {
+                    let count = 0;
-                                    const existingOption = matchingExistingFilter.options.find(
+                    const matchingExistingFilter = storedFilter.find(oldDef => oldDef.property === newDef.property);
-                                        o =>
+                    for (const option of newDef.options) {
-                                            typeof o !== 'string' &&
+                        if (typeof option === 'object') {
-                                            JSON.stringify(o.condition) === JSON.stringify(option.condition)
+                            if (matchingExistingFilter && matchingExistingFilter.options) {
-                                    ) as OsFilterOption;
+                                const existingOption = matchingExistingFilter.options.find(
-                                    if (existingOption) {
+                                    o =>
-                                        option.isActive = existingOption.isActive;
+                                        typeof o !== 'string' &&
-                                    }
+                                        JSON.stringify(o.condition) === JSON.stringify(option.condition)
-                                    if (option.isActive) {
+                                ) as OsFilterOption;
-                                        count++;
+                                if (existingOption) {
-                                    }
+                                    option.isActive = existingOption.isActive;
+                                }
+                                if (option.isActive) {
+                                    count++;
                                 }
                             }
                         }
-                        newDef.count = count;
                     }
+                    newDef.count = count;
                 }
+            }
 
-                this.filterDefinitions = newDefinitions;
+            this.filterDefinitions = newDefinitions;
-                this.storeActiveFilters();
+            this.storeActiveFilters();
-            });
         }
     }
 
@@ -302,7 +313,9 @@ export abstract class BaseFilterListService<V extends BaseViewModel> {
      */
     public storeActiveFilters(): void {
         this.updateFilteredData();
-        this.store.set('filter_' + this.name, this.filterDefinitions);
+        if (!this.OSStatus.isInHistoryMode) {
+            this.store.set('filter_' + this.name, this.filterDefinitions);
+        }
     }
 
     /**

client/src/app/core/ui-services/base-sort-list.service.ts

@@ -2,6 +2,7 @@ import { TranslateService } from '@ngx-translate/core';
 import { BehaviorSubject, Observable, Subscription } from 'rxjs';
 
 import { BaseViewModel } from '../../site/base/base-view-model';
+import { OpenSlidesStatusService } from '../core-services/openslides-status.service';
 import { StorageService } from '../core-services/storage.service';
 
 /**
@@ -119,7 +120,12 @@ export abstract class BaseSortListService<V extends BaseViewModel> {
      * @param translate required for language sensitive comparing
      * @param store to save and load sorting preferences
      */
-    public constructor(protected name: string, protected translate: TranslateService, private store: StorageService) {}
+    public constructor(
+        protected name: string,
+        protected translate: TranslateService,
+        private store: StorageService,
+        private OSStatus: OpenSlidesStatusService
+    ) {}
 
     /**
      * Enforce children to implement a method that returns the fault sorting
@@ -139,7 +145,12 @@ export abstract class BaseSortListService<V extends BaseViewModel> {
         }
 
         if (!this.sortDefinition) {
-            this.sortDefinition = await this.store.get<OsSortingDefinition<V> | null>('sorting_' + this.name);
+            if (this.OSStatus.isInHistoryMode) {
+                this.sortDefinition = null;
+            } else {
+                this.sortDefinition = await this.store.get<OsSortingDefinition<V> | null>('sorting_' + this.name);
+            }
+
             if (this.sortDefinition && this.sortDefinition.sortProperty) {
                 this.updateSortedData();
             } else {
@@ -198,7 +209,9 @@ export abstract class BaseSortListService<V extends BaseViewModel> {
      */
     private updateSortDefinitions(): void {
         this.updateSortedData();
-        this.store.set('sorting_' + this.name, this.sortDefinition);
+        if (!this.OSStatus.isInHistoryMode) {
+            this.store.set('sorting_' + this.name, this.sortDefinition);
+        }
     }
 
     /**

client/src/app/shared/models/core/history.ts

@@ -8,8 +8,7 @@ import { Deserializable } from '../base/deserializable';
 export class History implements Deserializable {
     public element_id: string;
     public timestamp: number;
-    public information: string;
+    public information: string[];
-    public restricted: boolean;
     public user_id: number;
 
     /**

client/src/app/site/agenda/services/agenda-filter-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { BaseFilterListService, OsFilter, OsFilterOption } from 'app/core/ui-services/base-filter-list.service';
 import { ItemVisibilityChoices } from 'app/shared/models/agenda/item';
@@ -20,8 +21,8 @@ export class AgendaFilterListService extends BaseFilterListService<ViewItem> {
      * @param store
      * @param translate Translation service
      */
-    public constructor(store: StorageService, private translate: TranslateService) {
+    public constructor(store: StorageService, OSStatus: OpenSlidesStatusService, private translate: TranslateService) {
-        super('Agenda', store);
+        super('Agenda', store, OSStatus);
     }
 
     /**

client/src/app/site/assignments/services/assignment-filter.service.ts

@@ -1,5 +1,6 @@
 import { Injectable } from '@angular/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { BaseFilterListService, OsFilter, OsFilterOption } from 'app/core/ui-services/base-filter-list.service';
 import { AssignmentPhases, ViewAssignment } from '../models/view-assignment';
@@ -17,8 +18,8 @@ export class AssignmentFilterListService extends BaseFilterListService<ViewAssig
      * @param store StorageService
      * @param translate translate service
      */
-    public constructor(store: StorageService) {
+    public constructor(store: StorageService, OSStatus: OpenSlidesStatusService) {
-        super('Assignments', store);
+        super('Assignments', store, OSStatus);
     }
 
     /**

client/src/app/site/assignments/services/assignment-sort-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { BaseSortListService, OsSortingDefinition, OsSortingOption } from 'app/core/ui-services/base-sort-list.service';
 import { ViewAssignment } from '../models/view-assignment';
@@ -29,8 +30,8 @@ export class AssignmentSortListService extends BaseSortListService<ViewAssignmen
      * @param translate required by parent
      * @param storage required by parent
      */
-    public constructor(translate: TranslateService, storage: StorageService) {
+    public constructor(translate: TranslateService, storage: StorageService, OSStatus: OpenSlidesStatusService) {
-        super('Assignment', translate, storage);
+        super('Assignment', translate, storage, OSStatus);
     }
 
     /**

client/src/app/site/mediafiles/services/mediafiles-sort-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { BaseSortListService, OsSortingDefinition, OsSortingOption } from 'app/core/ui-services/base-sort-list.service';
 import { ViewMediafile } from '../models/view-mediafile';
@@ -31,8 +32,8 @@ export class MediafilesSortListService extends BaseSortListService<ViewMediafile
      * @param translate required by parent
      * @param store required by parent
      */
-    public constructor(translate: TranslateService, store: StorageService) {
+    public constructor(translate: TranslateService, store: StorageService, OSStatus: OpenSlidesStatusService) {
-        super('Mediafiles', translate, store);
+        super('Mediafiles', translate, store, OSStatus);
     }
 
     /**

client/src/app/site/motions/services/motion-block-sort.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { BaseSortListService, OsSortingDefinition, OsSortingOption } from 'app/core/ui-services/base-sort-list.service';
 import { ViewMotionBlock } from '../models/view-motion-block';
@@ -12,8 +13,8 @@ import { ViewMotionBlock } from '../models/view-motion-block';
 export class MotionBlockSortService extends BaseSortListService<ViewMotionBlock> {
     public sortOptions: OsSortingOption<ViewMotionBlock>[] = [{ property: 'title' }];
 
-    public constructor(translate: TranslateService, store: StorageService) {
+    public constructor(translate: TranslateService, store: StorageService, OSStatus: OpenSlidesStatusService) {
-        super('Motion block', translate, store);
+        super('Motion block', translate, store, OSStatus);
     }
 
     protected async getDefaultDefinition(): Promise<OsSortingDefinition<ViewMotionBlock>> {

client/src/app/site/motions/services/motion-filter-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { OperatorService } from 'app/core/core-services/operator.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { CategoryRepositoryService } from 'app/core/repositories/motions/category-repository.service';
@@ -123,6 +124,7 @@ export class MotionFilterListService extends BaseFilterListService<ViewMotion> {
      */
     public constructor(
         store: StorageService,
+        OSStatus: OpenSlidesStatusService,
         categoryRepo: CategoryRepositoryService,
         motionBlockRepo: MotionBlockRepositoryService,
         commentRepo: MotionCommentSectionRepositoryService,
@@ -132,7 +134,7 @@ export class MotionFilterListService extends BaseFilterListService<ViewMotion> {
         private operator: OperatorService,
         private config: ConfigService
     ) {
-        super('Motion', store);
+        super('Motion', store, OSStatus);
         this.getWorkflowConfig();
 
         this.updateFilterForRepo(categoryRepo, this.categoryFilterOptions, this.translate.instant('No category set'));

client/src/app/site/motions/services/motion-sort-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { Deferred } from 'app/core/deferred';
 import { _ } from 'app/core/translate/translation-marker';
@@ -48,8 +49,13 @@ export class MotionSortListService extends BaseSortListService<ViewMotion> {
      * @param store required by parent
      * @param config set the default sorting according to OpenSlides configuration
      */
-    public constructor(translate: TranslateService, store: StorageService, private config: ConfigService) {
+    public constructor(
-        super('Motion', translate, store);
+        translate: TranslateService,
+        store: StorageService,
+        OSStatus: OpenSlidesStatusService,
+        private config: ConfigService
+    ) {
+        super('Motion', translate, store, OSStatus);
 
         this.config.get<string>('motions_motions_sorting').subscribe(defSortProp => {
             if (defSortProp) {

client/src/app/site/users/services/user-filter-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { GroupRepositoryService } from 'app/core/repositories/users/group-repository.service';
 import { BaseFilterListService, OsFilter } from 'app/core/ui-services/base-filter-list.service';
@@ -28,8 +29,13 @@ export class UserFilterListService extends BaseFilterListService<ViewUser> {
      * @param groupRepo to filter by groups
      * @param translate marking some translations that are unique here
      */
-    public constructor(store: StorageService, groupRepo: GroupRepositoryService, private translate: TranslateService) {
+    public constructor(
-        super('User', store);
+        store: StorageService,
+        OSStatus: OpenSlidesStatusService,
+        groupRepo: GroupRepositoryService,
+        private translate: TranslateService
+    ) {
+        super('User', store, OSStatus);
         this.updateFilterForRepo(groupRepo, this.userGroupFilterOptions, this.translate.instant('Default'), [1]);
     }
 

client/src/app/site/users/services/user-sort-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@angular/core';
 
 import { TranslateService } from '@ngx-translate/core';
 
+import { OpenSlidesStatusService } from 'app/core/core-services/openslides-status.service';
 import { StorageService } from 'app/core/core-services/storage.service';
 import { BaseSortListService, OsSortingDefinition, OsSortingOption } from 'app/core/ui-services/base-sort-list.service';
 import { ViewUser } from '../models/view-user';
@@ -34,8 +35,8 @@ export class UserSortListService extends BaseSortListService<ViewUser> {
      * @param translate required by parent
      * @param store requires by parent
      */
-    public constructor(translate: TranslateService, store: StorageService) {
+    public constructor(translate: TranslateService, store: StorageService, OSStatus: OpenSlidesStatusService) {
-        super('User', translate, store);
+        super('User', translate, store, OSStatus);
     }
 
     /**

openslides/core/views.py

@@ -1,5 +1,6 @@
 import datetime
 import os
+from collections import defaultdict
 from typing import Any, Dict
 
 from django.conf import settings
@@ -11,6 +12,8 @@ from django.utils.timezone import now
 from django.views import static
 from django.views.generic.base import View
 
+from openslides.utils.utils import split_element_id
+
 from .. import __license__ as license, __url__ as url, __version__ as version
 from ..users.models import User
 from ..utils import views as utils_views
@@ -523,15 +526,15 @@ class HistoryInformationView(utils_views.APIView):
         """
         data = []
         for instance in History.objects.filter(element_id=value).order_by("-now"):
-            data.append(
+            if instance.information:
-                {
+                data.append(
-                    "element_id": instance.element_id,
+                    {
-                    "timestamp": instance.now.timestamp(),
+                        "element_id": instance.element_id,
-                    "information": instance.information,
+                        "timestamp": instance.now.timestamp(),
-                    "resticted": instance.restricted,
+                        "information": instance.information,
-                    "user_id": instance.user.pk if instance.user else None,
+                        "user_id": instance.user.pk if instance.user else None,
-                }
+                    }
-            )
+                )
         return data
 
     def delete(self, request, *args, **kwargs):
@@ -563,8 +566,8 @@ class HistoryDataView(utils_views.APIView):
 
     def get_context_data(self, **context):
         """
-        Checks if user is in admin group. If yes all history data until
+        Checks if user is in admin group. If yes, all history data until
-        (including) timestamp are added to the response data.
+        (including) timestamp are collected to build a valid dataset for the client.
         """
         if not in_some_groups(self.request.user.pk or 0, [GROUP_ADMIN_PK]):
             self.permission_denied(self.request)
@@ -572,22 +575,25 @@ class HistoryDataView(utils_views.APIView):
             timestamp = int(self.request.query_params.get("timestamp", 0))
         except ValueError:
             raise ValidationError(
-                {"detail": "Invalid input. Timestamp  should be an integer."}
+                {"detail": "Invalid input. Timestamp should be an integer."}
             )
-        data = []
         queryset = History.objects.select_related("full_data")
         if timestamp:
             queryset = queryset.filter(
                 now__lte=datetime.datetime.fromtimestamp(timestamp)
             )
+
+        # collection <--> id <--> full_data
+        dataset: Dict[str, Dict[int, Any]] = defaultdict(dict)
         for instance in queryset:
-            data.append(
+            collection, id = split_element_id(instance.element_id)
-                {
+            full_data = instance.full_data.full_data
-                    "full_data": instance.full_data.full_data,
+            if full_data:
-                    "element_id": instance.element_id,
+                dataset[collection][id] = full_data
-                    "timestamp": instance.now.timestamp(),
+            else:
-                    "information": instance.information,
+                del dataset[collection][id]
-                    "user_id": instance.user.pk if instance.user else None,
+
-                }
+        return {
-            )
+            collection: list(dataset[collection].values())
-        return data
+            for collection in dataset.keys()
+        }