From be83d9b0aa6b7e7ba4823dac6d9d9d0bb013054f Mon Sep 17 00:00:00 2001 From: Maximilian Krambach Date: Mon, 25 Feb 2019 13:29:20 +0100 Subject: [PATCH] bulk import of users --- .../users/user-repository.service.ts | 35 ++++++++++++++----- .../core/ui-services/base-import.service.ts | 1 + client/src/app/site/base/base-import-list.ts | 13 ++++--- .../users/services/user-import.service.ts | 24 ++++++++++--- 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/client/src/app/core/repositories/users/user-repository.service.ts b/client/src/app/core/repositories/users/user-repository.service.ts index 3a5ea016f..a941e2a23 100644 --- a/client/src/app/core/repositories/users/user-repository.service.ts +++ b/client/src/app/core/repositories/users/user-repository.service.ts @@ -1,20 +1,21 @@ import { Injectable } from '@angular/core'; import { Observable, BehaviorSubject } from 'rxjs'; +import { TranslateService } from '@ngx-translate/core'; import { BaseRepository } from '../base-repository'; -import { ViewUser } from 'app/site/users/models/view-user'; -import { User } from 'app/shared/models/users/user'; -import { Group } from 'app/shared/models/users/group'; -import { DataStoreService } from '../../core-services/data-store.service'; -import { DataSendService } from '../../core-services/data-send.service'; -import { Identifiable } from 'app/shared/models/base/identifiable'; import { CollectionStringMapperService } from '../../core-services/collectionStringMapper.service'; import { ConfigService } from 'app/core/ui-services/config.service'; -import { HttpService } from 'app/core/core-services/http.service'; -import { TranslateService } from '@ngx-translate/core'; +import { DataSendService } from '../../core-services/data-send.service'; +import { DataStoreService } from '../../core-services/data-store.service'; import { environment } from '../../../../environments/environment'; -import { ViewModelStoreService } from 'app/core/core-services/view-model-store.service'; +import { Group } from 'app/shared/models/users/group'; +import { HttpService } from 'app/core/core-services/http.service'; +import { Identifiable } from 'app/shared/models/base/identifiable'; +import { NewEntry } from 'app/core/ui-services/base-import.service'; +import { User } from 'app/shared/models/users/user'; +import { ViewUser } from 'app/site/users/models/view-user'; import { ViewGroup } from 'app/site/users/models/view-group'; +import { ViewModelStoreService } from 'app/core/core-services/view-model-store.service'; /** * type for determining the user name from a string during import. @@ -120,6 +121,22 @@ export class UserRepositoryService extends BaseRepository { return await this.dataSend.createModel(newUser); } + /** + * Creates and saves a list of users in a bulk operation. + * + * @param newEntries + */ + public async bulkCreate(newEntries: NewEntry[]): Promise { + const data = newEntries.map(entry => { + return { ...entry.newEntry.user, importTrackId: entry.importTrackId }; + }); + const response = (await this.httpService.post(`rest/users/user/mass_import/`, { users: data })) as { + detail: string; + importedTrackIds: number[]; + }; + return response.importedTrackIds; + } + /** * Generates a random password * diff --git a/client/src/app/core/ui-services/base-import.service.ts b/client/src/app/core/ui-services/base-import.service.ts index c8e41c695..c872476b3 100644 --- a/client/src/app/core/ui-services/base-import.service.ts +++ b/client/src/app/core/ui-services/base-import.service.ts @@ -23,6 +23,7 @@ export interface NewEntry { status: CsvImportStatus; errors: string[]; duplicates: V[]; + importTrackId?: number; } /** diff --git a/client/src/app/site/base/base-import-list.ts b/client/src/app/site/base/base-import-list.ts index cef72bc85..eff9658a3 100644 --- a/client/src/app/site/base/base-import-list.ts +++ b/client/src/app/site/base/base-import-list.ts @@ -7,6 +7,7 @@ import { NewEntry, ValueLabelCombination, BaseImportService } from 'app/core/ui- import { Title } from '@angular/platform-browser'; import { TranslateService } from '@ngx-translate/core'; import { getLongPreview, getShortPreview } from 'app/shared/utils/previewStrings'; +import { auditTime } from 'rxjs/operators'; export abstract class BaseImportListComponent extends BaseViewComponent implements OnInit { /** @@ -131,10 +132,14 @@ export abstract class BaseImportListComponent extends B public initTable(): void { this.dataSource = new MatTableDataSource(); this.setFilter(); - this.importer.getNewEntries().subscribe(newEntries => { - this.dataSource.data = newEntries; - this.hasFile = newEntries.length > 0; - }); + this.importer + .getNewEntries() + .pipe(auditTime(100)) + .subscribe(newEntries => { + this.dataSource.data = []; + this.dataSource.data = newEntries; + this.hasFile = newEntries.length > 0; + }); } /** diff --git a/client/src/app/site/users/services/user-import.service.ts b/client/src/app/site/users/services/user-import.service.ts index 779a4d94b..358e564ac 100644 --- a/client/src/app/site/users/services/user-import.service.ts +++ b/client/src/app/site/users/services/user-import.service.ts @@ -47,7 +47,8 @@ export class UserImportService extends BaseImportService { Duplicates: 'This user already exists', NoName: 'Entry has no valid name', DuplicateImport: 'Entry cannot be imported twice. This line will be ommitted', - ParsingErrors: 'Some csv values could not be read correctly.' + ParsingErrors: 'Some csv values could not be read correctly.', + FailedImport: 'Imported user could not be imported.' }; /** @@ -134,6 +135,8 @@ export class UserImportService extends BaseImportService { */ public async doImport(): Promise { this.newGroups = await this.createNewGroups(); + const importUsers: NewEntry[] = []; + let trackId = 1; for (const entry of this.entries) { if (entry.status !== 'new') { continue; @@ -144,10 +147,23 @@ export class UserImportService extends BaseImportService { this.updatePreview(); continue; } - await this.repo.create(entry.newEntry.user); - entry.status = 'done'; + entry.importTrackId = trackId; + trackId += 1; + importUsers.push(entry); + } + while (importUsers.length) { + const subSet = importUsers.splice(0, 100); // don't send bulks too large + const importedTracks = await this.repo.bulkCreate(subSet); + subSet.map(entry => { + const importModel = this.entries.find(e => e.importTrackId === entry.importTrackId); + if (importModel && importedTracks.includes(importModel.importTrackId)) { + importModel.status = 'done'; + } else { + this.setError(importModel, 'FailedImport'); + } + }); + this.updatePreview(); } - this.updatePreview(); } /**