Merge pull request #4399 from MaximilianKrambach/bulkUser

bulk import of users
This commit is contained in:
Emanuel Schütze 2019-02-26 22:24:53 +01:00 committed by GitHub
commit 2a35a6010a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 56 additions and 17 deletions

View File

@ -1,20 +1,21 @@
import { Injectable } from '@angular/core'; import { Injectable } from '@angular/core';
import { Observable, BehaviorSubject } from 'rxjs'; import { Observable, BehaviorSubject } from 'rxjs';
import { TranslateService } from '@ngx-translate/core';
import { BaseRepository } from '../base-repository'; import { BaseRepository } from '../base-repository';
import { ViewUser } from 'app/site/users/models/view-user';
import { User } from 'app/shared/models/users/user';
import { Group } from 'app/shared/models/users/group';
import { DataStoreService } from '../../core-services/data-store.service';
import { DataSendService } from '../../core-services/data-send.service';
import { Identifiable } from 'app/shared/models/base/identifiable';
import { CollectionStringMapperService } from '../../core-services/collectionStringMapper.service'; import { CollectionStringMapperService } from '../../core-services/collectionStringMapper.service';
import { ConfigService } from 'app/core/ui-services/config.service'; import { ConfigService } from 'app/core/ui-services/config.service';
import { HttpService } from 'app/core/core-services/http.service'; import { DataSendService } from '../../core-services/data-send.service';
import { TranslateService } from '@ngx-translate/core'; import { DataStoreService } from '../../core-services/data-store.service';
import { environment } from '../../../../environments/environment'; import { environment } from '../../../../environments/environment';
import { ViewModelStoreService } from 'app/core/core-services/view-model-store.service'; import { Group } from 'app/shared/models/users/group';
import { HttpService } from 'app/core/core-services/http.service';
import { Identifiable } from 'app/shared/models/base/identifiable';
import { NewEntry } from 'app/core/ui-services/base-import.service';
import { User } from 'app/shared/models/users/user';
import { ViewUser } from 'app/site/users/models/view-user';
import { ViewGroup } from 'app/site/users/models/view-group'; import { ViewGroup } from 'app/site/users/models/view-group';
import { ViewModelStoreService } from 'app/core/core-services/view-model-store.service';
/** /**
* type for determining the user name from a string during import. * type for determining the user name from a string during import.
@ -120,6 +121,22 @@ export class UserRepositoryService extends BaseRepository<ViewUser, User> {
return await this.dataSend.createModel(newUser); return await this.dataSend.createModel(newUser);
} }
/**
* Creates and saves a list of users in a bulk operation.
*
* @param newEntries
*/
public async bulkCreate(newEntries: NewEntry<ViewUser>[]): Promise<number[]> {
const data = newEntries.map(entry => {
return { ...entry.newEntry.user, importTrackId: entry.importTrackId };
});
const response = (await this.httpService.post(`rest/users/user/mass_import/`, { users: data })) as {
detail: string;
importedTrackIds: number[];
};
return response.importedTrackIds;
}
/** /**
* Generates a random password * Generates a random password
* *

View File

@ -23,6 +23,7 @@ export interface NewEntry<V> {
status: CsvImportStatus; status: CsvImportStatus;
errors: string[]; errors: string[];
duplicates: V[]; duplicates: V[];
importTrackId?: number;
} }
/** /**

View File

@ -7,6 +7,7 @@ import { NewEntry, ValueLabelCombination, BaseImportService } from 'app/core/ui-
import { Title } from '@angular/platform-browser'; import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core'; import { TranslateService } from '@ngx-translate/core';
import { getLongPreview, getShortPreview } from 'app/shared/utils/previewStrings'; import { getLongPreview, getShortPreview } from 'app/shared/utils/previewStrings';
import { auditTime } from 'rxjs/operators';
export abstract class BaseImportListComponent<V extends BaseViewModel> extends BaseViewComponent implements OnInit { export abstract class BaseImportListComponent<V extends BaseViewModel> extends BaseViewComponent implements OnInit {
/** /**
@ -131,7 +132,11 @@ export abstract class BaseImportListComponent<V extends BaseViewModel> extends B
public initTable(): void { public initTable(): void {
this.dataSource = new MatTableDataSource(); this.dataSource = new MatTableDataSource();
this.setFilter(); this.setFilter();
this.importer.getNewEntries().subscribe(newEntries => { this.importer
.getNewEntries()
.pipe(auditTime(100))
.subscribe(newEntries => {
this.dataSource.data = [];
this.dataSource.data = newEntries; this.dataSource.data = newEntries;
this.hasFile = newEntries.length > 0; this.hasFile = newEntries.length > 0;
}); });

View File

@ -47,7 +47,8 @@ export class UserImportService extends BaseImportService<ViewUser> {
Duplicates: 'This user already exists', Duplicates: 'This user already exists',
NoName: 'Entry has no valid name', NoName: 'Entry has no valid name',
DuplicateImport: 'Entry cannot be imported twice. This line will be ommitted', DuplicateImport: 'Entry cannot be imported twice. This line will be ommitted',
ParsingErrors: 'Some csv values could not be read correctly.' ParsingErrors: 'Some csv values could not be read correctly.',
FailedImport: 'Imported user could not be imported.'
}; };
/** /**
@ -134,6 +135,8 @@ export class UserImportService extends BaseImportService<ViewUser> {
*/ */
public async doImport(): Promise<void> { public async doImport(): Promise<void> {
this.newGroups = await this.createNewGroups(); this.newGroups = await this.createNewGroups();
const importUsers: NewEntry<ViewUser>[] = [];
let trackId = 1;
for (const entry of this.entries) { for (const entry of this.entries) {
if (entry.status !== 'new') { if (entry.status !== 'new') {
continue; continue;
@ -144,11 +147,24 @@ export class UserImportService extends BaseImportService<ViewUser> {
this.updatePreview(); this.updatePreview();
continue; continue;
} }
await this.repo.create(entry.newEntry.user); entry.importTrackId = trackId;
entry.status = 'done'; trackId += 1;
importUsers.push(entry);
} }
while (importUsers.length) {
const subSet = importUsers.splice(0, 100); // don't send bulks too large
const importedTracks = await this.repo.bulkCreate(subSet);
subSet.map(entry => {
const importModel = this.entries.find(e => e.importTrackId === entry.importTrackId);
if (importModel && importedTracks.includes(importModel.importTrackId)) {
importModel.status = 'done';
} else {
this.setError(importModel, 'FailedImport');
}
});
this.updatePreview(); this.updatePreview();
} }
}
/** /**
* extracts the group(s) from a csv column and tries to match them against existing groups, * extracts the group(s) from a csv column and tries to match them against existing groups,