Merge pull request #4399 from MaximilianKrambach/bulkUser

bulk import of users

client/src/app/core/repositories/users/user-repository.service.ts

@@ -1,20 +1,21 @@
 import { Injectable } from '@angular/core';
 import { Observable, BehaviorSubject } from 'rxjs';
+import { TranslateService } from '@ngx-translate/core';
 
 import { BaseRepository } from '../base-repository';
-import { ViewUser } from 'app/site/users/models/view-user';
-import { User } from 'app/shared/models/users/user';
-import { Group } from 'app/shared/models/users/group';
-import { DataStoreService } from '../../core-services/data-store.service';
-import { DataSendService } from '../../core-services/data-send.service';
-import { Identifiable } from 'app/shared/models/base/identifiable';
 import { CollectionStringMapperService } from '../../core-services/collectionStringMapper.service';
 import { ConfigService } from 'app/core/ui-services/config.service';
-import { HttpService } from 'app/core/core-services/http.service';
+import { DataSendService } from '../../core-services/data-send.service';
-import { TranslateService } from '@ngx-translate/core';
+import { DataStoreService } from '../../core-services/data-store.service';
 import { environment } from '../../../../environments/environment';
-import { ViewModelStoreService } from 'app/core/core-services/view-model-store.service';
+import { Group } from 'app/shared/models/users/group';
+import { HttpService } from 'app/core/core-services/http.service';
+import { Identifiable } from 'app/shared/models/base/identifiable';
+import { NewEntry } from 'app/core/ui-services/base-import.service';
+import { User } from 'app/shared/models/users/user';
+import { ViewUser } from 'app/site/users/models/view-user';
 import { ViewGroup } from 'app/site/users/models/view-group';
+import { ViewModelStoreService } from 'app/core/core-services/view-model-store.service';
 
 /**
  * type for determining the user name from a string during import.
@@ -120,6 +121,22 @@ export class UserRepositoryService extends BaseRepository<ViewUser, User> {
         return await this.dataSend.createModel(newUser);
     }
 
+    /**
+     * Creates and saves a list of users in a bulk operation.
+     *
+     * @param newEntries
+     */
+    public async bulkCreate(newEntries: NewEntry<ViewUser>[]): Promise<number[]> {
+        const data = newEntries.map(entry => {
+            return { ...entry.newEntry.user, importTrackId: entry.importTrackId };
+        });
+        const response = (await this.httpService.post(`rest/users/user/mass_import/`, { users: data })) as {
+            detail: string;
+            importedTrackIds: number[];
+        };
+        return response.importedTrackIds;
+    }
+
     /**
      * Generates a random password
      *

client/src/app/core/ui-services/base-import.service.ts

@@ -23,6 +23,7 @@ export interface NewEntry<V> {
     status: CsvImportStatus;
     errors: string[];
     duplicates: V[];
+    importTrackId?: number;
 }
 
 /**

client/src/app/site/base/base-import-list.ts

@@ -7,6 +7,7 @@ import { NewEntry, ValueLabelCombination, BaseImportService } from 'app/core/ui-
 import { Title } from '@angular/platform-browser';
 import { TranslateService } from '@ngx-translate/core';
 import { getLongPreview, getShortPreview } from 'app/shared/utils/previewStrings';
+import { auditTime } from 'rxjs/operators';
 
 export abstract class BaseImportListComponent<V extends BaseViewModel> extends BaseViewComponent implements OnInit {
     /**
@@ -131,7 +132,11 @@ export abstract class BaseImportListComponent<V extends BaseViewModel> extends B
     public initTable(): void {
         this.dataSource = new MatTableDataSource();
         this.setFilter();
-        this.importer.getNewEntries().subscribe(newEntries => {
+        this.importer
+            .getNewEntries()
+            .pipe(auditTime(100))
+            .subscribe(newEntries => {
+                this.dataSource.data = [];
                 this.dataSource.data = newEntries;
                 this.hasFile = newEntries.length > 0;
             });

client/src/app/site/users/services/user-import.service.ts

@@ -47,7 +47,8 @@ export class UserImportService extends BaseImportService<ViewUser> {
         Duplicates: 'This user already exists',
         NoName: 'Entry has no valid name',
         DuplicateImport: 'Entry cannot be imported twice. This line will be ommitted',
-        ParsingErrors: 'Some csv values could not be read correctly.'
+        ParsingErrors: 'Some csv values could not be read correctly.',
+        FailedImport: 'Imported user could not be imported.'
     };
 
     /**
@@ -134,6 +135,8 @@ export class UserImportService extends BaseImportService<ViewUser> {
      */
     public async doImport(): Promise<void> {
         this.newGroups = await this.createNewGroups();
+        const importUsers: NewEntry<ViewUser>[] = [];
+        let trackId = 1;
         for (const entry of this.entries) {
             if (entry.status !== 'new') {
                 continue;
@@ -144,11 +147,24 @@ export class UserImportService extends BaseImportService<ViewUser> {
                 this.updatePreview();
                 continue;
             }
-            await this.repo.create(entry.newEntry.user);
+            entry.importTrackId = trackId;
-            entry.status = 'done';
+            trackId += 1;
+            importUsers.push(entry);
         }
+        while (importUsers.length) {
+            const subSet = importUsers.splice(0, 100); // don't send bulks too large
+            const importedTracks = await this.repo.bulkCreate(subSet);
+            subSet.map(entry => {
+                const importModel = this.entries.find(e => e.importTrackId === entry.importTrackId);
+                if (importModel && importedTracks.includes(importModel.importTrackId)) {
+                    importModel.status = 'done';
+                } else {
+                    this.setError(importModel, 'FailedImport');
+                }
+            });
             this.updatePreview();
         }
+    }
 
     /**
      * extracts the group(s) from a csv column and tries to match them against existing groups,