OpenSlides/client/src/app/site/users/services/user-import.service.ts

302 lines
11 KiB
TypeScript
Raw Normal View History

2019-01-11 18:55:09 +01:00
import { Injectable } from '@angular/core';
import { MatSnackBar } from '@angular/material/snack-bar';
import { TranslateService } from '@ngx-translate/core';
2019-01-11 18:55:09 +01:00
import { Papa } from 'ngx-papaparse';
import { GroupRepositoryService } from 'app/core/repositories/users/group-repository.service';
import { UserRepositoryService } from 'app/core/repositories/users/user-repository.service';
import { BaseImportService, NewEntry } from 'app/core/ui-services/base-import.service';
2019-01-11 18:55:09 +01:00
import { Group } from 'app/shared/models/users/group';
import { User } from 'app/shared/models/users/user';
2019-11-01 15:19:05 +01:00
import { CsvMapping, ImportCreateUser } from '../models/import-create-user';
2019-01-11 18:55:09 +01:00
@Injectable({
providedIn: 'root'
})
2019-11-01 15:19:05 +01:00
export class UserImportService extends BaseImportService<User> {
2019-01-11 18:55:09 +01:00
/**
* Helper for mapping the expected header in a typesafe way. Values and order
* will be passed to {@link expectedHeader}
*/
2019-11-01 15:19:05 +01:00
public headerMap: (keyof ImportCreateUser)[] = [
2019-01-11 18:55:09 +01:00
'title',
'first_name',
'last_name',
'structure_level',
'number',
2019-01-11 18:55:09 +01:00
'groups_id',
'comment',
'is_active',
'is_present',
'is_committee',
'default_password',
2019-11-01 15:19:05 +01:00
'email',
'username',
'gender',
'vote_weight'
2019-01-11 18:55:09 +01:00
];
/**
* The minimimal number of header entries needed to successfully create an entry
*/
public requiredHeaderLength = 3;
/**
* List of possible errors and their verbose explanation
*/
public errorList = {
Group: 'Group cannot be resolved',
Duplicates: 'This user already exists',
NoName: 'Entry has no valid name',
DuplicateImport: 'Entry cannot be imported twice. This line will be ommitted',
2019-02-25 13:29:20 +01:00
ParsingErrors: 'Some csv values could not be read correctly.',
2020-04-30 08:10:39 +02:00
FailedImport: 'Imported user could not be imported.',
vote_weight: 'The vote weight has too many decimal places (max.: 6).'
2019-01-11 18:55:09 +01:00
};
/**
* Storage for tracking new groups to be created prior to importing users
*/
public newGroups: CsvMapping[];
/**
* Constructor. Calls parent and sets the expected header
*
* @param repo The User repository
* @param groupRepo the Group repository
* @param translate TranslationService
* @param papa csvParser
* @param matSnackbar MatSnackBar for displaying error messages
*/
public constructor(
private repo: UserRepositoryService,
private groupRepo: GroupRepositoryService,
translate: TranslateService,
papa: Papa,
matSnackbar: MatSnackBar
) {
super(translate, papa, matSnackbar);
this.expectedHeader = this.headerMap;
}
/**
* Clears all temporary data specific to this importer
*/
public clearData(): void {
this.newGroups = [];
}
/**
* Parses a string representing an entry, extracting secondary data, appending
* the array of secondary imports as needed
*
* @param line
* @returns a new entry representing an User
*/
2019-11-01 15:19:05 +01:00
public mapData(line: string): NewEntry<User> {
2020-04-30 08:10:39 +02:00
const user = new ImportCreateUser();
2019-01-11 18:55:09 +01:00
const headerLength = Math.min(this.expectedHeader.length, line.length);
let hasErrors = false;
for (let idx = 0; idx < headerLength; idx++) {
switch (this.expectedHeader[idx]) {
case 'groups_id':
2020-04-30 08:10:39 +02:00
user.csvGroups = this.getGroups(line[idx]);
2019-01-11 18:55:09 +01:00
break;
case 'is_active':
case 'is_committee':
case 'is_present':
try {
2020-04-30 08:10:39 +02:00
user[this.expectedHeader[idx]] = this.toBoolean(line[idx]);
2019-01-11 18:55:09 +01:00
} catch (e) {
if (e instanceof TypeError) {
console.log(e);
hasErrors = true;
continue;
}
}
break;
case 'number':
2020-04-30 08:10:39 +02:00
user.number = line[idx];
2019-01-11 18:55:09 +01:00
break;
case 'vote_weight':
if (!line[idx]) {
2020-04-30 08:10:39 +02:00
user[this.expectedHeader[idx]] = 1;
} else {
2020-04-30 08:10:39 +02:00
user[this.expectedHeader[idx]] = line[idx];
}
break;
2019-01-11 18:55:09 +01:00
default:
2020-04-30 08:10:39 +02:00
user[this.expectedHeader[idx]] = line[idx];
2019-01-11 18:55:09 +01:00
break;
}
}
2020-04-30 08:10:39 +02:00
const newEntry = this.userToEntry(user);
2019-01-11 18:55:09 +01:00
if (hasErrors) {
this.setError(newEntry, 'ParsingErrors');
}
return newEntry;
}
/**
* Executing the import. Creates all secondary data, maps the newly created
* secondary data to the new entries, then creates all entries without errors
* by submitting them to the server. The entries will receive the status
* 'done' on success.
*/
public async doImport(): Promise<void> {
this.newGroups = await this.createNewGroups();
2019-11-01 15:19:05 +01:00
const importUsers: NewEntry<User>[] = [];
2019-02-25 13:29:20 +01:00
let trackId = 1;
2019-01-11 18:55:09 +01:00
for (const entry of this.entries) {
if (entry.status !== 'new') {
continue;
}
2020-04-30 08:10:39 +02:00
const openGroups = (entry.newEntry as ImportCreateUser).solveGroups(this.newGroups);
if (openGroups) {
2019-01-11 18:55:09 +01:00
this.setError(entry, 'Group');
this.updatePreview();
continue;
}
2019-02-25 13:29:20 +01:00
entry.importTrackId = trackId;
trackId += 1;
importUsers.push(entry);
}
while (importUsers.length) {
const subSet = importUsers.splice(0, 100); // don't send bulks too large
2020-04-30 08:10:39 +02:00
const result = await this.repo.bulkCreate(subSet);
subSet.forEach(importUser => {
// const importModel = this.entries.find(e => e.importTrackId === importUser.importTrackId);
if (importUser && result.importedTrackIds.includes(importUser.importTrackId)) {
importUser.status = 'done';
} else if (result.errors[importUser.importTrackId]) {
this.setError(importUser, result.errors[importUser.importTrackId]);
2019-02-25 13:29:20 +01:00
} else {
2020-04-30 08:10:39 +02:00
this.setError(importUser, 'FailedImport');
2019-02-25 13:29:20 +01:00
}
});
this.updatePreview();
2019-01-11 18:55:09 +01:00
}
}
/**
* extracts the group(s) from a csv column and tries to match them against existing groups,
* appending to {@link newGroups} if needed.
* Also checks for groups matching the translation between english and the language currently set
*
* @param groupString string from an entry line including one or more comma separated groups
* @returns a mapping with (if existing) ids to the group names
*/
private getGroups(groupString: string): CsvMapping[] {
const result: CsvMapping[] = [];
if (!groupString) {
return [];
}
groupString.trim();
const groupArray = groupString.split(',');
for (const item of groupArray) {
const newGroup = item.trim();
let existingGroup = this.groupRepo.getViewModelList().filter(grp => grp.name === newGroup);
if (!existingGroup.length) {
existingGroup = this.groupRepo
.getViewModelList()
.filter(grp => this.translate.instant(grp.name) === newGroup);
}
if (!existingGroup.length) {
if (!this.newGroups.find(listedGrp => listedGrp.name === newGroup)) {
this.newGroups.push({ name: newGroup });
}
result.push({ name: newGroup });
} else if (existingGroup.length === 1) {
result.push({
name: existingGroup[0].name,
id: existingGroup[0].id
});
}
}
return result;
}
/**
* Handles the creation of new groups collected in {@link newGroups}.
*
* @returns The group mapping with (on success) new ids
*/
private async createNewGroups(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
for (const group of this.newGroups) {
promises.push(
this.groupRepo.create(new Group({ name: group.name })).then(identifiable => {
return { name: group.name, id: identifiable.id };
})
);
}
return await Promise.all(promises);
}
/**
* translates a string into a boolean
*
* @param data
* @returns a boolean from the string
*/
private toBoolean(data: string): Boolean {
if (!data || data === '0' || data === 'false') {
return false;
} else if (data === '1' || data === 'true') {
return true;
} else {
throw new TypeError('Value cannot be translated into boolean: ' + data);
}
}
/**
* parses the data given by the textArea. Expects user names separated by lines.
* Comma separated values will be read as Surname(s), given name(s) (lastCommaFirst)
*
* @param data a string as produced by textArea input
*/
public parseTextArea(data: string): void {
2019-11-01 15:19:05 +01:00
const newEntries: NewEntry<User>[] = [];
2019-01-11 18:55:09 +01:00
this.clearData();
this.clearPreview();
const lines = data.split('\n');
lines.forEach(line => {
if (!line.length) {
return;
}
const nameSchema = line.includes(',') ? 'lastCommaFirst' : 'firstSpaceLast';
2019-11-01 15:19:05 +01:00
const newUser = new ImportCreateUser(this.repo.parseUserString(line, nameSchema));
2019-01-11 18:55:09 +01:00
const newEntry = this.userToEntry(newUser);
newEntries.push(newEntry);
});
this.setParsedEntries(newEntries);
}
/**
* Checks a newly created ViewCsvCreateuser for validity and duplicates,
*
2020-04-30 08:10:39 +02:00
* @param user
2019-01-11 18:55:09 +01:00
* @returns a NewEntry with duplicate/error information
*/
2020-04-30 08:10:39 +02:00
private userToEntry(user: ImportCreateUser): NewEntry<User> {
2019-11-01 15:19:05 +01:00
const newEntry: NewEntry<User> = {
2020-04-30 08:10:39 +02:00
newEntry: user,
2019-07-17 16:13:49 +02:00
hasDuplicates: false,
2019-01-11 18:55:09 +01:00
status: 'new',
errors: []
};
2020-04-30 08:10:39 +02:00
if (user.isValid) {
2019-11-01 15:19:05 +01:00
newEntry.hasDuplicates = this.repo
.getViewModelList()
2020-04-30 08:10:39 +02:00
.some(existingUser => existingUser.full_name === this.repo.getFullName(user));
2019-07-17 16:13:49 +02:00
if (newEntry.hasDuplicates) {
2019-01-11 18:55:09 +01:00
this.setError(newEntry, 'Duplicates');
}
} else {
this.setError(newEntry, 'NoName');
}
return newEntry;
}
}