ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #4114 from MaximilianKrambach/csvimport/all

Browse Source 
csv-import for user and agenda(topics)
This commit is contained in:
Sean 2019-01-18 14:20:39 +01:00 committed by GitHub
commit eec3fba34a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 2602 additions and 819 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

395
client/src/app/core/services/base-import.service.ts Normal file
View File

@ -0,0 +1,395 @@
import { BehaviorSubject, Observable } from 'rxjs';
import { Injectable, EventEmitter } from '@angular/core';
import { MatSnackBar } from '@angular/material';
import { Papa, PapaParseConfig } from 'ngx-papaparse';
import { TranslateService } from '@ngx-translate/core';
import { BaseViewModel } from 'app/site/base/base-view-model';
/**
* Interface for value- Label combinations.
* Map objects didn't work, TODO: Use map objects (needs iterating through all objects of a map)
*/
export interface ValueLabelCombination {
value: string;
label: string;
}
/**
* Interface matching a newly created entry with their duplicates and an import status
*/
export interface NewEntry<V> {
newEntry: V;
status: CsvImportStatus;
errors: string[];
duplicates: V[];
}
/**
* interface for a preview summary
*/
export interface ImportCSVPreview {
total: number;
duplicates: number;
errors: number;
new: number;
done: number;
}
/**
* The permitted states of a new entry. Only a 'new' entry should be imported
* and then be set to 'done'.
*/
type CsvImportStatus = 'new' | 'error' | 'done';
/**
* Abstract service for imports
*/
@Injectable({
providedIn: 'root'
})
export abstract class BaseImportService<V extends BaseViewModel> {
/**
* List of possible errors and their verbose explanation
*/
public abstract errorList: Object;
/**
* The headers expected in the CSV matching import properties (in order)
*/
public expectedHeader: (string)[];
/**
* The minimimal number of header entries needed to successfully create an entry
*/
public abstract requiredHeaderLength: number;
/**
* The last parsed file object (may be reparsed with new encoding, thus kept in memory)
*/
private _rawFile: File;
/**
* The used column separator. If left on an empty string (default),
* the papaparse parser will automatically decide on separators.
*/
public columnSeparator = '';
/**
* The used text separator.
*/
public textSeparator = '"';
/**
* The encoding used by the FileReader object.
*/
public encoding = 'utf-8';
/**
* List of possible encodings and their label. values should be values accepted
* by the FileReader API
*/
public encodings: ValueLabelCombination[] = [
{ value: 'utf-8', label: 'UTF 8 - Unicode' },
{ value: 'iso-8859-1', label: 'ISO 8859-1 - West European' },
{ value: 'iso-8859-15', label: 'ISO 8859-15 - West European (with €)' }
];
/**
* List of possible column separators to pass on to papaParse
*/
public columnSeparators: ValueLabelCombination[] = [
{ label: 'Comma', value: ',' },
{ label: 'Semicolon', value: ';' },
{ label: 'Automatic', value: '' }
];
/**
* List of possible text separators to pass on to papaParse. Note that
* it cannot automatically detect textseparators (value must not be an empty string)
*/
public textSeparators: ValueLabelCombination[] = [
{ label: 'Double quotes (")', value: '"' },
{ label: "Single quotes (')", value: "'" }
];
/**
* FileReader object for file import
*/
private reader = new FileReader();
/**
* the list of parsed models that have been extracted from the opened file
*/
private _entries: NewEntry<V>[] = [];
/**
* BehaviorSubject for displaying a preview for the currently selected entries
*/
public newEntries = new BehaviorSubject<NewEntry<V>[]>([]);
/**
* Emits an error string to display if a file import cannot be done
*/
public errorEvent = new EventEmitter<string>();
/**
* storing the summary preview for the import, to avoid recalculating it
* at each display change.
*/
protected _preview: ImportCSVPreview;
/**
* Returns a summary on actions that will be taken/not taken.
*/
public get summary(): ImportCSVPreview {
if (!this._preview) {
this.updatePreview();
}
return this._preview;
}
/**
* Returns the current entries. For internal use in extending classes, as it
* might not be filled with data at all times (see {@link newEntries} for a BehaviorSubject)
*/
protected get entries(): NewEntry<V>[] {
return this._entries;
}
/**
* Constructor. Creates a fileReader to subscribe to it for incoming parsed
* strings
*
* @param translate Translation service
* @param papa External csv parser (ngx-papaparser)
* @param matSnackBar snackBar to display import errors
*/
public constructor(protected translate: TranslateService, private papa: Papa, protected matSnackbar: MatSnackBar) {
this.reader.onload = (event: any) => {
// TODO type: event is a progressEvent,
// but has a property target.result, which typescript doesn't recognize
this.parseInput(event.target.result);
};
}
/**
* Clears all stored secondary data
* TODO: Merge with clearPreview()
*/
public abstract clearData(): void;
/**
* Parses the data input. Expects a string as returned by via a
* File.readAsText() operation
*
* @param file
*/
public parseInput(file: string): void {
this.clearData();
this.clearPreview();
const papaConfig: PapaParseConfig = {
header: false,
skipEmptyLines: true,
quoteChar: this.textSeparator
};
if (this.columnSeparator) {
papaConfig.delimiter = this.columnSeparator;
}
const entryLines = this.papa.parse(file, papaConfig).data;
const valid = this.checkHeader(entryLines.shift());
if (!valid) {
return;
}
entryLines.forEach(line => {
const item = this.mapData(line);
if (item) {
this._entries.push(item);
}
});
this.newEntries.next(this._entries);
this.updatePreview();
}
/**
* parses pre-prepared entries (e.g. from a textarea) instead of a csv structure
*
* @param entries: an array of prepared newEntry objects
*/
public setParsedEntries(entries: NewEntry<V>[]): void {
this.clearData();
this.clearPreview();
if (!entries) {
return;
}
this._entries = entries;
this.newEntries.next(this._entries);
this.updatePreview();
}
/**
* Parsing an string representing an entry, extracting secondary data,
* returning a new entry object
* @param line a line extracted by the CSV (not including the header)
*/
public abstract mapData(line: string): NewEntry<V>;
/**
* Trigger for executing the import.
*/
public abstract async doImport(): Promise<void>;
/**
* counts the amount of duplicates that have no decision on the action to
* be taken
*/
public updatePreview(): void {
const summary = {
total: 0,
new: 0,
duplicates: 0,
errors: 0,
done: 0
};
this._entries.forEach(entry => {
summary.total += 1;
if (entry.status === 'done') {
summary.done += 1;
return;
} else if (entry.status === 'error' && !entry.duplicates.length) {
// errors that are not due to duplicates
summary.errors += 1;
return;
} else if (entry.duplicates.length) {
summary.duplicates += 1;
return;
} else if (entry.status === 'new') {
summary.new += 1;
}
});
this._preview = summary;
}
/**
* a subscribable representation of the new items to be imported
*
* @returns an observable BehaviorSubject
*/
public getNewEntries(): Observable<NewEntry<V>[]> {
return this.newEntries.asObservable();
}
/**
* Handler after a file was selected. Basic checking for type, then hand
* over to parsing
*
* @param event type is Event, but has target.files, which typescript doesn't seem to recognize
*/
public onSelectFile(event: any): void {
// TODO type
if (event.target.files && event.target.files.length === 1) {
if (event.target.files[0].type === 'text/csv') {
this._rawFile = event.target.files[0];
this.readFile(event.target.files[0]);
} else {
this.matSnackbar.open(this.translate.instant('Wrong file type detected. Import failed.'), '', {
duration: 3000
});
this.clearPreview();
this._rawFile = null;
}
}
}
/**
* Rereads the (previously selected) file, if present. Thought to be triggered
* by parameter changes on encoding, column, text separators
*/
public refreshFile(): void {
if (this._rawFile) {
this.readFile(this._rawFile);
}
}
/**
* (re)-reads a given file with the current parameter
*/
private readFile(file: File): void {
this.reader.readAsText(file, this.encoding);
}
/**
* Checks the first line of the csv (the header) for consistency (length)
*
* @param row expected to be an array parsed from the first line of a csv file
* @returns true if the line has at least the minimum amount of columns
*/
private checkHeader(row: string[]): boolean {
const snackbarDuration = 3000;
if (row.length < this.requiredHeaderLength) {
this.matSnackbar.open(this.translate.instant('The file has too few columns to be parsed properly.'), '', {
duration: snackbarDuration
});
this.clearPreview();
return false;
} else if (row.length < this.expectedHeader.length) {
this.matSnackbar.open(
this.translate.instant('The file seems to have some ommitted columns. They will be considered empty.'),
'',
{ duration: snackbarDuration }
);
} else if (row.length > this.expectedHeader.length) {
this.matSnackbar.open(
this.translate.instant('The file seems to have additional columns. They will be ignored.'),
'',
{ duration: snackbarDuration }
);
}
return true;
}
/**
* Resets the data and preview (triggered upon selecting an invalid file)
*/
public clearPreview(): void {
this._entries = [];
this.newEntries.next([]);
this._preview = null;
}
/**
* set a list of short names for error, indicating which column failed
*/
public setError(entry: NewEntry<V>, error: string): void {
if (this.errorList.hasOwnProperty(error)) {
if (!entry.errors) {
entry.errors = [error];
} else if (!entry.errors.includes(error)) {
entry.errors.push(error);
entry.status = 'error';
}
}
}
/**
* Get an extended error description.
*
* @param error
* @returns the extended error desription for that error
*/
public verbose(error: string): string {
return this.errorList[error];
}
/**
* Queries if a given error is present in the given entry
*
* @param entry the entry to check for the error.
* @param error The error to check for
* @returns true if the error is present
*/
public hasError(entry: NewEntry<V>, error: string): boolean {
return entry.errors.includes(error);
}
}

0
client/src/app/site/core/services/duration.service.spec.ts → client/src/app/core/services/duration.service.spec.ts
View File

0
client/src/app/site/core/services/duration.service.ts → client/src/app/core/services/duration.service.ts
View File

25
client/src/app/shared/models/users/user.ts
View File

@ -33,12 +33,12 @@ export class User extends ProjectableBaseModel implements Searchable {
const addition: string[] = [];
// addition: add number and structure level
const structure_level = this.structure_level.trim();
const structure_level = this.structure_level ? this.structure_level.trim() : '';
if (structure_level) {
addition.push(structure_level);
}
const number = this.number.trim();
const number = this.number ? this.number.trim() : null;
if (number) {
// TODO Translate
addition.push('No.' + ' ' + number);
@ -54,12 +54,17 @@ export class User extends ProjectableBaseModel implements Searchable {
return this.groups_id.some(groupId => groupId === id);
}
// TODO read config values for "users_sort_by"
// TODO read config values for "users_sort_by"
/**
* Getter for the short name (Title, given name, surname)
*
* @returns a non-empty string
*/
public get short_name(): string {
const title = this.title.trim();
const firstName = this.first_name.trim();
const lastName = this.last_name.trim();
let shortName = '';
const title = this.title ? this.title.trim() : '';
const firstName = this.first_name ? this.first_name.trim() : '';
const lastName = this.last_name ? this.last_name.trim() : '';
// TODO need DS adjustment first first
// if (this.DS.getConfig('users_sort_by').value === 'last_name') {
@ -70,9 +75,9 @@ export class User extends ProjectableBaseModel implements Searchable {
// }
// }
shortName += `${firstName} ${lastName}`;
let shortName = `${firstName} ${lastName}`;
if (shortName.trim() === '') {
if (!shortName) {
shortName = this.username;
}
@ -80,7 +85,7 @@ export class User extends ProjectableBaseModel implements Searchable {
shortName = `${title} ${shortName}`;
}
return shortName.trim();
return shortName;
}
public getTitle(): string {

3
client/src/app/shared/shared.module.ts
View File

@ -25,6 +25,7 @@ import {
MatButtonToggleModule,
MatBadgeModule,
MatStepperModule,
MatTabsModule,
MatBottomSheetModule
} from '@angular/material';
import { MatAutocompleteModule } from '@angular/material/autocomplete';
@ -118,6 +119,7 @@ import { LogoComponent } from './components/logo/logo.component';
MatRadioModule,
MatButtonToggleModule,
MatStepperModule,
MatTabsModule,
DragDropModule,
TranslateModule.forChild(),
RouterModule,
@ -151,6 +153,7 @@ import { LogoComponent } from './components/logo/logo.component';
MatSnackBarModule,
MatChipsModule,
MatTooltipModule,
MatTabsModule,
MatBadgeModule,
MatIconModule,
MatRadioModule,

17
client/src/app/site/agenda/agenda-import.service.spec.ts Normal file
View File

@ -0,0 +1,17 @@
import { TestBed } from '@angular/core/testing';
import { AgendaImportService } from './agenda-import.service';
import { E2EImportsModule } from 'e2e-imports.module';
describe('AgendaImportService', () => {
beforeEach(() =>
TestBed.configureTestingModule({
imports: [E2EImportsModule]
})
);
it('should be created', () => {
const service: AgendaImportService = TestBed.get(AgendaImportService);
expect(service).toBeTruthy();
});
});

216
client/src/app/site/agenda/agenda-import.service.ts Normal file
View File

@ -0,0 +1,216 @@
import { Injectable } from '@angular/core';
import { MatSnackBar } from '@angular/material';
import { Papa } from 'ngx-papaparse';
import { TranslateService } from '@ngx-translate/core';
import { BaseImportService, NewEntry } from 'app/core/services/base-import.service';
import { CreateTopic } from './models/create-topic';
import { DurationService } from 'app/core/services/duration.service';
import { itemVisibilityChoices } from 'app/shared/models/agenda/item';
import { TopicRepositoryService } from './services/topic-repository.service';
import { ViewCreateTopic } from './models/view-create-topic';
@Injectable({
providedIn: 'root'
})
export class AgendaImportService extends BaseImportService<ViewCreateTopic> {
/**
* Helper for mapping the expected header in a typesafe way. Values will be passed to
* {@link expectedHeader}
*/
public headerMap: (keyof ViewCreateTopic)[] = ['title', 'text', 'duration', 'comment', 'type'];
/**
* The minimimal number of header entries needed to successfully create an entry
*/
public requiredHeaderLength = 1;
/**
* List of possible errors and their verbose explanation
*/
public errorList = {
NoTitle: 'A Topic needs a title',
Duplicates: 'A topic tiwh this title already exists',
ParsingErrors: 'Some csv values could not be read correctly.'
};
/**
* Constructor. Calls the abstract class and sets the expected header
*
* @param durationService: a service for converting time strings and numbers
* @param repo: The Agenda repository service
* @param translate A translation service for translating strings
* @param papa Csv parser
* @param matSnackBar MatSnackBar for displaying errors
*/
public constructor(
private durationService: DurationService,
private repo: TopicRepositoryService,
translate: TranslateService,
papa: Papa,
matSnackBar: MatSnackBar
) {
super(translate, papa, matSnackBar);
this.expectedHeader = this.headerMap;
}
/**
* Clear all secondary import data. As agenda items have no secondary imports,
* this is an empty function
*/
public clearData(): void {}
/**
* Parses a string representing an entry
*
* @param line a line extracted by the CSV (without the header)
* @returns a new entry for a Topic
*/
public mapData(line: string): NewEntry<ViewCreateTopic> {
const newEntry = new ViewCreateTopic(new CreateTopic());
const headerLength = Math.min(this.expectedHeader.length, line.length);
let hasErrors = false;
for (let idx = 0; idx < headerLength; idx++) {
switch (this.expectedHeader[idx]) {
case 'duration':
try {
const duration = this.parseDuration(line[idx]);
if (duration > 0) {
newEntry.duration = duration;
}
} catch (e) {
if (e instanceof TypeError) {
hasErrors = true;
continue;
}
}
break;
case 'type':
try {
newEntry.type = this.parseType(line[idx]);
} catch (e) {
if (e instanceof TypeError) {
hasErrors = true;
continue;
}
}
break;
default:
newEntry[this.expectedHeader[idx]] = line[idx];
}
}
const updateModels = this.repo.getTopicDuplicates(newEntry) as ViewCreateTopic[];
// set type to 'public' if none is given in import
if (!newEntry.type) {
newEntry.type = 1;
}
const mappedEntry: NewEntry<ViewCreateTopic> = {
newEntry: newEntry,
duplicates: [],
status: 'new',
errors: []
};
if (updateModels.length) {
mappedEntry.duplicates = updateModels;
this.setError(mappedEntry, 'Duplicates');
}
if (hasErrors) {
this.setError(mappedEntry, 'ParsingErrors');
}
if (!mappedEntry.newEntry.isValid) {
this.setError(mappedEntry, 'NoTitle');
}
return mappedEntry;
}
/**
* Executing the import. Parses all entries without errors and submits them
* to the server. The entries will receive the status 'done' on success.
*/
public async doImport(): Promise<void> {
for (const entry of this.entries) {
if (entry.status !== 'new') {
continue;
}
await this.repo.create(entry.newEntry.topic);
entry.status = 'done';
}
this.updatePreview();
}
/**
* Matching the duration string/number to the time model in use
*
* @param input
* @returns duration as defined in durationService
*/
public parseDuration(input: string): number {
return this.durationService.stringToDuration(input);
}
/**
* Converts information from 'item type' to a model-based type number.
* Accepts either old syntax (numbers) or new visibility choice csv names;
* both defined in {@link itemVisibilityChoices}
* Empty values will be interpreted as default 'public' agenda topics
*
* @param input
* @returns a number as defined for the itemVisibilityChoices
*/
public parseType(input: string | number): number {
if (!input) {
return 1; // default, public item
} else if (typeof input === 'string') {
const visibility = itemVisibilityChoices.find(choice => choice.csvName === input);
if (visibility) {
return visibility.key;
}
} else if (input === 1) {
// Compatibility with the old client's isInternal column
const visibility = itemVisibilityChoices.find(choice => choice.name === 'Internal item');
if (visibility) {
return visibility.key;
}
} else {
throw new TypeError('type could not be matched');
}
}
/**
* parses the data given by the textArea. Expects an agenda title per line
*
* @param data a string as produced by textArea input
*/
public parseTextArea(data: string): void {
const newEntries: NewEntry<ViewCreateTopic>[] = [];
this.clearData();
this.clearPreview();
const lines = data.split('\n');
lines.forEach(line => {
if (!line.length) {
return;
}
const newTopic = new ViewCreateTopic(
new CreateTopic({
title: line,
agenda_type: 1 // set type to 'public item' by default
})
);
const newEntry: NewEntry<ViewCreateTopic> = {
newEntry: newTopic,
duplicates: [],
status: 'new',
errors: []
};
const duplicates = this.repo.getTopicDuplicates(newTopic);
if (duplicates.length) {
// TODO this is a dishonest casting. duplicates should not be required to be View
newEntry.duplicates = duplicates as ViewCreateTopic[];
this.setError(newEntry, 'Duplicates');
}
newEntries.push(newEntry);
});
this.setParsedEntries(newEntries);
}
}

3
client/src/app/site/agenda/agenda-routing.module.ts
View File

@ -1,11 +1,14 @@
import { NgModule } from '@angular/core';
import { Routes, RouterModule } from '@angular/router';
import { AgendaImportListComponent } from './components/agenda-import-list/agenda-import-list.component';
import { AgendaListComponent } from './components/agenda-list/agenda-list.component';
import { TopicDetailComponent } from './components/topic-detail/topic-detail.component';
import { SpeakerListComponent } from './components/speaker-list/speaker-list.component';
const routes: Routes = [
{ path: '', component: AgendaListComponent },
{ path: 'import', component: AgendaImportListComponent },
{ path: 'topics/new', component: TopicDetailComponent },
{ path: 'topics/:id', component: TopicDetailComponent },
{ path: ':id/speakers', component: SpeakerListComponent }

7
client/src/app/site/agenda/agenda.module.ts
View File

@ -1,11 +1,12 @@
import { NgModule } from '@angular/core';
import { CommonModule } from '@angular/common';
import { AgendaImportListComponent } from './components/agenda-import-list/agenda-import-list.component';
import { AgendaListComponent } from './components/agenda-list/agenda-list.component';
import { ItemInfoDialogComponent } from './components/item-info-dialog/item-info-dialog.component';
import { AgendaRoutingModule } from './agenda-routing.module';
import { SharedModule } from '../../shared/shared.module';
import { AgendaListComponent } from './components/agenda-list/agenda-list.component';
import { TopicDetailComponent } from './components/topic-detail/topic-detail.component';
import { ItemInfoDialogComponent } from './components/item-info-dialog/item-info-dialog.component';
/**
* AppModule for the agenda and it's children.
@ -13,6 +14,6 @@ import { ItemInfoDialogComponent } from './components/item-info-dialog/item-info
@NgModule({
imports: [CommonModule, AgendaRoutingModule, SharedModule],
entryComponents: [ItemInfoDialogComponent],
declarations: [AgendaListComponent, TopicDetailComponent, ItemInfoDialogComponent]
declarations: [AgendaListComponent, TopicDetailComponent, ItemInfoDialogComponent, AgendaImportListComponent]
})
export class AgendaModule {}

226
client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.html Normal file
View File

@ -0,0 +1,226 @@
<os-head-bar [nav]="false">
<!-- Title -->
<div class="title-slot"><h2 translate>Import topics</h2></div>
<div class="menu-slot">
<button *ngIf="hasFile && newCount" mat-button (click)="doImport()">
<span class="upper" translate> Import</span>
</button>
</div>
</os-head-bar>
<mat-card class="os-form-card import-table">
<mat-tab-group (selectedTabChange)="onTabChange()">
<mat-tab label="{{ 'CSV import' | translate }}">
<span translate>Required comma or semicolon separated values with these column header names in the first row:</span>
<br />
<div class="code red-warning-text">
<span translate>Title</span>,&nbsp;
<span translate>Text</span>,&nbsp;
<span translate>Duration</span>,&nbsp;
<span translate>Comment</span>,&nbsp;
<span translate>Internal item</span>
</div>
<ul>
<li translate>Title is required. All other fields are optional and may be empty.
</li>
<li translate>Additional columns after the required ones may be present and won't affect the import.</li>
</ul>
<button mat-button color="accent" (click)="downloadCsvExample()" translate>Download CSV example file</button>
<div class="wrapper">
<mat-form-field>
<mat-label translate>Encoding of the file</mat-label>
<mat-select
class="selection"
placeholder="translate.instant('Select encoding')"
(selectionChange)="selectEncoding($event)"
[value]="encodings[0].value"
>
<mat-option *ngFor="let option of encodings" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field>
<mat-label translate> Column Separator</mat-label>
<mat-select class="selection" (selectionChange)="selectColSep($event)" value="">
<mat-option *ngFor="let option of columnSeparators" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field>
<mat-label translate>Text separator</mat-label>
<mat-select class="selection" (selectionChange)="selectTextSep($event)" value="&quot;">
<mat-option *ngFor="let option of textSeparators" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
</div>
<div>
<div>
<input
id="agenda-import-file-input"
type="file"
class="hidden-input"
accept="text"
#fileInput
(change)="onSelectFile($event)"
/>
<button mat-button onclick="document.getElementById('agenda-import-file-input').click()">
<span translate> Select file</span>
</button>
</div>
</div>
</mat-tab>
<!-- textAreaImport-->
<mat-tab label="{{ 'Text import' | translate }}">
<div [formGroup]="textAreaForm">
<div>
<span translate>
Paste/write your topics in this textbox.</span>
<span translate>
Keep each item in a single line.
</span>
</div>
<mat-form-field>
<textarea
matInput
formControlName="inputtext"
placeholder="{{ 'Insert topics here' | translate }}"
cdkTextareaAutosize
cdkAutosizeMinRows="3"
cdkAutosizeMaxRows="10"
></textarea>
</mat-form-field>
</div>
<div>
<button mat-button color="accent" (click)="parseTextArea()"><span translate>Preview</span></button>
</div>
</mat-tab>
</mat-tab-group>
</mat-card>
<!-- preview table -->
<mat-card *ngIf="hasFile" class="os-form-card import-table">
<h3 translate> Preview</h3>
<div class="summary">
<!-- new entries -->
<div *ngIf="newCount">
&nbsp;
<mat-icon inline>playlist_add</mat-icon>
<span>&nbsp;{{ newCount }}&nbsp;</span> <span translate>Topics(s) will be imported.</span>
</div>
<!-- errors/duplicates -->
<div *ngIf="nonImportableCount" class="red-warning-text">
&nbsp;
<mat-icon inline>warning</mat-icon>
<span>&nbsp;{{ nonImportableCount }}&nbsp;</span> <span translate>entries will be ommitted.</span>
</div>
<!-- have been imported -->
<div *ngIf="doneCount" class="green-text">
&nbsp;
<mat-icon inline>done</mat-icon>
<span>&nbsp;{{ doneCount }}&nbsp;</span> <span translate>Topics have been imported.</span>
</div>
</div>
<div *ngIf="newCount">
<span translate>Click on 'import' (right top corner) to import the new topics.
</span>
</div>
<div>
<mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()">
<mat-option value="all" translate> Show all </mat-option>
<mat-option value="error" translate> Show errors only </mat-option>
<mat-option value="noerror" translate> Show correct entries </mat-option>
</mat-select>
</div>
<div class="table-container">
<table mat-table class="on-transition-fade" [dataSource]="dataSource" matSort>
<!-- Status column -->
<ng-container matColumnDef="status" class="first-column" sticky>
<mat-header-cell *matHeaderCellDef></mat-header-cell>
<mat-cell *matCellDef="let entry" class="first-column">
<div *ngIf="entry.status === 'error'">
<mat-icon
class="red-warning-text"
matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
>
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.status === 'new'">
<mat-icon matTooltip="{{ 'Topic will be imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.status === 'done'">
<mat-icon matTooltip="{{ 'Topic has been imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
</mat-cell>
</ng-container>
<!-- title column -->
<ng-container matColumnDef="title">
<mat-header-cell *matHeaderCellDef translate>Title</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-icon
color="warn"
inline
*ngIf="hasError(entry, 'Duplicates')"
matTooltip="{{ getVerboseError('Duplicates') | translate }}"
>
warning
</mat-icon>
<mat-icon
color="warn"
inline
*ngIf="hasError(entry, 'NoTitle')"
matTooltip="{{ getVerboseError('NoTitle') | translate }}"
>
warning
</mat-icon>
{{ entry.newEntry.title }}
</mat-cell>
</ng-container>
<ng-container matColumnDef="text">
<mat-header-cell *matHeaderCellDef translate>Item text</mat-header-cell>
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newEntry.text) }}">
{{ getShortPreview(entry.newEntry.text) }}
</mat-cell>
</ng-container>
<!-- duration column -->
<ng-container matColumnDef="duration">
<mat-header-cell *matHeaderCellDef translate>Duration</mat-header-cell>
<mat-cell *matCellDef="let entry">
{{ getDuration(entry.newEntry.duration) }}
</mat-cell>
</ng-container>
<!-- comment column-->
<ng-container matColumnDef="comment">
<mat-header-cell *matHeaderCellDef translate>Comment</mat-header-cell>
<mat-cell *matCellDef="let entry">
{{ entry.newEntry.comment }}
</mat-cell>
</ng-container>
<!-- type column -->
<ng-container matColumnDef="type">
<mat-header-cell *matHeaderCellDef translate>Type</mat-header-cell>
<mat-cell *matCellDef="let entry">
{{ getTypeString(entry.newEntry.type) | translate }}
</mat-cell>
</ng-container>
<mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
<mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
</table>
</div>
</mat-card>

26
client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.spec.ts Normal file
View File

@ -0,0 +1,26 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { AgendaImportListComponent } from './agenda-import-list.component';
import { E2EImportsModule } from 'e2e-imports.module';
describe('AgendaImportListComponent', () => {
let component: AgendaImportListComponent;
let fixture: ComponentFixture<AgendaImportListComponent>;
beforeEach(async(() => {
TestBed.configureTestingModule({
declarations: [AgendaImportListComponent],
imports: [E2EImportsModule]
}).compileComponents();
}));
beforeEach(() => {
fixture = TestBed.createComponent(AgendaImportListComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

156
client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.ts Normal file
View File

@ -0,0 +1,156 @@
import { Component } from '@angular/core';
import { FormGroup, FormBuilder } from '@angular/forms';
import { MatSnackBar } from '@angular/material';
import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core';
import { AgendaImportService } from '../../agenda-import.service';
import { BaseImportListComponent } from 'app/site/base/base-import-list';
import { DurationService } from 'app/core/services/duration.service';
import { FileExportService } from 'app/core/services/file-export.service';
import { itemVisibilityChoices } from 'app/shared/models/agenda/item';
import { ViewCreateTopic } from '../../models/view-create-topic';
/**
* Component for the agenda import list view.
*/
@Component({
selector: 'os-agenda-import-list',
templateUrl: './agenda-import-list.component.html'
})
export class AgendaImportListComponent extends BaseImportListComponent<ViewCreateTopic> {
/**
* A form for text input
*/
public textAreaForm: FormGroup;
/**
* Constructor for list view bases
*
* @param titleService the title serivce
* @param matSnackBar snackbar for displaying errors
* @param translate the translate service
* @param importer: The agenda csv import service
* @param formBuilder: FormBuilder for the textarea
* @param exporter: ExportService for example download
* @param durationService: Service converting numbers to time strings
*/
public constructor(
titleService: Title,
matSnackBar: MatSnackBar,
translate: TranslateService,
importer: AgendaImportService,
formBuilder: FormBuilder,
private exporter: FileExportService,
private durationService: DurationService
) {
super(importer, titleService, translate, matSnackBar);
this.textAreaForm = formBuilder.group({ inputtext: [''] });
}
/**
* Get the first characters of a string, for preview purposes
*
* @param input any string
* @returns a string with at most 50 characters
*/
public getShortPreview(input: string): string {
if (!input) {
return '';
}
if (input.length > 50) {
return this.stripHtmlTags(input.substring(0, 47)) + '...';
}
return this.stripHtmlTags(input);
}
/**
* Fetch the first and last 150 characters of a string; used within
* tooltips for the preview
*
* @param input any string
* @returns a string with the first and last 150 characters of the input
* string
*/
public getLongPreview(input: string): string {
if (!input) {
return '';
}
if (input.length < 300) {
return this.stripHtmlTags(input);
}
return (
this.stripHtmlTags(input.substring(0, 147)) +
' [...] ' +
this.stripHtmlTags(input.substring(input.length - 150, input.length))
);
}
/**
* Helper to remove html tags from a string.
* CAUTION: It is just a basic "don't show distracting html tags in a
* preview", not an actual tested sanitizer!
*
* @param inputString
* @returns a string without hatml tags
*/
private stripHtmlTags(inputString: string): string {
const regexp = new RegExp(/<[^ ][^<>]*(>|$)/g);
return inputString.replace(regexp, '').trim();
}
/**
* Triggers an example csv download
*/
public downloadCsvExample(): void {
const headerRow = ['Title', 'Text', 'Duration', 'Comment', 'Internal item']
.map(item => this.translate.instant(item))
.join(',');
const rows = [
headerRow,
'Demo 1,Demo text 1,1:00,test comment,',
'Break,,0:10,,internal',
'Demo 2,Demo text 2,1:30,,hidden'
];
this.exporter.saveFile(rows.join('\n'), this.translate.instant('Topic example') + '.csv');
}
/**
* Fetches the string to a item_type
*
* @param type
* @returns A string, which may be empty if the type is not found in the visibilityChoices
*/
public getTypeString(type: number): string {
const visibility = itemVisibilityChoices.find(choice => choice.key === type);
return visibility ? visibility.name : '';
}
/**
* Sends the data in the text field input area to the importer
*/
public parseTextArea(): void {
(this.importer as AgendaImportService).parseTextArea(this.textAreaForm.get('inputtext').value);
}
/**
* Triggers a change in the tab group: Clearing the preview selection
*/
public onTabChange(): void {
this.importer.clearPreview();
}
/**
* Gets a readable string for a duration
*
* @param duration
* @returns a duration string, an empty string if the duration is not set or negative
*/
public getDuration(duration: number): string {
if (duration >= 0) {
return this.durationService.durationToString(duration);
} else {
return '';
}
}
}

4
client/src/app/site/agenda/components/agenda-list/agenda-list.component.html
View File

@ -107,6 +107,10 @@
<mat-icon>archive</mat-icon>
<span translate>Export as CSV</span>
</button>
<button mat-menu-item *osPerms="'agenda.can_manage'" routerLink="import">
<mat-icon>save_alt</mat-icon>
<span translate>Import</span><span>&nbsp;...</span>
</button>
</div>
<div *ngIf="isMultiSelect">

4
client/src/app/site/agenda/components/agenda-list/agenda-list.component.ts
View File

@ -11,10 +11,10 @@ import { PromptService } from '../../../../core/services/prompt.service';
import { ViewItem } from '../../models/view-item';
import { AgendaCsvExportService } from '../../services/agenda-csv-export.service';
import { ConfigService } from 'app/core/services/config.service';
import { DurationService } from 'app/core/services/duration.service';
import { ItemInfoDialogComponent } from '../item-info-dialog/item-info-dialog.component';
import { ViewportService } from 'app/core/services/viewport.service';
import { DurationService } from 'app/site/core/services/duration.service';
import { ConfigService } from 'app/core/services/config.service';
/**
* List view for the agenda.

2
client/src/app/site/agenda/components/item-info-dialog/item-info-dialog.component.ts
View File

@ -4,7 +4,7 @@ import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material';
import { ViewItem } from '../../models/view-item';
import { itemVisibilityChoices } from 'app/shared/models/agenda/item';
import { DurationService } from 'app/site/core/services/duration.service';
import { DurationService } from 'app/core/services/duration.service';
/**
* Dialog component to change agenda item details

3
client/src/app/site/agenda/components/topic-detail/topic-detail.component.ts
View File

@ -15,6 +15,7 @@ import { BehaviorSubject } from 'rxjs';
import { DataStoreService } from 'app/core/services/data-store.service';
import { Mediafile } from 'app/shared/models/mediafiles/mediafile';
import { Item, itemVisibilityChoices } from 'app/shared/models/agenda/item';
import { CreateTopic } from '../../models/create-topic';
/**
* Detail page for topics.
@ -125,7 +126,7 @@ export class TopicDetailComponent extends BaseViewComponent {
if (!this.topicForm.value.agenda_parent_id) {
delete this.topicForm.value.agenda_parent_id;
}
await this.repo.create(this.topicForm.value);
await this.repo.create(new CreateTopic(this.topicForm.value));
this.router.navigate([`/agenda/`]);
} else {
this.setEditMode(false);

17
client/src/app/site/agenda/models/create-topic.ts Normal file
View File

@ -0,0 +1,17 @@
import { Topic } from 'app/shared/models/topics/topic';
/**
* Representation of Topic during creation.
*/
export class CreateTopic extends Topic {
public attachments_id: number[];
public agenda_type: number;
public agenda_parent_id: number;
public agenda_comment: string;
public agenda_duration: number;
public agenda_weight: number;
public constructor(input?: any) {
super(input);
}
}

110
client/src/app/site/agenda/models/view-create-topic.ts Normal file
View File

@ -0,0 +1,110 @@
import { CreateTopic } from './create-topic';
import { ViewTopic } from './view-topic';
/**
* View model for Topic('Agenda item') creation.
*
*/
export class ViewCreateTopic extends ViewTopic {
public get topic(): CreateTopic {
return this._topic as CreateTopic;
}
/**
* Fetches the field representing the new title
*
* @returns title string as set during import (may be different from getTitle)
*/
public get title(): string {
return this.topic.title;
}
/**
* Setter for the title. Sets the title of the underlying CreateTopic
*
* @param title
*/
public set title(title: string) {
this.topic.title = title;
}
/**
* @returns the duration in minutes
*/
public get duration(): number {
return this.topic.agenda_duration;
}
/**
* Setter for the duration. Expects values as in {@link DurationService}
*/
public set duration(duration: number) {
this.topic.agenda_duration = duration;
}
/**
* @returns the comment string as set during the import
*/
public get comment(): string {
return this.topic.agenda_comment;
}
/**
* Sets the comment string of the underlying topic
* @param comment A string to set as comment
*/
public set comment(comment: string) {
this.topic.agenda_comment = comment;
}
/**
* @returns a number representing the item type
*/
public get type(): number {
return this.topic.agenda_type;
}
/**
* sets the item type for the topic's agenda entry. No validation is done here.
*
* @param A number representing the item's type. See {@link itemVisibilityChoices}
* for the interpretation of type numbers.
*/
public set type(type: number) {
this.topic.agenda_type = type;
}
/**
* Sets the text string of the underlying topic
*
* @param text A string.
*/
public set text(text: string) {
this.topic.text = text;
}
/**
* @returns the comment string of the underlying topic
*/
public get text(): string {
return this.topic.text;
}
/**
* Checks if the CreateTopic is valid. Currently only requires an existing title
*
* @returns true if it is a valid Topic
*/
public get isValid(): boolean {
return this.title ? true : false;
}
/**
* Constructor. Empty
*
* @param topic A CreateTopic
*/
public constructor(topic: CreateTopic) {
super(topic);
}
}

2
client/src/app/site/agenda/models/view-topic.ts
View File

@ -9,7 +9,7 @@ import { BaseModel } from 'app/shared/models/base/base-model';
* @ignore
*/
export class ViewTopic extends BaseViewModel {
private _topic: Topic;
protected _topic: Topic;
private _attachments: Mediafile[];
private _agenda_item: Item;

19
client/src/app/site/agenda/services/topic-repository.service.ts
View File

@ -9,6 +9,7 @@ import { DataSendService } from 'app/core/services/data-send.service';
import { ViewTopic } from '../models/view-topic';
import { Identifiable } from 'app/shared/models/base/identifiable';
import { CollectionStringModelMapperService } from 'app/core/services/collectionStringModelMapper.service';
import { CreateTopic } from '../models/create-topic';
/**
* Repository for topics
@ -61,10 +62,8 @@ export class TopicRepositoryService extends BaseRepository<ViewTopic, Topic> {
* @param topicData Partial topic data to be created
* @returns an Identifiable (usually id) as promise
*/
public async create(topicData: Partial<Topic>): Promise<Identifiable> {
const newTopic = new Topic();
newTopic.patchValues(topicData);
return await this.dataSend.createModel(newTopic);
public async create(topic: CreateTopic): Promise<Identifiable> {
return await this.dataSend.createModel(topic);
}
/**
@ -89,4 +88,16 @@ export class TopicRepositoryService extends BaseRepository<ViewTopic, Topic> {
public async delete(viewTopic: ViewTopic): Promise<void> {
return await this.dataSend.deleteModel(viewTopic.topic);
}
/**
* Returns an array of all duplicates for a topic
*
* @param topic
*/
public getTopicDuplicates(topic: ViewTopic): ViewTopic[] {
const duplicates = this.DS.filter(Topic, item => topic.title === item.title);
const viewTopics: ViewTopic[] = [];
duplicates.forEach(item => viewTopics.push(this.createViewModel(item)));
return viewTopics;
}
}

276
client/src/app/site/base/base-import-list.ts Normal file
View File

@ -0,0 +1,276 @@
import { MatTableDataSource, MatTable, MatSnackBar, MatSelectChange } from '@angular/material';
import { ViewChild, OnInit } from '@angular/core';
import { BaseViewComponent } from './base-view';
import { BaseViewModel } from './base-view-model';
import { NewEntry, ValueLabelCombination, BaseImportService } from 'app/core/services/base-import.service';
import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core';
export abstract class BaseImportListComponent<V extends BaseViewModel> extends BaseViewComponent implements OnInit {
/**
* The data source for a table. Requires to be initialised with a BaseViewModel
*/
public dataSource: MatTableDataSource<NewEntry<V>>;
/**
* Switch that turns true if a file has been selected in the input
*/
public hasFile = false;
/**
* Currently selected encoding. Is set and changed by the config's available
* encodings and user mat-select input
*/
public selectedEncoding = 'utf-8';
/**
* indicator on which elements to display
*/
public shown: 'all' | 'error' | 'noerror' = 'all';
/**
* The table itself
*/
@ViewChild(MatTable)
protected table: MatTable<NewEntry<V>>;
/**
* @returns the amount of total item successfully parsed
*/
public get totalCount(): number {
return this.importer && this.hasFile ? this.importer.summary.total : null;
}
/**
* @returns the encodings available and their labels
*/
public get encodings(): ValueLabelCombination[] {
return this.importer.encodings;
}
/**
* @returns the available column separators and their labels
*/
public get columnSeparators(): ValueLabelCombination[] {
return this.importer.columnSeparators;
}
/**
* @eturns the available text separators and their labels
*/
public get textSeparators(): ValueLabelCombination[] {
return this.importer.textSeparators;
}
/**
* @returns the amount of import items that will be imported
*/
public get newCount(): number {
return this.importer && this.hasFile ? this.importer.summary.new : 0;
}
/**
* @returns the number of import items that cannot be imported
*/
public get nonImportableCount(): number {
if (this.importer && this.hasFile) {
return this.importer.summary.errors + this.importer.summary.duplicates;
}
return 0;
}
/**
* @returns the number of import items that have been successfully imported
*/
public get doneCount(): number {
return this.importer && this.hasFile ? this.importer.summary.done : 0;
}
/**
* Constructor. Initializes the table and subscribes to import errors
*
* @param importer The import service, depending on the implementation
* @param titleService A title service
* @param translate TranslationService for translating strings
* @param matSnackBar MatSnackBar for displaying errors
*/
public constructor(
protected importer: BaseImportService<V>,
titleService: Title,
translate: TranslateService,
matSnackBar: MatSnackBar
) {
super(titleService, translate, matSnackBar);
this.initTable();
this.importer.errorEvent.subscribe(this.raiseError);
}
/**
* Starts with a clean preview (removing any previously existing import previews)
*/
public ngOnInit(): void {
this.importer.clearPreview();
}
/**
* Initializes the table
*/
public initTable(): void {
this.dataSource = new MatTableDataSource();
this.setFilter();
this.importer.getNewEntries().subscribe(newEntries => {
this.dataSource.data = newEntries;
this.hasFile = newEntries.length > 0;
});
}
/**
* Returns the table column definition. Fetches all headers from
* {@link MotionImportService} and an additional status column
*
* @returns An array of the columns forming the import header, and an additional 'status' bar on the front
*/
public getColumnDefinition(): string[] {
return ['status'].concat(this.importer.expectedHeader);
}
/**
* triggers the importer's onSelectFile after a file has been chosen
*/
public onSelectFile(event: any): void {
this.importer.onSelectFile(event);
}
/**
* Triggers the importer's import
*
*/
public async doImport(): Promise<void> {
await this.importer.doImport();
this.setFilter();
}
/**
* Updates and manually triggers the filter function.
* See {@link hidden} for options
* (changed from default mat-table filter)
*/
public setFilter(): void {
this.dataSource.filter = '';
if (this.shown === 'all') {
this.dataSource.filterPredicate = (data, filter) => {
return true;
};
} else if (this.shown === 'noerror') {
this.dataSource.filterPredicate = (data, filter) => {
if (data.status === 'done') {
return true;
} else if (data.status !== 'error') {
return true;
}
};
} else if (this.shown === 'error') {
this.dataSource.filterPredicate = (data, filter) => {
if (data.errors.length || data.duplicates.length) {
return true;
}
return false;
};
}
this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter
}
/**
* Get the appropiate css class for a row according to the import state
*
* @param row a newEntry object with a current status
* @returns a css class name
*/
public getStateClass(row: NewEntry<V>): string {
switch (row.status) {
case 'done':
return 'import-done import-decided';
case 'error':
return 'import-error';
default:
return '';
}
}
/**
* Get the icon for the action of the item
* @param entry a newEntry object with a current status
* @eturn the icon for the action of the item
*/
public getActionIcon(entry: NewEntry<V>): string {
switch (entry.status) {
case 'error': // no import possible
return 'block';
case 'new':
return '';
case 'done': // item has been imported
return 'done';
default:
// fallback: Error
return 'block';
}
}
/**
* A function to trigger the csv example download.
*/
public abstract downloadCsvExample(): void;
/**
* Trigger for the column separator selection.
*
* @param event
*/
public selectColSep(event: MatSelectChange): void {
this.importer.columnSeparator = event.value;
this.importer.refreshFile();
}
/**
* Trigger for the column separator selection
*
* @param event
*/
public selectTextSep(event: MatSelectChange): void {
this.importer.textSeparator = event.value;
this.importer.refreshFile();
}
/**
* Trigger for the encoding selection
*
* @param event
*/
public selectEncoding(event: MatSelectChange): void {
this.importer.encoding = event.value;
this.importer.refreshFile();
}
/**
* Returns a descriptive string for an import error
*
* @param error The short string for the error as listed in the {@lilnk errorList}
* @returns a predefined descriptive error string from the importer
*/
public getVerboseError(error: string): string {
return this.importer.verbose(error);
}
/**
* Checks if an error is present in a new entry
*
* @param row the NewEntry
* @param error An error as defined as key of {@link errorList}
* @returns true if the error is present in the entry described in the row
*/
public hasError(row: NewEntry<V>, error: string): boolean {
return this.importer.hasError(row, error);
}
}

80
client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html
View File

@ -9,11 +9,10 @@
</div>
</os-head-bar>
<mat-card>
<mat-card class="os-form-card import-table">
<span translate>Requires comma or semicolon separated values with these column header names in the first row</span>:
<br />
<div class="code red-warning-text">
<!-- TODO: class : indent, warning color -->
<span translate>Identifier</span>, <span translate>Title</span>, <span translate>Text</span>,
<span translate>Reason</span>, <span translate>Submitter</span>, <span translate>Category</span>,
<span translate>Origin</span>, <span translate>Motion block</span>
@ -23,7 +22,6 @@
Identifier, reason, submitter, category, origin and motion block are optional and may be empty.
</li>
<li translate>Additional columns after the required ones may be present and won't affect the import.</li>
<li translate>Only double quotes are accepted as text delimiter (no single quotes).</li>
</ul>
<button mat-button color="accent" (click)="downloadCsvExample()" translate>Download CSV example file</button>
<div class="wrapper">
@ -71,12 +69,12 @@
<span translate> Select file</span>
</button>
</div>
&nbsp; <span *ngIf="hasFile">{{ totalCount }}&nbsp;<span translate>entries found.</span></span>
</div>
</mat-card>
<!-- preview table -->
<mat-card *ngIf="hasFile">
<mat-card *ngIf="hasFile" class="os-form-card import-table">
<h3 translate>Preview</h3>
<div class="summary">
<!-- new entries -->
<div *ngIf="newCount">
@ -97,35 +95,38 @@
<span>&nbsp;{{ doneCount }}&nbsp;</span> <span translate>Motions have been imported.</span>
</div>
</div>
<div *ngIf="newCount">
<span translate>Click on 'import' (right top corner) to import the new motions.
</span>
</div>
<div>
<mat-select [(value)]="shown" (selectionChange)="setFilter()">
<!-- TODO: reduce item width to sane value -->
<mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()">
<mat-option value="all" translate> Show all </mat-option>
<mat-option *ngIf="nonImportableCount" value="error" translate> Show errors only </mat-option>
<mat-option *ngIf="nonImportableCount" value="noerror" translate> Show correct entries </mat-option>
<mat-option value="error" translate> Show errors only </mat-option>
<mat-option value="noerror" translate> Show correct entries </mat-option>
</mat-select>
<!-- TODO: Button to hide imported ones -->
</div>
<div class="table-container">
<table mat-table class="on-transition-fade" [dataSource]="dataSource" matSort>
<!-- Status column -->
<ng-container matColumnDef="status" sticky>
<mat-header-cell *matHeaderCellDef></mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.status === 'error'">
<mat-header-cell *matHeaderCellDef class="first-column"></mat-header-cell>
<mat-cell *matCellDef="let entry" class="first-column">
<div *ngIf="entry.status === 'error'">
<mat-icon
class="red-warning-text"
matTooltip="{{ entry.newMotion.errors.length }} + {{ 'errors' | translate }}"
matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
>
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.newMotion.status === 'new'">
<div *ngIf="entry.status === 'new'">
<mat-icon matTooltip="{{ 'Motion will be imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.newMotion.status === 'done'">
<div *ngIf="entry.status === 'done'">
<mat-icon matTooltip="{{ 'Motion has been imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
@ -140,12 +141,12 @@
<mat-icon
color="warn"
inline
*ngIf="entry.newMotion.hasError('Duplicates')"
*ngIf="hasError(entry, 'Duplicates')"
matTooltip="{{ getVerboseError('Duplicates') | translate }}"
>
warning
</mat-icon>
&nbsp;{{ entry.newMotion.identifier }}
{{ entry.newEntry.identifier }}
</mat-cell>
</ng-container>
@ -155,35 +156,35 @@
<mat-cell *matCellDef="let entry">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Title')"
*ngIf="hasError(entry, 'Title')"
matTooltip="{{ getVerboseError('Title') | translate }}"
>
warning
</mat-icon>
&nbsp;{{ entry.newMotion.title }}
{{ entry.newEntry.title }}
</mat-cell>
</ng-container>
<!-- tect column TODO: Bigger-->
<!-- text column -->
<ng-container matColumnDef="text">
<mat-header-cell *matHeaderCellDef translate>Motion text</mat-header-cell>
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newMotion.text) }}">
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newEntry.text) }}">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Text')"
*ngIf="hasError(entry, 'Text')"
matTooltip="{{ getVerboseError('Text') | translate }}"
>
warning
</mat-icon>
&nbsp;{{ getShortPreview(entry.newMotion.text) }}
{{ getShortPreview(entry.newEntry.text) }}
</mat-cell>
</ng-container>
<!-- reason column -->
<ng-container matColumnDef="reason">
<mat-header-cell *matHeaderCellDef translate>Reason</mat-header-cell>
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newMotion.reason) }}">
{{ getShortPreview(entry.newMotion.reason) }}
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newEntry.reason) }}">
{{ getShortPreview(entry.newEntry.reason) }}
</mat-cell>
</ng-container>
@ -191,15 +192,15 @@
<ng-container matColumnDef="submitters">
<mat-header-cell *matHeaderCellDef translate>Submitters</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.csvSubmitters.length">
<div *ngIf="entry.newEntry.csvSubmitters.length">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Submitters')"
*ngIf="hasError(entry, 'Submitters')"
matTooltip="{{ getVerboseError('Submitters') | translate }}"
>
warning
</mat-icon>
<span *ngFor="let submitter of entry.newMotion.csvSubmitters">
<span *ngFor="let submitter of entry.newEntry.csvSubmitters">
{{ submitter.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!submitter.id">add</mat-icon>
&nbsp;
@ -212,16 +213,16 @@
<ng-container matColumnDef="category">
<mat-header-cell *matHeaderCellDef translate>Category</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.csvCategory">
<div *ngIf="entry.newEntry.csvCategory">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Category')"
*ngIf="hasError(entry, 'Category')"
matTooltip="{{ getVerboseError('Category') | translate }}"
>
warning
</mat-icon>
{{ entry.newMotion.csvCategory.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newMotion.csvCategory.id"
{{ entry.newEntry.csvCategory.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newEntry.csvCategory.id"
>add</mat-icon
>&nbsp;
</div>
@ -231,23 +232,23 @@
<!-- origin column -->
<ng-container matColumnDef="origin">
<mat-header-cell *matHeaderCellDef translate>Origin</mat-header-cell>
<mat-cell *matCellDef="let entry">{{ entry.newMotion.origin }}</mat-cell>
<mat-cell *matCellDef="let entry">{{ entry.newEntry.origin }}</mat-cell>
</ng-container>
<!-- motion block column -->
<ng-container matColumnDef="motion block">
<ng-container matColumnDef="motion_block">
<mat-header-cell *matHeaderCellDef translate>Motion block</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.csvMotionblock">
<div *ngIf="entry.newEntry.csvMotionblock">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('MotionBlock')"
*ngIf="hasError(entry, 'MotionBlock')"
matTooltip="{{ getVerboseError('MotionBlock') | translate }}"
>
warning
</mat-icon>
{{ entry.newMotion.csvMotionblock.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newMotion.csvMotionblock.id">
{{ entry.newEntry.csvMotionblock.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newEntry.csvMotionblock.id">
add
</mat-icon>
&nbsp;
@ -256,7 +257,6 @@
</ng-container>
<mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
<!-- TODO: class for invalid/undecided -->
<mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
</table>
</div>

53
client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss
View File

@ -1,53 +0,0 @@
.table-container {
width: 100%;
overflow-x: scroll;
margin-top: 5px;
}
table {
width: 100%;
overflow: scroll;
}
.mat-header-cell {
min-width: 100px;
padding-right: 8px;
}
.mat-cell {
min-width: 100px;
padding-top: 2px;
padding-right: 8px;
}
.selection {
min-width: 80px;
}
.import-done {
background-color: #cfc;
}
.import-error {
background-color: #fcc;
}
.code {
padding-left: 1em;
font-style: italic;
}
div.wrapper {
display: flex;
vertical-align: bottom;
padding: 10px;
}
div.summary {
display: flex;
}
.hidden-input {
display: none;
}
.newBadge {
margin-left: -3px;
}

244
client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts
View File

@ -1,100 +1,21 @@
import { MatTableDataSource, MatTable, MatSnackBar, MatSelectChange } from '@angular/material';
import { Component } from '@angular/core';
import { MatSnackBar } from '@angular/material';
import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core';
import { ViewChild, Component, OnInit } from '@angular/core';
import { BaseViewComponent } from 'app/site/base/base-view';
import { BaseImportListComponent } from 'app/site/base/base-import-list';
import { MotionCsvExportService } from '../../services/motion-csv-export.service';
import { MotionImportService, NewMotionEntry, ValueLabelCombination } from '../../services/motion-import.service';
import { MotionImportService } from '../../services/motion-import.service';
import { ViewMotion } from '../../models/view-motion';
/**
* Component for the motion import list view.
*/
@Component({
selector: 'os-motion-import-list',
templateUrl: './motion-import-list.component.html',
styleUrls: ['./motion-import-list.component.scss']
templateUrl: './motion-import-list.component.html'
})
export class MotionImportListComponent extends BaseViewComponent implements OnInit {
/**
* The data source for a table. Requires to be initialised with a BaseViewModel
*/
public dataSource: MatTableDataSource<NewMotionEntry>;
/**
* Switch that turns true if a file has been selected in the input
*/
public hasFile = false;
/**
* Currently selected encoding. Is set and changed by the config's available
* encodings and user mat-select input
*/
public selectedEncoding = 'utf-8';
/**
* indicator on which elements to display
*/
public shown: 'all' | 'error' | 'noerror' = 'all';
/**
* The table itself
*/
@ViewChild(MatTable)
protected table: MatTable<NewMotionEntry>;
/**
* Returns the amount of total item successfully parsed
*/
public get totalCount(): number {
return this.importer && this.hasFile ? this.importer.summary.total : null;
}
/**
* Returns the encodings available and their labels
*/
public get encodings(): ValueLabelCombination[] {
return this.importer.encodings;
}
/**
* Returns the available column separators and their labels
*/
public get columnSeparators(): ValueLabelCombination[] {
return this.importer.columnSeparators;
}
/**
* Returns the available text separators and their labels
*/
public get textSeparators(): ValueLabelCombination[] {
return this.importer.textSeparators;
}
/**
* Returns the amount of import items that will be imported
*/
public get newCount(): number {
return this.importer && this.hasFile ? this.importer.summary.new : 0;
}
/**
* Returns the number of import items that cannot be imported
*/
public get nonImportableCount(): number {
if (this.importer && this.hasFile) {
return this.importer.summary.errors + this.importer.summary.duplicates;
}
return 0;
}
/**
* Returns the number of import items that have been successfully imported
*/
public get doneCount(): number {
return this.importer && this.hasFile ? this.importer.summary.done : 0;
}
export class MotionImportListComponent extends BaseImportListComponent<ViewMotion> {
/**
* Constructor for list view bases
*
@ -107,101 +28,11 @@ export class MotionImportListComponent extends BaseViewComponent implements OnIn
public constructor(
titleService: Title,
matSnackBar: MatSnackBar,
public translate: TranslateService,
private importer: MotionImportService,
translate: TranslateService,
importer: MotionImportService,
private motionCSVExport: MotionCsvExportService
) {
super(titleService, translate, matSnackBar);
this.initTable();
this.importer.errorEvent.subscribe(this.raiseError);
}
/**
* Starts with a clean preview (removing any previously existing import previews)
*/
public ngOnInit(): void {
this.importer.clearPreview();
}
/**
* Initializes the table
*/
public initTable(): void {
this.dataSource = new MatTableDataSource();
this.setFilter();
this.importer.getNewEntries().subscribe(newEntries => {
this.dataSource.data = newEntries;
this.hasFile = newEntries.length > 0;
});
}
/**
* Returns the table column definition. Fetches all headers from
* {@link MotionImportService} and an additional status column
*/
public getColumnDefinition(): string[] {
return ['status'].concat(this.importer.expectedHeader);
}
/**
* triggers the importer's onSelectFile after a file has been chosen
*/
public onSelectFile(event: any): void {
this.importer.onSelectFile(event);
}
/**
* Triggers the importer's import
*/
public async doImport(): Promise<void> {
await this.importer.doImport();
this.setFilter();
}
/**
* Updates and manually triggers the filter function.
* See {@link hidden} for options
* (changed from default mat-table filter)
*/
public setFilter(): void {
this.dataSource.filter = '';
if (this.shown === 'all') {
this.dataSource.filterPredicate = (data, filter) => {
return true;
};
} else if (this.shown === 'noerror') {
this.dataSource.filterPredicate = (data, filter) => {
if (data.newMotion.status === 'done') {
return true;
} else if (!(data.newMotion.status !== 'error') && !data.duplicates.length) {
return true;
}
};
} else if (this.shown === 'error') {
this.dataSource.filterPredicate = (data, filter) => {
if (data.newMotion.errors.length || data.duplicates.length) {
return true;
}
return false;
};
}
this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter
}
/**
* Returns the appropiate css class for a row according to the import state
*
* @param row
*/
public getStateClass(row: NewMotionEntry): string {
switch (row.newMotion.status) {
case 'done':
return 'import-done import-decided';
case 'error':
return 'import-error';
default:
return '';
}
super(importer, titleService, translate, matSnackBar);
}
/**
@ -233,24 +64,6 @@ export class MotionImportListComponent extends BaseViewComponent implements OnIn
);
}
/**
* Return the icon for the action of the item
* @param entry
*/
public getActionIcon(entry: NewMotionEntry): string {
switch (entry.newMotion.status) {
case 'error': // no import possible
return 'block';
case 'new': // new item, will be imported
return 'playlist_add';
case 'done': // item has been imported
return 'done';
default:
// fallback: Error
return 'block';
}
}
/**
* Helper to remove html tags from a string.
* CAUTION: It is just a basic "don't show distracting html tags in a
@ -268,41 +81,4 @@ export class MotionImportListComponent extends BaseViewComponent implements OnIn
public downloadCsvExample(): void {
this.motionCSVExport.exportDummyMotion();
}
/**
* Trigger for the column separator selection
*
* @param event
*/
public selectColSep(event: MatSelectChange): void {
this.importer.columnSeparator = event.value;
this.importer.refreshFile();
}
/**
* Trigger for the column separator selection
*
* @param event
*/
public selectTextSep(event: MatSelectChange): void {
this.importer.textSeparator = event.value;
this.importer.refreshFile();
}
/**
* Trigger for the encoding selection
*
* @param event
*/
public selectEncoding(event: MatSelectChange): void {
this.importer.encoding = event.value;
this.importer.refreshFile();
}
/**
* Returns a descriptive string for an import error
*/
public getVerboseError(error: string): string {
return this.importer.verbose(error);
}
}

65
client/src/app/site/motions/models/view-csv-create-motion.ts
View File

@ -2,9 +2,8 @@ import { ViewCreateMotion } from './view-create-motion';
import { CreateMotion } from './create-motion';
/**
* Interface for imported secondary data. A name can be matched to an existing
* model instance by the solve... functions.
* TODO MultiId will be filled if there is more than one match (to be used in case of 'I want to select one of these matches)
* Interface for correlating between strings representing BaseModels and existing
* BaseModels.
*/
export interface CsvMapping {
name: string;
@ -12,8 +11,6 @@ export interface CsvMapping {
multiId?: number[];
}
type CsvImportStatus = 'new' | 'error' | 'done';
/**
* Create motion class for the View. Its different to ViewMotion in fact that the submitter handling is different
* on motion creation.
@ -38,68 +35,10 @@ export class ViewCsvCreateMotion extends ViewCreateMotion {
*/
public csvSubmitters: CsvMapping[];
/**
* The current import status of this motion.
* Starts as 'new', if set to 'done', a proper {@link Motion} model will
* probably exist in the dataStore. error status will be set if the import
* cannot be done
*/
private _status: CsvImportStatus = 'new';
/**
* list of import errors See {@link MotionImportService}
*/
public errors: string[] = [];
/**
* Returns the current status.
*/
public get status(): CsvImportStatus {
return this._status;
}
public set status(status: CsvImportStatus) {
this._status = status;
}
public get motion(): CreateMotion {
return this._motion;
}
public constructor(motion?: CreateMotion) {
super(motion);
}
/**
* Duplicate this motion into a copy of itself
*/
public copy(): ViewCreateMotion {
return new ViewCreateMotion(
this._motion,
this._category,
this._submitters,
this._supporters,
this._workflow,
this._state
);
}
/**
* Checks if a given error is present. TODO: Is more a ViewModel option
*
* @param error
*/
public hasError(error: string): boolean {
return this.errors.includes(error);
}
/**
* Toggle to set a CreateMotion to a 'successfully parsed' status
*/
public done(): void {
this._status = 'done';
}
/**
* takes a list of motion block mappings to update the current csvMotionblock.
* Returns the amount of entries that remain unmatched

438
client/src/app/site/motions/services/motion-import.service.ts
View File

@ -1,7 +1,6 @@
import { BehaviorSubject, Observable } from 'rxjs';
import { Injectable, EventEmitter } from '@angular/core';
import { Injectable } from '@angular/core';
import { MatSnackBar } from '@angular/material';
import { Papa, PapaParseConfig } from 'ngx-papaparse';
import { Papa } from 'ngx-papaparse';
import { TranslateService } from '@ngx-translate/core';
import { Category } from 'app/shared/models/motions/category';
@ -11,49 +10,9 @@ import { MotionBlock } from 'app/shared/models/motions/motion-block';
import { MotionBlockRepositoryService } from './motion-block-repository.service';
import { MotionRepositoryService } from './motion-repository.service';
import { UserRepositoryService } from '../../users/services/user-repository.service';
import { ViewMotion } from '../models/view-motion';
import { ViewCsvCreateMotion, CsvMapping } from '../models/view-csv-create-motion';
/**
* Interface for value- Label combinations.
* Map objects didn't work, TODO: Use map objects (needs iterating through all objects of a map)
*/
export interface ValueLabelCombination {
value: string;
label: string;
}
/**
* Interface for a new Motion and their (if any) duplicates
*/
export interface NewMotionEntry {
newMotion: ViewCsvCreateMotion;
duplicates: ViewMotion[];
}
/**
* interface for a preview summary
*/
interface ImportMotionCSVPreview {
total: number;
duplicates: number;
errors: number;
new: number;
done: number;
}
/**
* List of possible import errors specific for motion imports.
*/
const errorList = {
MotionBlock: 'Could not resolve the motion block',
Category: 'Could not resolve the category',
Submitters: 'Could not resolve the submitters',
Title: 'A title is required',
Text: "A content in the 'text' column is required",
Duplicates: 'A motion with this identifier already exists.',
generic: 'Server upload failed' // TODO
};
import { BaseImportService, NewEntry } from 'app/core/services/base-import.service';
import { ViewMotion } from '../models/view-motion';
/**
* Service for motion imports
@ -61,59 +20,23 @@ const errorList = {
@Injectable({
providedIn: 'root'
})
export class MotionImportService {
/** The header (order and items) that is expected from the imported file
*
export class MotionImportService extends BaseImportService<ViewMotion> {
/**
* List of possible errors and their verbose explanation
*/
public expectedHeader = [
'identifier',
'title',
'text',
'reason',
'submitters',
'category',
'origin',
'motion block'
];
public errorList = {
MotionBlock: 'Could not resolve the motion block',
Category: 'Could not resolve the category',
Submitters: 'Could not resolve the submitters',
Title: 'A title is required',
Text: "A content in the 'text' column is required",
Duplicates: 'A motion with this identifier already exists.'
};
/**
* The last parsed file object (may be reparsed with new encoding, thus kept in memory)
* The minimimal number of header entries needed to successfully create an entry
*/
private _rawFile: File;
/**
* The used column Separator. If left on an empty string (default),
* the papaparse parser will automatically decide on separators.
*/
public columnSeparator = '';
public textSeparator = '"';
public encoding = 'utf-8';
/**
* List of possible encodings and their label
*/
public encodings: ValueLabelCombination[] = [
{ value: 'utf-8', label: 'UTF 8 - Unicode' },
{ value: 'iso-8859-1', label: 'ISO 8859-1 - West European' },
{ value: 'iso-8859-15', label: 'ISO 8859-15 - West European (with €)' }
];
/**
* List of possible column separators
*/
public columnSeparators: ValueLabelCombination[] = [
{ label: 'Comma', value: ',' },
{ label: 'Semicolon', value: ';' },
// {label: 'Tabulator', value: '\t'},
{ label: 'Automatic', value: '' }
];
public textSeparators: ValueLabelCombination[] = [
{ label: 'Double quotes (")', value: '"' },
{ label: "Single quotes (')", value: "'" }
];
public requiredHeaderLength = 3;
/**
* submitters that need to be created prior to importing
@ -131,44 +54,8 @@ export class MotionImportService {
public newMotionBlocks: CsvMapping[] = [];
/**
* FileReader object for file import
*/
private reader = new FileReader();
/**
* the list of parsed models that have been extracted from the opened file
*/
private _entries: NewMotionEntry[] = [];
/**
* BehaviorSubject for displaying a preview for the currently selected entries
*/
public newEntries = new BehaviorSubject<NewMotionEntry[]>([]);
/**
* Emits an error string to display if a file import cannot be done
*/
public errorEvent = new EventEmitter<string>();
/**
* storing the summary preview for the import, to avoid recalculating it
* at each display change.
*/
private _preview: ImportMotionCSVPreview;
/**
* Returns a summary on actions that will be taken/not taken.
*/
public get summary(): ImportMotionCSVPreview {
if (!this._preview) {
this.updatePreview();
}
return this._preview;
}
/**
* Constructor. Creates a fileReader to subscribe to it for incoming parsed
* strings
* Constructor. Defines the headers expected and calls the abstract class
* @param repo: The repository for motions.
* @param categoryRepo Repository to fetch pre-existing categories
* @param motionBlockRepo Repository to fetch pre-existing motionBlocks
* @param userRepo Repository to query/ create users
@ -181,158 +68,112 @@ export class MotionImportService {
private categoryRepo: CategoryRepositoryService,
private motionBlockRepo: MotionBlockRepositoryService,
private userRepo: UserRepositoryService,
private translate: TranslateService,
private papa: Papa,
private matSnackbar: MatSnackBar
translate: TranslateService,
papa: Papa,
matSnackbar: MatSnackBar
) {
this.reader.onload = (event: any) => {
// TODO type: event is a progressEvent,
// but has a property target.result, which typescript doesn't recognize
this.parseInput(event.target.result);
};
super(translate, papa, matSnackbar);
this.expectedHeader = [
'identifier',
'title',
'text',
'reason',
'submitters',
'category',
'origin',
'motion_block'
];
}
/**
* Parses the data input. Expects a string as returned by via a
* File.readAsText() operation
*
* @param file
* Clears all temporary data specific to this importer.
*/
public parseInput(file: string): void {
this._entries = [];
public clearData(): void {
this.newSubmitters = [];
this.newCategories = [];
this.newMotionBlocks = [];
const papaConfig: PapaParseConfig = {
header: false,
skipEmptyLines: true,
quoteChar: this.textSeparator
};
if (this.columnSeparator) {
papaConfig.delimiter = this.columnSeparator;
}
const entryLines = this.papa.parse(file, papaConfig).data;
const valid = this.checkHeader(entryLines.shift());
if (!valid) {
return;
}
entryLines.forEach(line => {
const newMotion = new ViewCsvCreateMotion(new CreateMotion());
const headerLength = Math.min(this.expectedHeader.length, line.length);
for (let idx = 0; idx < headerLength; idx++) {
// iterate over items, find existing ones (thier id) and collect new entries
switch (this.expectedHeader[idx]) {
case 'submitters':
newMotion.csvSubmitters = this.getSubmitters(line[idx]);
break;
case 'category':
newMotion.csvCategory = this.getCategory(line[idx]);
break;
case 'motion block':
newMotion.csvMotionblock = this.getMotionBlock(line[idx]);
break;
default:
newMotion.motion[this.expectedHeader[idx]] = line[idx];
}
}
const updateModels = this.getDuplicates(newMotion.motion);
if (updateModels.length) {
this.setError(newMotion, 'Duplicates');
}
this._entries.push({ newMotion: newMotion, duplicates: updateModels });
});
this.newEntries.next(this._entries);
this.updatePreview();
}
/**
* Triggers the import.
* Parses a string representing an entry, extracting secondary data, appending
* the array of secondary imports as needed
*
* @param line
* @returns a new Entry representing a Motion
*/
public mapData(line: string): NewEntry<ViewMotion> {
const newEntry = new ViewCsvCreateMotion(new CreateMotion());
const headerLength = Math.min(this.expectedHeader.length, line.length);
for (let idx = 0; idx < headerLength; idx++) {
switch (this.expectedHeader[idx]) {
case 'submitters':
newEntry.csvSubmitters = this.getSubmitters(line[idx]);
break;
case 'category':
newEntry.csvCategory = this.getCategory(line[idx]);
break;
case 'motion_block':
newEntry.csvMotionblock = this.getMotionBlock(line[idx]);
break;
default:
newEntry.motion[this.expectedHeader[idx]] = line[idx];
}
}
const updateModels = this.repo.getMotionDuplicates(newEntry);
return {
newEntry: newEntry,
duplicates: updateModels,
status: updateModels.length ? 'error' : 'new',
errors: updateModels.length ? ['Duplicates'] : []
};
}
/**
* Executes the import. Creates all secondary data, maps the newly created
* secondary data to the new entries, then creates all entries without errors
* by submitting them to the server. The entries will receive the status
* 'done' on success.
*/
public async doImport(): Promise<void> {
this.newMotionBlocks = await this.createNewMotionBlocks();
this.newCategories = await this.createNewCategories();
this.newSubmitters = await this.createNewUsers();
for (const entry of this._entries) {
if (entry.newMotion.status !== 'new') {
for (const entry of this.entries) {
if (entry.status !== 'new') {
continue;
}
const openBlocks = entry.newMotion.solveMotionBlocks(this.newMotionBlocks);
const openBlocks = (entry.newEntry as ViewCsvCreateMotion).solveMotionBlocks(this.newMotionBlocks);
if (openBlocks) {
this.setError(entry.newMotion, 'MotionBlock');
// TODO error handling if not all submitters could be matched
this.setError(entry, 'MotionBlock');
this.updatePreview();
continue;
}
const openCategories = entry.newMotion.solveCategory(this.newCategories);
const openCategories = (entry.newEntry as ViewCsvCreateMotion).solveCategory(this.newCategories);
if (openCategories) {
this.setError(entry.newMotion, 'Category');
this.setError(entry, 'Category');
this.updatePreview();
continue;
}
const openUsers = entry.newMotion.solveSubmitters(this.newSubmitters);
const openUsers = (entry.newEntry as ViewCsvCreateMotion).solveSubmitters(this.newSubmitters);
if (openUsers) {
this.setError(entry.newMotion, 'Submitters');
this.setError(entry, 'Submitters');
this.updatePreview();
continue;
}
await this.repo.create(entry.newMotion.motion);
entry.newMotion.done();
await this.repo.create((entry.newEntry as ViewCsvCreateMotion).motion);
entry.status = 'done';
}
this.updatePreview();
}
/**
* Checks the dataStore for duplicates
* @returns an array of duplicates with the same identifier.
* @param motion
*/
public getDuplicates(motion: CreateMotion): ViewMotion[] {
return this.repo.getMotionDuplicates(motion);
}
/**
* counts the amount of duplicates that have no decision on the action to
* be taken
*/
public updatePreview(): void {
const summary = {
total: 0,
new: 0,
duplicates: 0,
errors: 0,
done: 0
};
this._entries.forEach(entry => {
summary.total += 1;
if (entry.newMotion.status === 'done') {
summary.done += 1;
return;
} else if (entry.newMotion.status === 'error' && !entry.duplicates.length) {
// errors that are not due to duplicates
summary.errors += 1;
return;
} else if (entry.duplicates.length) {
summary.duplicates += 1;
return;
} else if (entry.newMotion.status === 'new') {
summary.new += 1;
}
});
this._preview = summary;
}
/**
* returns a subscribable representation of the new Users to be imported
*/
public getNewEntries(): Observable<NewMotionEntry[]> {
return this.newEntries.asObservable();
}
/**
* Checks the provided submitter(s) and returns an object with mapping of
* existing users and of users that need to be created
*
* @param submitterlist
* @returns a list of submitters mapped with (if already existing) their id
*/
public getSubmitters(submitterlist: string): CsvMapping[] {
const result: CsvMapping[] = [];
@ -375,7 +216,9 @@ export class MotionImportService {
* characters at the beginning, separated by ' - ' from the name.
* It will also accept a registered translation between the current user's
* language and english
*
* @param categoryString
* @returns categories mapped to existing categories
*/
public getCategory(categoryString: string): CsvMapping {
if (!categoryString) {
@ -411,7 +254,9 @@ export class MotionImportService {
* Checks the motionBlock provided in the string for existance, expands newMotionBlocks
* if needed. Note that it will also check for translation between the current
* user's language and english
*
* @param blockString
* @returns a CSVMap with the MotionBlock and an id (if the motionBlock is already in the dataStore)
*/
public getMotionBlock(blockString: string): CsvMapping {
if (!blockString) {
@ -434,6 +279,8 @@ export class MotionImportService {
/**
* Creates all new Users needed for the import.
*
* @returns a promise with list of new Submitters, updated with newly created ids
*/
private async createNewUsers(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
@ -445,6 +292,8 @@ export class MotionImportService {
/**
* Creates all new Motion Blocks needed for the import.
*
* @returns a promise with list of new MotionBlocks, updated with newly created ids
*/
private async createNewMotionBlocks(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
@ -460,6 +309,8 @@ export class MotionImportService {
/**
* Creates all new Categories needed for the import.
*
* @returns a promise with list of new Categories, updated with newly created ids
*/
private async createNewCategories(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
@ -481,105 +332,12 @@ export class MotionImportService {
return await Promise.all(promises);
}
/**
* Handler after a file was selected. Basic checking for type, then hand
* over to parsing
*
* @param event type is Event, but has target.files, which typescript doesn't seem to recognize
*/
public onSelectFile(event: any): void {
// TODO type
if (event.target.files && event.target.files.length === 1) {
if (event.target.files[0].type === 'text/csv') {
this._rawFile = event.target.files[0];
this.readFile(event.target.files[0]);
} else {
this.matSnackbar.open(this.translate.instant('Wrong file type detected. Import failed.'), '', {
duration: 3000
});
this.clearPreview();
this._rawFile = null;
}
}
}
/**
* Rereads the (previously selected) file, if present. Thought to be triggered
* by parameter changes on encoding, column, text separators
*/
public refreshFile(): void {
if (this._rawFile) {
this.readFile(this._rawFile);
}
}
/**
* (re)-reads a given file with the current parameter
*/
private readFile(file: File): void {
this.reader.readAsText(file, this.encoding);
}
/**
* Checks the first line of the csv (the header) for consistency (length)
* @param row expected to be an array parsed from the first line of a csv file
*/
private checkHeader(row: string[]): boolean {
const snackbarDuration = 3000;
if (row.length < 4) {
this.matSnackbar.open(this.translate.instant('The file has too few columns to be parsed properly.'), '', {
duration: snackbarDuration
});
this.clearPreview();
return false;
} else if (row.length < this.expectedHeader.length) {
this.matSnackbar.open(
this.translate.instant('The file seems to have some ommitted columns. They will be considered empty.'),
'',
{ duration: snackbarDuration }
);
} else if (row.length > this.expectedHeader.length) {
this.matSnackbar.open(
this.translate.instant('The file seems to have additional columns. They will be ignored.'),
'',
{ duration: snackbarDuration }
);
}
return true;
}
/**
* Resets the data and preview (triggered upon selecting an invalid file)
*/
public clearPreview(): void {
this._entries = [];
this.newEntries.next([]);
this._preview = null;
}
/**
* set a list of short names for error, indicating which column failed
*/
public setError(motion: ViewCsvCreateMotion, error: string): void {
if (errorList.hasOwnProperty(error) && !motion.errors.includes(error)) {
motion.errors.push(error);
motion.status = 'error';
}
}
/**
* Get an extended error description.
* @param error
*/
public verbose(error: string): string {
return errorList[error];
}
/**
* Helper to separate a category string from its' prefix. Assumes that a prefix is no longer
* than 5 chars and separated by a ' - '
*
* @param categoryString the string to parse
* @returns an object with .prefix and .name strings
*/
private splitCategoryString(categoryString: string): { prefix: string; name: string } {
let prefixSeparator = ' - ';

7
client/src/app/site/motions/services/motion-repository.service.ts
View File

@ -607,12 +607,13 @@ export class MotionRepositoryService extends BaseRepository<ViewMotion, Motion>
}
/**
* Returns all Motion duplicates (sharing specific values given in input)
* Returns motion duplicates (sharing the identifier)
*
* @param viewMotion the ViewMotion to compare against the list of Motions
* in the data
* @param sharedValues properties that must be equal to consider it a duplicate
* @returns An Array of ViewMotions with the same identifier of the input, or an empty array
*/
public getMotionDuplicates(motion: Motion): ViewMotion[] {
public getMotionDuplicates(motion: ViewMotion): ViewMotion[] {
const duplicates = this.DS.filter(Motion, item => motion.identifier === item.identifier);
const viewMotions: ViewMotion[] = [];
duplicates.forEach(item => viewMotions.push(this.createViewModel(item)));

283
client/src/app/site/users/components/user-import/user-import-list.component.html Normal file
View File

@ -0,0 +1,283 @@
<os-head-bar [nav]="false">
<!-- Title -->
<div class="title-slot"><h2 translate>Import users</h2></div>
<div class="menu-slot">
<button *ngIf="hasFile && newCount" mat-button (click)="doImport()">
<span class="upper" translate> Import</span>
</button>
</div>
</os-head-bar>
<mat-card class="os-form-card import-table">
<mat-tab-group (selectedTabChange)="onTabChange()">
<mat-tab label="{{ 'CSV import' | translate }}">
<span translate
>Requires comma or semicolon separated values with these column header names in the first row</span
>: <br />
<div class="code red-warning-text">
<span translate>Title</span>, <span translate>Given name</span>, <span translate>Surname</span> ,
<span translate>Structure level</span>, <span translate>Participant number</span>,
<span translate>Groups</span> , <span translate>Comment</span>, <span translate>Is active</span>,
<span translate>Is present</span> , <span translate>Is committee</span>,
<span translate>Initial password</span>, <span translate>Email</span>
</div>
<ul>
<li translate>
At least given name or surname have to be filled in. All other fields are optional and may be empty.
</li>
<li translate>
Additional columns after the required ones may be present and won't affect the import.
</li>
</ul>
<button mat-button color="accent" (click)="downloadCsvExample()" translate>
Download CSV example file
</button>
<div class="wrapper">
<mat-form-field>
<mat-label translate>Encoding of the file</mat-label>
<mat-select
class="selection"
placeholder="translate.instant('Select encoding')"
(selectionChange)="selectEncoding($event)"
[value]="encodings[0].value"
>
<mat-option *ngFor="let option of encodings" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field>
<mat-label translate> Column Separator</mat-label>
<mat-select class="selection" (selectionChange)="selectColSep($event)" value="">
<mat-option *ngFor="let option of columnSeparators" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field>
<mat-label translate>Text separator</mat-label>
<mat-select class="selection" (selectionChange)="selectTextSep($event)" value="&quot;">
<mat-option *ngFor="let option of textSeparators" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
</div>
<div>
<div>
<input
id="user-import-file-input"
type="file"
class="hidden-input"
accept="text"
#fileInput
(change)="onSelectFile($event)"
/>
<button mat-button color="accent" onclick="document.getElementById('user-import-file-input').click()">
<span translate> Select file</span>
</button>
</div>
</div>
</mat-tab>
<mat-tab label="{{ 'Text import' | translate }}">
<div [formGroup]="textAreaForm">
<div>
<span translate>
Copy and paste your participant names in this textbox.</span>
<span translate>
Keep each person in a single line.
</span><br />
<span translate> Comma separated names will be read as 'Surname(s), given name(s)'. </span>
</div>
<mat-form-field>
<textarea
matInput
formControlName="inputtext"
placeholder="{{ 'Insert users here' | translate }}"
cdkTextareaAutosize
cdkAutosizeMinRows="3"
cdkAutosizeMaxRows="10"
></textarea>
</mat-form-field>
</div>
<div>
<button mat-button color="accent" (click)="parseTextArea()"><span translate>Preview</span></button>
</div>
</mat-tab>
</mat-tab-group>
</mat-card>
<!-- preview table -->
<mat-card *ngIf="hasFile" class="os-form-card import-table">
<h3 translate> Preview</h3>
<div class="summary">
<!-- new entries -->
<div *ngIf="newCount">
&nbsp;
<mat-icon inline>playlist_add</mat-icon>
<span>&nbsp;{{ newCount }}&nbsp;</span> <span translate>User(s) will be imported.</span>
</div>
<!-- errors/duplicates -->
<div *ngIf="nonImportableCount" class="red-warning-text">
&nbsp;
<mat-icon inline>warning</mat-icon>
<span>&nbsp;{{ nonImportableCount }}&nbsp;</span> <span translate>entries will be ommitted.</span>
</div>
<!-- have been imported -->
<div *ngIf="doneCount" class="green-text">
&nbsp;
<mat-icon inline>done</mat-icon>
<span>&nbsp;{{ doneCount }}&nbsp;</span> <span translate>Users have been imported.</span>
</div>
</div>
<div *ngIf="newCount">
<span translate>Click on 'import' (right top corner) to import the new users.
</span>
</div>
<mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()">
<mat-option value="all" translate> Show all </mat-option>
<mat-option value="error" translate> Show errors only </mat-option>
<mat-option value="noerror" translate> Show correct entries </mat-option>
</mat-select>
<div class="table-container">
<table mat-table class="on-transition-fade" [dataSource]="dataSource" matSort>
<!-- Status column -->
<ng-container matColumnDef="status" sticky>
<mat-header-cell *matHeaderCellDef class="first-column"></mat-header-cell>
<mat-cell *matCellDef="let entry" class="first-column">
<div *ngIf="entry.status === 'error'">
<mat-icon
class="red-warning-text"
matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
>
{{ getActionIcon(entry) }}
</mat-icon>
<mat-icon
color="warn"
*ngIf="hasError(entry, 'ParsingErrors')"
matTooltip="{{ getVerboseError('ParsingErrors') | translate }}"
>
warning
</mat-icon>
</div>
<div *ngIf="entry.status === 'new'">
<mat-icon matTooltip="{{ 'User will be imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.status === 'done'">
<mat-icon matTooltip="{{ 'User has been imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
</mat-cell>
</ng-container>
<!-- Title column -->
<ng-container matColumnDef="title">
<mat-header-cell *matHeaderCellDef translate>Title</mat-header-cell>
<mat-cell *matCellDef="let entry">
<span *ngIf="nameErrors(entry)">
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
warning
</mat-icon>
&nbsp;
</span>
{{ entry.newEntry.title }}
</mat-cell>
</ng-container>
<!-- title column -->
<ng-container matColumnDef="first_name">
<mat-header-cell *matHeaderCellDef translate>Given name</mat-header-cell>
<mat-cell *matCellDef="let entry">
<span *ngIf="nameErrors(entry)">
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
warning
</mat-icon>
&nbsp;
</span>
{{ entry.newEntry.first_name }}
</mat-cell>
</ng-container>
<ng-container matColumnDef="last_name">
<mat-header-cell *matHeaderCellDef translate>Surname</mat-header-cell>
<mat-cell *matCellDef="let entry">
<span *ngIf="nameErrors(entry)">
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
warning
</mat-icon>
&nbsp;
</span>
{{ entry.newEntry.last_name }}
</mat-cell>
</ng-container>
<ng-container matColumnDef="structure_level">
<mat-header-cell *matHeaderCellDef translate>Structure level</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.structure_level }} </mat-cell>
</ng-container>
<ng-container matColumnDef="participant_number">
<mat-header-cell *matHeaderCellDef translate>Participant number</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.user.number }} </mat-cell>
</ng-container>
<!-- groups column -->
<ng-container matColumnDef="groups_id">
<mat-header-cell *matHeaderCellDef translate>Groups</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newEntry.csvGroups.length">
<span *ngIf="hasError(entry, 'Groups')">
<mat-icon color="warn"matTooltip="{{ getVerboseError('Groups') | translate }}">
warning
</mat-icon>
</span>
<span *ngFor="let group of entry.newEntry.csvGroups">
{{ group.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!group.id">add</mat-icon>
&nbsp;
</span>
</div>
</mat-cell>
</ng-container>
<ng-container matColumnDef="comment">
<mat-header-cell *matHeaderCellDef translate>Comment</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.comment }} </mat-cell>
</ng-container>
<ng-container matColumnDef="is_active">
<mat-header-cell *matHeaderCellDef translate>Is Active</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_active"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="is_present">
<mat-header-cell *matHeaderCellDef translate>Is Present</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_present"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="is_committee">
<mat-header-cell *matHeaderCellDef translate>Is Committee</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_committee"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="default_password">
<mat-header-cell *matHeaderCellDef translate>Initial password</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.default_password }} </mat-cell>
</ng-container>
<ng-container matColumnDef="email">
<mat-header-cell *matHeaderCellDef translate>Email</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.email }} </mat-cell>
</ng-container>
<mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
<mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
</table>
</div>
</mat-card>

26
client/src/app/site/users/components/user-import/user-import-list.component.spec.ts Normal file
View File

@ -0,0 +1,26 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { UserImportListComponent } from './user-import-list.component';
import { E2EImportsModule } from 'e2e-imports.module';
describe('UserImportListComponent', () => {
let component: UserImportListComponent;
let fixture: ComponentFixture<UserImportListComponent>;
beforeEach(async(() => {
TestBed.configureTestingModule({
declarations: [UserImportListComponent],
imports: [E2EImportsModule]
}).compileComponents();
}));
beforeEach(() => {
fixture = TestBed.createComponent(UserImportListComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

103
client/src/app/site/users/components/user-import/user-import-list.component.ts Normal file
View File

@ -0,0 +1,103 @@
import { Component } from '@angular/core';
import { MatSnackBar } from '@angular/material';
import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core';
import { BaseImportListComponent } from 'app/site/base/base-import-list';
import { FileExportService } from 'app/core/services/file-export.service';
import { FormBuilder, FormGroup } from '@angular/forms';
import { NewEntry } from 'app/core/services/base-import.service';
import { UserImportService } from '../../services/user-import.service';
import { ViewUser } from '../../models/view-user';
/**
* Component for the user import list view.
*/
@Component({
selector: 'os-user-import-list',
templateUrl: './user-import-list.component.html'
})
export class UserImportListComponent extends BaseImportListComponent<ViewUser> {
public textAreaForm: FormGroup;
/**
* Constructor for list view bases
*
* @param titleService the title serivce
* @param matSnackBar snackbar for displaying errors
* @param formBuilder: FormBuilder for the textArea
* @param translate the translate service
* @param exporter: csv export service for dummy dat
* @param importer: The motion csv import service
*/
public constructor(
titleService: Title,
matSnackBar: MatSnackBar,
formBuilder: FormBuilder,
public translate: TranslateService,
private exporter: FileExportService,
importer: UserImportService
) {
super(importer, titleService, translate, matSnackBar);
this.textAreaForm = formBuilder.group({ inputtext: [''] });
}
/**
* Triggers an example csv download
*/
public downloadCsvExample(): void {
const headerRow = [
'Title',
'Given name',
'Surname',
'Structure level',
'Participant number',
'Groups',
'Comment',
'Is active',
'Is present',
'Is a committee',
'Initial password',
'Email'
]
.map(item => this.translate.instant(item))
.join(',');
const rows = [
headerRow,
'Dr.,Max,Mustermann,"Berlin",1234567890,"Delegates, Staff",xyz,1,1,,initialPassword,',
',John,Doe,Washington,75/99/8-2,Committees,"This is a comment, without doubt",1,1,,,john.doe@email.com',
',Fred,Bloggs,London,,,,,,,,',
',,Executive Board,,,,,,,1,,'
];
this.exporter.saveFile(rows.join('\n'), this.translate.instant('User example') + '.csv');
}
/**
* Shorthand for getVerboseError on name fields checking for duplicates and invalid fields
*
* @param row
* @returns an error string similar to getVerboseError
*/
public nameErrors(row: NewEntry<ViewUser>): string {
for (const name of ['NoName', 'Duplicates', 'DuplicateImport']) {
if (this.importer.hasError(row, name)) {
return this.importer.verbose(name);
}
}
return '';
}
/**
* Sends the data in the text field input area to the importer
*/
public parseTextArea(): void {
(this.importer as UserImportService).parseTextArea(this.textAreaForm.get('inputtext').value);
}
/**
* Triggers a change in the tab group: Clearing the preview selection
*/
public onTabChange(): void {
this.importer.clearPreview();
}
}

6
client/src/app/site/users/components/user-list/user-list.component.html
View File

@ -90,6 +90,12 @@
<mat-icon>archive</mat-icon>
<span translate>Export as CSV</span>
</button>
<button mat-menu-item *osPerms="'users.can_manage'" routerLink="import">
<mat-icon>save_alt</mat-icon>
<span translate>Import</span><span>&nbsp;...</span>
</button>
</div>
<div *ngIf="isMultiSelect">
<button mat-menu-item (click)="selectAll()">

9
client/src/app/site/users/components/user-list/user-list.component.ts
View File

@ -79,15 +79,6 @@ export class UserListComponent extends ListViewBaseComponent<ViewUser> implement
});
}
/**
* Navigate to import page or do it inline
*
* TODO: implement importing of users
*/
public import(): void {
console.log('click on Import');
}
/**
* Handles the click on a user row if not in multiSelect modus
* @param row selected row

73
client/src/app/site/users/models/view-csv-create-user.ts Normal file
View File

@ -0,0 +1,73 @@
import { ViewUser } from './view-user';
import { User } from 'app/shared/models/users/user';
/**
* Interface for correlating between strings representing BaseModels and existing
* BaseModels.
*/
export interface CsvMapping {
name: string;
id?: number;
multiId?: number[];
}
/**
* View class for a new User during text imports. Offers a mapping and matching
* to secondary import data (groups)
*
* @ignore
*/
export class ViewCsvCreateUser extends ViewUser {
/**
* Mapping for a new/existing groups.
*/
public csvGroups: CsvMapping[] = [];
public title: string;
/**
* Getter if the minimum requrements for a user are met: A name
*
* @returns false if the user has neither first nor last name
*/
public get isValid(): boolean {
if (this.user && (this.first_name || this.last_name)) {
return true;
}
return false;
}
public constructor(user?: User) {
super(user);
}
/**
* takes a list of solved group maps to update. Returns the amount of
* entries that remain unmatched
*
* @param groups
*/
public solveGroups(groups: CsvMapping[]): number {
let open = 0;
const ids: number[] = [];
this.csvGroups.forEach(group => {
if (group.id) {
ids.push(group.id);
return;
}
if (!groups.length) {
open += 1;
return;
}
const mapped = groups.find(newGroup => newGroup.name === group.name);
if (mapped) {
group.id = mapped.id;
ids.push(mapped.id);
} else {
open += 1;
}
});
this.user.groups_id = ids;
return open;
}
}

17
client/src/app/site/users/services/user-import.service.spec.ts Normal file
View File

@ -0,0 +1,17 @@
import { TestBed } from '@angular/core/testing';
import { E2EImportsModule } from 'e2e-imports.module';
import { UserImportService } from './user-import.service';
describe('UserImportService', () => {
beforeEach(() =>
TestBed.configureTestingModule({
imports: [E2EImportsModule]
})
);
it('should be created', () => {
const service: UserImportService = TestBed.get(UserImportService);
expect(service).toBeTruthy();
});
});

271
client/src/app/site/users/services/user-import.service.ts Normal file
View File

@ -0,0 +1,271 @@
import { Injectable } from '@angular/core';
import { MatSnackBar } from '@angular/material';
import { Papa } from 'ngx-papaparse';
import { BaseImportService, NewEntry } from 'app/core/services/base-import.service';
import { Group } from 'app/shared/models/users/group';
import { GroupRepositoryService } from './group-repository.service';
import { TranslateService } from '@ngx-translate/core';
import { User } from 'app/shared/models/users/user';
import { UserRepositoryService } from './user-repository.service';
import { ViewCsvCreateUser, CsvMapping } from '../models/view-csv-create-user';
import { ViewUser } from '../models/view-user';
@Injectable({
providedIn: 'root'
})
export class UserImportService extends BaseImportService<ViewUser> {
/**
* Helper for mapping the expected header in a typesafe way. Values and order
* will be passed to {@link expectedHeader}
*/
public headerMap: (keyof ViewCsvCreateUser)[] = [
'title',
'first_name',
'last_name',
'structure_level',
'participant_number',
'groups_id',
'comment',
'is_active',
'is_present',
'is_committee',
'default_password',
'email'
];
/**
* The minimimal number of header entries needed to successfully create an entry
*/
public requiredHeaderLength = 3;
/**
* List of possible errors and their verbose explanation
*/
public errorList = {
Group: 'Group cannot be resolved',
Duplicates: 'This user already exists',
NoName: 'Entry has no valid name',
DuplicateImport: 'Entry cannot be imported twice. This line will be ommitted',
ParsingErrors: 'Some csv values could not be read correctly.'
};
/**
* Storage for tracking new groups to be created prior to importing users
*/
public newGroups: CsvMapping[];
/**
* Constructor. Calls parent and sets the expected header
*
* @param repo The User repository
* @param groupRepo the Group repository
* @param translate TranslationService
* @param papa csvParser
* @param matSnackbar MatSnackBar for displaying error messages
*/
public constructor(
private repo: UserRepositoryService,
private groupRepo: GroupRepositoryService,
translate: TranslateService,
papa: Papa,
matSnackbar: MatSnackBar
) {
super(translate, papa, matSnackbar);
this.expectedHeader = this.headerMap;
}
/**
* Clears all temporary data specific to this importer
*/
public clearData(): void {
this.newGroups = [];
}
/**
* Parses a string representing an entry, extracting secondary data, appending
* the array of secondary imports as needed
*
* @param line
* @returns a new entry representing an User
*/
public mapData(line: string): NewEntry<ViewUser> {
const newViewUser = new ViewCsvCreateUser(new User());
const headerLength = Math.min(this.expectedHeader.length, line.length);
let hasErrors = false;
for (let idx = 0; idx < headerLength; idx++) {
switch (this.expectedHeader[idx]) {
case 'groups_id':
newViewUser.csvGroups = this.getGroups(line[idx]);
break;
case 'is_active':
case 'is_committee':
case 'is_present':
try {
newViewUser.user[this.expectedHeader[idx]] = this.toBoolean(line[idx]);
} catch (e) {
if (e instanceof TypeError) {
console.log(e);
hasErrors = true;
continue;
}
}
break;
case 'participant_number':
newViewUser.user.number = line[idx];
break;
default:
newViewUser.user[this.expectedHeader[idx]] = line[idx];
break;
}
}
const newEntry = this.userToEntry(newViewUser);
if (hasErrors) {
this.setError(newEntry, 'ParsingErrors');
}
return newEntry;
}
/**
* Executing the import. Creates all secondary data, maps the newly created
* secondary data to the new entries, then creates all entries without errors
* by submitting them to the server. The entries will receive the status
* 'done' on success.
*/
public async doImport(): Promise<void> {
this.newGroups = await this.createNewGroups();
for (const entry of this.entries) {
if (entry.status !== 'new') {
continue;
}
const openBlocks = (entry.newEntry as ViewCsvCreateUser).solveGroups(this.newGroups);
if (openBlocks) {
this.setError(entry, 'Group');
this.updatePreview();
continue;
}
await this.repo.create(entry.newEntry.user);
entry.status = 'done';
}
this.updatePreview();
}
/**
* extracts the group(s) from a csv column and tries to match them against existing groups,
* appending to {@link newGroups} if needed.
* Also checks for groups matching the translation between english and the language currently set
*
* @param groupString string from an entry line including one or more comma separated groups
* @returns a mapping with (if existing) ids to the group names
*/
private getGroups(groupString: string): CsvMapping[] {
const result: CsvMapping[] = [];
if (!groupString) {
return [];
}
groupString.trim();
const groupArray = groupString.split(',');
for (const item of groupArray) {
const newGroup = item.trim();
let existingGroup = this.groupRepo.getViewModelList().filter(grp => grp.name === newGroup);
if (!existingGroup.length) {
existingGroup = this.groupRepo
.getViewModelList()
.filter(grp => this.translate.instant(grp.name) === newGroup);
}
if (!existingGroup.length) {
if (!this.newGroups.find(listedGrp => listedGrp.name === newGroup)) {
this.newGroups.push({ name: newGroup });
}
result.push({ name: newGroup });
} else if (existingGroup.length === 1) {
result.push({
name: existingGroup[0].name,
id: existingGroup[0].id
});
}
}
return result;
}
/**
* Handles the creation of new groups collected in {@link newGroups}.
*
* @returns The group mapping with (on success) new ids
*/
private async createNewGroups(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
for (const group of this.newGroups) {
promises.push(
this.groupRepo.create(new Group({ name: group.name })).then(identifiable => {
return { name: group.name, id: identifiable.id };
})
);
}
return await Promise.all(promises);
}
/**
* translates a string into a boolean
*
* @param data
* @returns a boolean from the string
*/
private toBoolean(data: string): Boolean {
if (!data || data === '0' || data === 'false') {
return false;
} else if (data === '1' || data === 'true') {
return true;
} else {
throw new TypeError('Value cannot be translated into boolean: ' + data);
}
}
/**
* parses the data given by the textArea. Expects user names separated by lines.
* Comma separated values will be read as Surname(s), given name(s) (lastCommaFirst)
*
* @param data a string as produced by textArea input
*/
public parseTextArea(data: string): void {
const newEntries: NewEntry<ViewUser>[] = [];
this.clearData();
this.clearPreview();
const lines = data.split('\n');
lines.forEach(line => {
if (!line.length) {
return;
}
const nameSchema = line.includes(',') ? 'lastCommaFirst' : 'firstSpaceLast';
const newUser = new ViewCsvCreateUser(this.repo.parseUserString(line, nameSchema));
const newEntry = this.userToEntry(newUser);
newEntries.push(newEntry);
});
this.setParsedEntries(newEntries);
}
/**
* Checks a newly created ViewCsvCreateuser for validity and duplicates,
*
* @param newUser
* @returns a NewEntry with duplicate/error information
*/
private userToEntry(newUser: ViewCsvCreateUser): NewEntry<ViewUser> {
const newEntry: NewEntry<ViewUser> = {
newEntry: newUser,
duplicates: [],
status: 'new',
errors: []
};
if (newUser.isValid) {
const updateModels = this.repo.getUserDuplicates(newUser);
if (updateModels.length) {
newEntry.duplicates = updateModels;
this.setError(newEntry, 'Duplicates');
}
} else {
this.setError(newEntry, 'NoName');
}
return newEntry;
}
}

73
client/src/app/site/users/services/user-repository.service.ts
View File

@ -13,6 +13,12 @@ import { HttpService } from 'app/core/services/http.service';
import { TranslateService } from '@ngx-translate/core';
import { environment } from '../../../../environments/environment';
/**
* type for determining the user name from a string during import.
* See {@link parseUserString} for implementations
*/
type StringNamingSchema = 'lastCommaFirst' | 'firstSpaceLast';
/**
* Repository service for users
*
@ -222,24 +228,63 @@ export class UserRepositoryService extends BaseRepository<ViewUser, User> {
/**
* Creates a new User from a string
*
* @param user: String to create the user from
* TODO: return 'user' + new id
* @returns Promise with a created user id and the raw name used as input
*/
public async createFromString(user: string): Promise<{ id: number; name: string }> {
const splitUser = user.split(' ');
const newUser: Partial<User> = {};
switch (splitUser.length) {
case 1:
newUser.first_name = splitUser[0];
break;
case 2:
newUser.first_name = splitUser[0];
newUser.last_name = splitUser[1];
break;
default:
newUser.first_name = user;
}
const newUser = this.parseUserString(user);
const createdUser = await this.create(newUser);
return { id: createdUser.id, name: user };
}
/**
* Tries to convert a user string into an user. If it is two words, expect
* a first and a last name, if one word only, expect a first name only.
* If more than two words, they will all be put as the first name
* TODO: More advanced logic to fit names
*
* @param inputUser A raw user string
* @param schema optional hint on how to handle the strings. TODO: Not fully implemented.
* @returns A User object (not uploaded to the server)
*/
public parseUserString(inputUser: string, schema?: StringNamingSchema): User {
const newUser: Partial<User> = {};
if (schema === 'lastCommaFirst') {
const commaSeparated = inputUser.split(',');
switch (commaSeparated.length) {
case 1:
newUser.first_name = commaSeparated[0];
break;
case 2:
newUser.last_name = commaSeparated[0];
newUser.first_name = commaSeparated[1];
break;
default:
newUser.first_name = inputUser;
}
} else if (!schema || schema === 'firstSpaceLast') {
const splitUser = inputUser.split(' ');
switch (splitUser.length) {
case 1:
newUser.first_name = splitUser[0];
break;
case 2:
newUser.first_name = splitUser[0];
newUser.last_name = splitUser[1];
break;
default:
newUser.first_name = inputUser;
}
}
return new User(newUser);
}
/**
* Returns all duplicates of an user (currently: full name matches)
* @param user
*/
public getUserDuplicates(user: ViewUser): ViewUser[] {
return this.getViewModelList().filter(existingUser => existingUser.full_name === user.full_name);
}
}

10
client/src/app/site/users/users-routing.module.ts
View File

@ -1,9 +1,11 @@
import { NgModule } from '@angular/core';
import { Routes, RouterModule } from '@angular/router';
import { UserListComponent } from './components/user-list/user-list.component';
import { UserDetailComponent } from './components/user-detail/user-detail.component';
import { GroupListComponent } from './components/group-list/group-list.component';
import { PasswordComponent } from './components/password/password.component';
import { UserDetailComponent } from './components/user-detail/user-detail.component';
import { UserImportListComponent } from './components/user-import/user-import-list.component';
import { UserListComponent } from './components/user-list/user-list.component';
const routes: Routes = [
{
@ -22,6 +24,10 @@ const routes: Routes = [
path: 'new',
component: UserDetailComponent
},
{
path: 'import',
component: UserImportListComponent
},
{
path: 'groups',
component: GroupListComponent

17
client/src/app/site/users/users.module.ts
View File

@ -1,15 +1,22 @@
import { NgModule } from '@angular/core';
import { CommonModule } from '@angular/common';
import { UsersRoutingModule } from './users-routing.module';
import { SharedModule } from '../../shared/shared.module';
import { UserListComponent } from './components/user-list/user-list.component';
import { UserDetailComponent } from './components/user-detail/user-detail.component';
import { GroupListComponent } from './components/group-list/group-list.component';
import { PasswordComponent } from './components/password/password.component';
import { SharedModule } from '../../shared/shared.module';
import { UserDetailComponent } from './components/user-detail/user-detail.component';
import { UserImportListComponent } from './components/user-import/user-import-list.component';
import { UserListComponent } from './components/user-list/user-list.component';
import { UsersRoutingModule } from './users-routing.module';
@NgModule({
imports: [CommonModule, UsersRoutingModule, SharedModule],
declarations: [UserListComponent, UserDetailComponent, GroupListComponent, PasswordComponent]
declarations: [
UserListComponent,
UserDetailComponent,
GroupListComponent,
PasswordComponent,
UserImportListComponent
]
})
export class UsersModule {}

135
client/src/styles.scss
View File

@ -16,7 +16,6 @@
@import '~angular-tree-component/dist/angular-tree-component.css';
* {
font-family: Fira Sans, Roboto, Arial, Helvetica, sans-serif;
}
@ -72,14 +71,15 @@ img {
a {
text-decoration: none;
color: #039BE5; /*TODO: move to theme*/
color: #039be5; /*TODO: move to theme*/
&:hover {
text-decoration: underline;
}
}
strong, b {
strong,
b {
font-weight: 500;
}
@ -113,8 +113,9 @@ strong, b {
color: mat-color($openslides-accent);
}
.green-text { // TODO better name/theming
color: #5A5;
.green-text {
// TODO better name/theming
color: #5a5;
}
.icon-text-distance {
@ -126,7 +127,7 @@ strong, b {
color: gray;
}
mat-card {
box-shadow: 0 1px 4px 0 rgba(0,0,0,0.37) !important;
box-shadow: 0 1px 4px 0 rgba(0, 0, 0, 0.37) !important;
}
.os-card {
max-width: 770px;
@ -152,7 +153,6 @@ mat-card {
}
}
// Shared table definitions
%os-table {
width: 100%;
@ -175,8 +175,14 @@ mat-card {
}
@keyframes fadeIn {
0% {width:0%; margin-left:0;}
100% {width:100%;margin-left:-100%;}
0% {
width: 0%;
margin-left: 0;
}
100% {
width: 100%;
margin-left: -100%;
}
}
//custom table header for search button, filtering and more. Used in ListViews
@ -198,7 +204,7 @@ mat-card {
position: relative;
max-width: 400px;
z-index: 2;
background-color: #EEE;
background-color: #eee;
padding-right: 5px;
margin-right: 0px;
}
@ -338,7 +344,6 @@ button.mat-menu-item.selected {
font-size: 18px;
}
/** helper classes for margin/padding */
.spacer-top-10 {
margin-top: 10px;
@ -366,10 +371,10 @@ button.mat-menu-item.selected {
line-height: 24px;
.mat-icon {
font-size: 24px;
width: 24px;
height: 24px;
line-height: 24px;
font-size: 24px;
width: 24px;
height: 24px;
line-height: 24px;
}
}
.title-left {
@ -381,13 +386,11 @@ button.mat-menu-item.selected {
padding-right: 20px;
}
/** more helper classes **/
.center {
text-align: center;
}
/** Colors **/
.lightblue {
background-color: rgb(33, 150, 243) !important;
@ -427,28 +430,92 @@ button.mat-menu-item.selected {
color: rgba(0, 0, 0, 0.87) !important;
}
/* TODO: move to site.component.scss-theme.scss (does not work currently) */
/* make the .user-menu expansion panel look like the nav-toolbar above */
.user-menu {
min-height: 48px;
/* make the .user-menu expansion panel look like the nav-toolbar above */
.user-menu {
min-height: 48px;
.mat-expansion-panel-header {
padding: 0 16px !important;
}
.mat-expansion-panel-body {
padding: 0 !important;
}
.mat-expansion-panel-header {
padding: 0 16px !important;
}
.mat-drawer-inner-container {
scrollbar-width: none; /* hide scrollbars in Firefox */
-ms-overflow-style: -ms-autohiding-scrollbar; /* hide scrollbars in Edge until the scrollable area is hovered */
.mat-expansion-panel-body {
padding: 0 !important;
}
}
.mat-drawer-inner-container {
scrollbar-width: none; /* hide scrollbars in Firefox */
-ms-overflow-style: -ms-autohiding-scrollbar; /* hide scrollbars in Edge until the scrollable area is hovered */
}
.mat-drawer-inner-container::-webkit-scrollbar {
display: none !important; /* hide scrollbars in webkit browsers */
}
.import-table {
.table-container {
width: 100%;
overflow-x: scroll;
margin-top: 5px;
}
.mat-drawer-inner-container::-webkit-scrollbar {
display: none !important; /* hide scrollbars in webkit browsers */
table {
width: 100%;
overflow: scroll;
}
.mat-header-cell {
min-width: 100px;
flex: 2;
padding-right: 8px;
}
.mat-cell {
min-width: 100px;
flex: 2;
padding-top: 2px;
padding-right: 8px;
}
.selection {
min-width: 80px;
}
.import-done {
background-color: #cfc;
}
.import-error {
background-color: #fcc;
}
.code {
padding-left: 1em;
font-style: italic;
}
div.wrapper {
display: flex;
vertical-align: bottom;
padding: 10px;
}
div.summary {
display: flex;
}
.hidden-input {
display: none;
}
.newBadge {
margin-left: -3px;
}
.first-column {
flex: 1;
min-width: 0px;
}
.filter-imports {
max-width: 50%;
}
}