From c52fdaae6c5b4bdf5552169039ca509ea995d511 Mon Sep 17 00:00:00 2001 From: Maximilian Krambach Date: Fri, 11 Jan 2019 18:55:09 +0100 Subject: [PATCH] csv-import for user and agenda(topics) --- .../app/core/services/base-import.service.ts | 395 ++++++++++++++++ .../core/services/duration.service.spec.ts | 0 .../core/services/duration.service.ts | 0 client/src/app/shared/models/users/user.ts | 25 +- client/src/app/shared/shared.module.ts | 3 + .../site/agenda/agenda-import.service.spec.ts | 17 + .../app/site/agenda/agenda-import.service.ts | 216 +++++++++ .../app/site/agenda/agenda-routing.module.ts | 3 + client/src/app/site/agenda/agenda.module.ts | 7 +- .../agenda-import-list.component.html | 226 +++++++++ .../agenda-import-list.component.spec.ts | 26 ++ .../agenda-import-list.component.ts | 156 +++++++ .../agenda-list/agenda-list.component.html | 4 + .../agenda-list/agenda-list.component.ts | 4 +- .../item-info-dialog.component.ts | 2 +- .../topic-detail/topic-detail.component.ts | 3 +- .../app/site/agenda/models/create-topic.ts | 17 + .../site/agenda/models/view-create-topic.ts | 110 +++++ .../src/app/site/agenda/models/view-topic.ts | 2 +- .../services/topic-repository.service.ts | 19 +- client/src/app/site/base/base-import-list.ts | 276 +++++++++++ .../motion-import-list.component.html | 80 ++-- .../motion-import-list.component.scss | 53 --- .../motion-import-list.component.ts | 244 +--------- .../motions/models/view-csv-create-motion.ts | 65 +-- .../motions/services/motion-import.service.ts | 438 ++++-------------- .../services/motion-repository.service.ts | 7 +- .../user-import-list.component.html | 283 +++++++++++ .../user-import-list.component.spec.ts | 26 ++ .../user-import/user-import-list.component.ts | 103 ++++ .../user-list/user-list.component.html | 6 + .../user-list/user-list.component.ts | 9 - .../site/users/models/view-csv-create-user.ts | 73 +++ .../services/user-import.service.spec.ts | 17 + .../users/services/user-import.service.ts | 271 +++++++++++ .../users/services/user-repository.service.ts | 73 ++- .../app/site/users/users-routing.module.ts | 10 +- client/src/app/site/users/users.module.ts | 17 +- client/src/styles.scss | 135 ++++-- 39 files changed, 2602 insertions(+), 819 deletions(-) create mode 100644 client/src/app/core/services/base-import.service.ts rename client/src/app/{site => }/core/services/duration.service.spec.ts (100%) rename client/src/app/{site => }/core/services/duration.service.ts (100%) create mode 100644 client/src/app/site/agenda/agenda-import.service.spec.ts create mode 100644 client/src/app/site/agenda/agenda-import.service.ts create mode 100644 client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.html create mode 100644 client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.spec.ts create mode 100644 client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.ts create mode 100644 client/src/app/site/agenda/models/create-topic.ts create mode 100644 client/src/app/site/agenda/models/view-create-topic.ts create mode 100644 client/src/app/site/base/base-import-list.ts delete mode 100644 client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss create mode 100644 client/src/app/site/users/components/user-import/user-import-list.component.html create mode 100644 client/src/app/site/users/components/user-import/user-import-list.component.spec.ts create mode 100644 client/src/app/site/users/components/user-import/user-import-list.component.ts create mode 100644 client/src/app/site/users/models/view-csv-create-user.ts create mode 100644 client/src/app/site/users/services/user-import.service.spec.ts create mode 100644 client/src/app/site/users/services/user-import.service.ts diff --git a/client/src/app/core/services/base-import.service.ts b/client/src/app/core/services/base-import.service.ts new file mode 100644 index 000000000..d223bdad6 --- /dev/null +++ b/client/src/app/core/services/base-import.service.ts @@ -0,0 +1,395 @@ +import { BehaviorSubject, Observable } from 'rxjs'; +import { Injectable, EventEmitter } from '@angular/core'; +import { MatSnackBar } from '@angular/material'; +import { Papa, PapaParseConfig } from 'ngx-papaparse'; +import { TranslateService } from '@ngx-translate/core'; + +import { BaseViewModel } from 'app/site/base/base-view-model'; + +/** + * Interface for value- Label combinations. + * Map objects didn't work, TODO: Use map objects (needs iterating through all objects of a map) + */ +export interface ValueLabelCombination { + value: string; + label: string; +} + +/** + * Interface matching a newly created entry with their duplicates and an import status + */ +export interface NewEntry { + newEntry: V; + status: CsvImportStatus; + errors: string[]; + duplicates: V[]; +} + +/** + * interface for a preview summary + */ +export interface ImportCSVPreview { + total: number; + duplicates: number; + errors: number; + new: number; + done: number; +} + +/** + * The permitted states of a new entry. Only a 'new' entry should be imported + * and then be set to 'done'. + */ +type CsvImportStatus = 'new' | 'error' | 'done'; + +/** + * Abstract service for imports + */ +@Injectable({ + providedIn: 'root' +}) +export abstract class BaseImportService { + /** + * List of possible errors and their verbose explanation + */ + public abstract errorList: Object; + + /** + * The headers expected in the CSV matching import properties (in order) + */ + public expectedHeader: (string)[]; + + /** + * The minimimal number of header entries needed to successfully create an entry + */ + public abstract requiredHeaderLength: number; + + /** + * The last parsed file object (may be reparsed with new encoding, thus kept in memory) + */ + private _rawFile: File; + + /** + * The used column separator. If left on an empty string (default), + * the papaparse parser will automatically decide on separators. + */ + public columnSeparator = ''; + + /** + * The used text separator. + */ + public textSeparator = '"'; + + /** + * The encoding used by the FileReader object. + */ + public encoding = 'utf-8'; + + /** + * List of possible encodings and their label. values should be values accepted + * by the FileReader API + */ + public encodings: ValueLabelCombination[] = [ + { value: 'utf-8', label: 'UTF 8 - Unicode' }, + { value: 'iso-8859-1', label: 'ISO 8859-1 - West European' }, + { value: 'iso-8859-15', label: 'ISO 8859-15 - West European (with €)' } + ]; + + /** + * List of possible column separators to pass on to papaParse + */ + public columnSeparators: ValueLabelCombination[] = [ + { label: 'Comma', value: ',' }, + { label: 'Semicolon', value: ';' }, + { label: 'Automatic', value: '' } + ]; + + /** + * List of possible text separators to pass on to papaParse. Note that + * it cannot automatically detect textseparators (value must not be an empty string) + */ + public textSeparators: ValueLabelCombination[] = [ + { label: 'Double quotes (")', value: '"' }, + { label: "Single quotes (')", value: "'" } + ]; + + /** + * FileReader object for file import + */ + private reader = new FileReader(); + + /** + * the list of parsed models that have been extracted from the opened file + */ + private _entries: NewEntry[] = []; + + /** + * BehaviorSubject for displaying a preview for the currently selected entries + */ + public newEntries = new BehaviorSubject[]>([]); + + /** + * Emits an error string to display if a file import cannot be done + */ + public errorEvent = new EventEmitter(); + + /** + * storing the summary preview for the import, to avoid recalculating it + * at each display change. + */ + protected _preview: ImportCSVPreview; + + /** + * Returns a summary on actions that will be taken/not taken. + */ + public get summary(): ImportCSVPreview { + if (!this._preview) { + this.updatePreview(); + } + return this._preview; + } + + /** + * Returns the current entries. For internal use in extending classes, as it + * might not be filled with data at all times (see {@link newEntries} for a BehaviorSubject) + */ + protected get entries(): NewEntry[] { + return this._entries; + } + + /** + * Constructor. Creates a fileReader to subscribe to it for incoming parsed + * strings + * + * @param translate Translation service + * @param papa External csv parser (ngx-papaparser) + * @param matSnackBar snackBar to display import errors + */ + public constructor(protected translate: TranslateService, private papa: Papa, protected matSnackbar: MatSnackBar) { + this.reader.onload = (event: any) => { + // TODO type: event is a progressEvent, + // but has a property target.result, which typescript doesn't recognize + this.parseInput(event.target.result); + }; + } + + /** + * Clears all stored secondary data + * TODO: Merge with clearPreview() + */ + public abstract clearData(): void; + + /** + * Parses the data input. Expects a string as returned by via a + * File.readAsText() operation + * + * @param file + */ + public parseInput(file: string): void { + this.clearData(); + this.clearPreview(); + const papaConfig: PapaParseConfig = { + header: false, + skipEmptyLines: true, + quoteChar: this.textSeparator + }; + if (this.columnSeparator) { + papaConfig.delimiter = this.columnSeparator; + } + const entryLines = this.papa.parse(file, papaConfig).data; + const valid = this.checkHeader(entryLines.shift()); + if (!valid) { + return; + } + entryLines.forEach(line => { + const item = this.mapData(line); + if (item) { + this._entries.push(item); + } + }); + this.newEntries.next(this._entries); + this.updatePreview(); + } + + /** + * parses pre-prepared entries (e.g. from a textarea) instead of a csv structure + * + * @param entries: an array of prepared newEntry objects + */ + public setParsedEntries(entries: NewEntry[]): void { + this.clearData(); + this.clearPreview(); + if (!entries) { + return; + } + this._entries = entries; + this.newEntries.next(this._entries); + this.updatePreview(); + } + + /** + * Parsing an string representing an entry, extracting secondary data, + * returning a new entry object + * @param line a line extracted by the CSV (not including the header) + */ + public abstract mapData(line: string): NewEntry; + + /** + * Trigger for executing the import. + */ + public abstract async doImport(): Promise; + + /** + * counts the amount of duplicates that have no decision on the action to + * be taken + */ + public updatePreview(): void { + const summary = { + total: 0, + new: 0, + duplicates: 0, + errors: 0, + done: 0 + }; + this._entries.forEach(entry => { + summary.total += 1; + if (entry.status === 'done') { + summary.done += 1; + return; + } else if (entry.status === 'error' && !entry.duplicates.length) { + // errors that are not due to duplicates + summary.errors += 1; + return; + } else if (entry.duplicates.length) { + summary.duplicates += 1; + return; + } else if (entry.status === 'new') { + summary.new += 1; + } + }); + this._preview = summary; + } + + /** + * a subscribable representation of the new items to be imported + * + * @returns an observable BehaviorSubject + */ + public getNewEntries(): Observable[]> { + return this.newEntries.asObservable(); + } + + /** + * Handler after a file was selected. Basic checking for type, then hand + * over to parsing + * + * @param event type is Event, but has target.files, which typescript doesn't seem to recognize + */ + public onSelectFile(event: any): void { + // TODO type + if (event.target.files && event.target.files.length === 1) { + if (event.target.files[0].type === 'text/csv') { + this._rawFile = event.target.files[0]; + this.readFile(event.target.files[0]); + } else { + this.matSnackbar.open(this.translate.instant('Wrong file type detected. Import failed.'), '', { + duration: 3000 + }); + this.clearPreview(); + this._rawFile = null; + } + } + } + + /** + * Rereads the (previously selected) file, if present. Thought to be triggered + * by parameter changes on encoding, column, text separators + */ + public refreshFile(): void { + if (this._rawFile) { + this.readFile(this._rawFile); + } + } + + /** + * (re)-reads a given file with the current parameter + */ + private readFile(file: File): void { + this.reader.readAsText(file, this.encoding); + } + + /** + * Checks the first line of the csv (the header) for consistency (length) + * + * @param row expected to be an array parsed from the first line of a csv file + * @returns true if the line has at least the minimum amount of columns + */ + private checkHeader(row: string[]): boolean { + const snackbarDuration = 3000; + if (row.length < this.requiredHeaderLength) { + this.matSnackbar.open(this.translate.instant('The file has too few columns to be parsed properly.'), '', { + duration: snackbarDuration + }); + + this.clearPreview(); + return false; + } else if (row.length < this.expectedHeader.length) { + this.matSnackbar.open( + this.translate.instant('The file seems to have some ommitted columns. They will be considered empty.'), + '', + { duration: snackbarDuration } + ); + } else if (row.length > this.expectedHeader.length) { + this.matSnackbar.open( + this.translate.instant('The file seems to have additional columns. They will be ignored.'), + '', + { duration: snackbarDuration } + ); + } + return true; + } + + /** + * Resets the data and preview (triggered upon selecting an invalid file) + */ + public clearPreview(): void { + this._entries = []; + this.newEntries.next([]); + this._preview = null; + } + + /** + * set a list of short names for error, indicating which column failed + */ + public setError(entry: NewEntry, error: string): void { + if (this.errorList.hasOwnProperty(error)) { + if (!entry.errors) { + entry.errors = [error]; + } else if (!entry.errors.includes(error)) { + entry.errors.push(error); + entry.status = 'error'; + } + } + } + + /** + * Get an extended error description. + * + * @param error + * @returns the extended error desription for that error + */ + public verbose(error: string): string { + return this.errorList[error]; + } + + /** + * Queries if a given error is present in the given entry + * + * @param entry the entry to check for the error. + * @param error The error to check for + * @returns true if the error is present + */ + public hasError(entry: NewEntry, error: string): boolean { + return entry.errors.includes(error); + } +} diff --git a/client/src/app/site/core/services/duration.service.spec.ts b/client/src/app/core/services/duration.service.spec.ts similarity index 100% rename from client/src/app/site/core/services/duration.service.spec.ts rename to client/src/app/core/services/duration.service.spec.ts diff --git a/client/src/app/site/core/services/duration.service.ts b/client/src/app/core/services/duration.service.ts similarity index 100% rename from client/src/app/site/core/services/duration.service.ts rename to client/src/app/core/services/duration.service.ts diff --git a/client/src/app/shared/models/users/user.ts b/client/src/app/shared/models/users/user.ts index 6c64b2abc..9331dda0a 100644 --- a/client/src/app/shared/models/users/user.ts +++ b/client/src/app/shared/models/users/user.ts @@ -33,12 +33,12 @@ export class User extends ProjectableBaseModel implements Searchable { const addition: string[] = []; // addition: add number and structure level - const structure_level = this.structure_level.trim(); + const structure_level = this.structure_level ? this.structure_level.trim() : ''; if (structure_level) { addition.push(structure_level); } - const number = this.number.trim(); + const number = this.number ? this.number.trim() : null; if (number) { // TODO Translate addition.push('No.' + ' ' + number); @@ -54,12 +54,17 @@ export class User extends ProjectableBaseModel implements Searchable { return this.groups_id.some(groupId => groupId === id); } - // TODO read config values for "users_sort_by" + // TODO read config values for "users_sort_by" + + /** + * Getter for the short name (Title, given name, surname) + * + * @returns a non-empty string + */ public get short_name(): string { - const title = this.title.trim(); - const firstName = this.first_name.trim(); - const lastName = this.last_name.trim(); - let shortName = ''; + const title = this.title ? this.title.trim() : ''; + const firstName = this.first_name ? this.first_name.trim() : ''; + const lastName = this.last_name ? this.last_name.trim() : ''; // TODO need DS adjustment first first // if (this.DS.getConfig('users_sort_by').value === 'last_name') { @@ -70,9 +75,9 @@ export class User extends ProjectableBaseModel implements Searchable { // } // } - shortName += `${firstName} ${lastName}`; + let shortName = `${firstName} ${lastName}`; - if (shortName.trim() === '') { + if (!shortName) { shortName = this.username; } @@ -80,7 +85,7 @@ export class User extends ProjectableBaseModel implements Searchable { shortName = `${title} ${shortName}`; } - return shortName.trim(); + return shortName; } public getTitle(): string { diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index 71c17ac88..beeabf7ca 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts @@ -25,6 +25,7 @@ import { MatButtonToggleModule, MatBadgeModule, MatStepperModule, + MatTabsModule, MatBottomSheetModule } from '@angular/material'; import { MatAutocompleteModule } from '@angular/material/autocomplete'; @@ -118,6 +119,7 @@ import { LogoComponent } from './components/logo/logo.component'; MatRadioModule, MatButtonToggleModule, MatStepperModule, + MatTabsModule, DragDropModule, TranslateModule.forChild(), RouterModule, @@ -151,6 +153,7 @@ import { LogoComponent } from './components/logo/logo.component'; MatSnackBarModule, MatChipsModule, MatTooltipModule, + MatTabsModule, MatBadgeModule, MatIconModule, MatRadioModule, diff --git a/client/src/app/site/agenda/agenda-import.service.spec.ts b/client/src/app/site/agenda/agenda-import.service.spec.ts new file mode 100644 index 000000000..38cd89ada --- /dev/null +++ b/client/src/app/site/agenda/agenda-import.service.spec.ts @@ -0,0 +1,17 @@ +import { TestBed } from '@angular/core/testing'; + +import { AgendaImportService } from './agenda-import.service'; +import { E2EImportsModule } from 'e2e-imports.module'; + +describe('AgendaImportService', () => { + beforeEach(() => + TestBed.configureTestingModule({ + imports: [E2EImportsModule] + }) + ); + + it('should be created', () => { + const service: AgendaImportService = TestBed.get(AgendaImportService); + expect(service).toBeTruthy(); + }); +}); diff --git a/client/src/app/site/agenda/agenda-import.service.ts b/client/src/app/site/agenda/agenda-import.service.ts new file mode 100644 index 000000000..e05833a8e --- /dev/null +++ b/client/src/app/site/agenda/agenda-import.service.ts @@ -0,0 +1,216 @@ +import { Injectable } from '@angular/core'; +import { MatSnackBar } from '@angular/material'; +import { Papa } from 'ngx-papaparse'; +import { TranslateService } from '@ngx-translate/core'; + +import { BaseImportService, NewEntry } from 'app/core/services/base-import.service'; +import { CreateTopic } from './models/create-topic'; +import { DurationService } from 'app/core/services/duration.service'; +import { itemVisibilityChoices } from 'app/shared/models/agenda/item'; +import { TopicRepositoryService } from './services/topic-repository.service'; +import { ViewCreateTopic } from './models/view-create-topic'; + +@Injectable({ + providedIn: 'root' +}) +export class AgendaImportService extends BaseImportService { + /** + * Helper for mapping the expected header in a typesafe way. Values will be passed to + * {@link expectedHeader} + */ + public headerMap: (keyof ViewCreateTopic)[] = ['title', 'text', 'duration', 'comment', 'type']; + + /** + * The minimimal number of header entries needed to successfully create an entry + */ + public requiredHeaderLength = 1; + + /** + * List of possible errors and their verbose explanation + */ + public errorList = { + NoTitle: 'A Topic needs a title', + Duplicates: 'A topic tiwh this title already exists', + ParsingErrors: 'Some csv values could not be read correctly.' + }; + + /** + * Constructor. Calls the abstract class and sets the expected header + * + * @param durationService: a service for converting time strings and numbers + * @param repo: The Agenda repository service + * @param translate A translation service for translating strings + * @param papa Csv parser + * @param matSnackBar MatSnackBar for displaying errors + */ + public constructor( + private durationService: DurationService, + private repo: TopicRepositoryService, + translate: TranslateService, + papa: Papa, + matSnackBar: MatSnackBar + ) { + super(translate, papa, matSnackBar); + this.expectedHeader = this.headerMap; + } + + /** + * Clear all secondary import data. As agenda items have no secondary imports, + * this is an empty function + */ + public clearData(): void {} + + /** + * Parses a string representing an entry + * + * @param line a line extracted by the CSV (without the header) + * @returns a new entry for a Topic + */ + public mapData(line: string): NewEntry { + const newEntry = new ViewCreateTopic(new CreateTopic()); + const headerLength = Math.min(this.expectedHeader.length, line.length); + let hasErrors = false; + for (let idx = 0; idx < headerLength; idx++) { + switch (this.expectedHeader[idx]) { + case 'duration': + try { + const duration = this.parseDuration(line[idx]); + if (duration > 0) { + newEntry.duration = duration; + } + } catch (e) { + if (e instanceof TypeError) { + hasErrors = true; + continue; + } + } + break; + case 'type': + try { + newEntry.type = this.parseType(line[idx]); + } catch (e) { + if (e instanceof TypeError) { + hasErrors = true; + continue; + } + } + break; + default: + newEntry[this.expectedHeader[idx]] = line[idx]; + } + } + const updateModels = this.repo.getTopicDuplicates(newEntry) as ViewCreateTopic[]; + + // set type to 'public' if none is given in import + if (!newEntry.type) { + newEntry.type = 1; + } + const mappedEntry: NewEntry = { + newEntry: newEntry, + duplicates: [], + status: 'new', + errors: [] + }; + if (updateModels.length) { + mappedEntry.duplicates = updateModels; + this.setError(mappedEntry, 'Duplicates'); + } + if (hasErrors) { + this.setError(mappedEntry, 'ParsingErrors'); + } + if (!mappedEntry.newEntry.isValid) { + this.setError(mappedEntry, 'NoTitle'); + } + return mappedEntry; + } + + /** + * Executing the import. Parses all entries without errors and submits them + * to the server. The entries will receive the status 'done' on success. + */ + public async doImport(): Promise { + for (const entry of this.entries) { + if (entry.status !== 'new') { + continue; + } + await this.repo.create(entry.newEntry.topic); + entry.status = 'done'; + } + this.updatePreview(); + } + + /** + * Matching the duration string/number to the time model in use + * + * @param input + * @returns duration as defined in durationService + */ + public parseDuration(input: string): number { + return this.durationService.stringToDuration(input); + } + + /** + * Converts information from 'item type' to a model-based type number. + * Accepts either old syntax (numbers) or new visibility choice csv names; + * both defined in {@link itemVisibilityChoices} + * Empty values will be interpreted as default 'public' agenda topics + * + * @param input + * @returns a number as defined for the itemVisibilityChoices + */ + public parseType(input: string | number): number { + if (!input) { + return 1; // default, public item + } else if (typeof input === 'string') { + const visibility = itemVisibilityChoices.find(choice => choice.csvName === input); + if (visibility) { + return visibility.key; + } + } else if (input === 1) { + // Compatibility with the old client's isInternal column + const visibility = itemVisibilityChoices.find(choice => choice.name === 'Internal item'); + if (visibility) { + return visibility.key; + } + } else { + throw new TypeError('type could not be matched'); + } + } + + /** + * parses the data given by the textArea. Expects an agenda title per line + * + * @param data a string as produced by textArea input + */ + public parseTextArea(data: string): void { + const newEntries: NewEntry[] = []; + this.clearData(); + this.clearPreview(); + const lines = data.split('\n'); + lines.forEach(line => { + if (!line.length) { + return; + } + const newTopic = new ViewCreateTopic( + new CreateTopic({ + title: line, + agenda_type: 1 // set type to 'public item' by default + }) + ); + const newEntry: NewEntry = { + newEntry: newTopic, + duplicates: [], + status: 'new', + errors: [] + }; + const duplicates = this.repo.getTopicDuplicates(newTopic); + if (duplicates.length) { + // TODO this is a dishonest casting. duplicates should not be required to be View + newEntry.duplicates = duplicates as ViewCreateTopic[]; + this.setError(newEntry, 'Duplicates'); + } + newEntries.push(newEntry); + }); + this.setParsedEntries(newEntries); + } +} diff --git a/client/src/app/site/agenda/agenda-routing.module.ts b/client/src/app/site/agenda/agenda-routing.module.ts index 992272b0c..f4d341122 100644 --- a/client/src/app/site/agenda/agenda-routing.module.ts +++ b/client/src/app/site/agenda/agenda-routing.module.ts @@ -1,11 +1,14 @@ import { NgModule } from '@angular/core'; import { Routes, RouterModule } from '@angular/router'; + +import { AgendaImportListComponent } from './components/agenda-import-list/agenda-import-list.component'; import { AgendaListComponent } from './components/agenda-list/agenda-list.component'; import { TopicDetailComponent } from './components/topic-detail/topic-detail.component'; import { SpeakerListComponent } from './components/speaker-list/speaker-list.component'; const routes: Routes = [ { path: '', component: AgendaListComponent }, + { path: 'import', component: AgendaImportListComponent }, { path: 'topics/new', component: TopicDetailComponent }, { path: 'topics/:id', component: TopicDetailComponent }, { path: ':id/speakers', component: SpeakerListComponent } diff --git a/client/src/app/site/agenda/agenda.module.ts b/client/src/app/site/agenda/agenda.module.ts index 8b6414194..2b68d67a8 100644 --- a/client/src/app/site/agenda/agenda.module.ts +++ b/client/src/app/site/agenda/agenda.module.ts @@ -1,11 +1,12 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; +import { AgendaImportListComponent } from './components/agenda-import-list/agenda-import-list.component'; +import { AgendaListComponent } from './components/agenda-list/agenda-list.component'; +import { ItemInfoDialogComponent } from './components/item-info-dialog/item-info-dialog.component'; import { AgendaRoutingModule } from './agenda-routing.module'; import { SharedModule } from '../../shared/shared.module'; -import { AgendaListComponent } from './components/agenda-list/agenda-list.component'; import { TopicDetailComponent } from './components/topic-detail/topic-detail.component'; -import { ItemInfoDialogComponent } from './components/item-info-dialog/item-info-dialog.component'; /** * AppModule for the agenda and it's children. @@ -13,6 +14,6 @@ import { ItemInfoDialogComponent } from './components/item-info-dialog/item-info @NgModule({ imports: [CommonModule, AgendaRoutingModule, SharedModule], entryComponents: [ItemInfoDialogComponent], - declarations: [AgendaListComponent, TopicDetailComponent, ItemInfoDialogComponent] + declarations: [AgendaListComponent, TopicDetailComponent, ItemInfoDialogComponent, AgendaImportListComponent] }) export class AgendaModule {} diff --git a/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.html b/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.html new file mode 100644 index 000000000..fa4d96011 --- /dev/null +++ b/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.html @@ -0,0 +1,226 @@ + + + +

Import topics

+ +
+ +
+ + + + + + Required comma or semicolon separated values with these column header names in the first row: +
+
+ Title,  + Text,  + Duration,  + Comment,  + Internal item +
+
    +
  • Title is required. All other fields are optional and may be empty. +
    • +
  • Additional columns after the required ones may be present and won't affect the import.
    • +
+ +
+ + Encoding of the file + + + {{ option.label | translate }} + + + + + Column Separator + + + {{ option.label | translate }} + + + + + Text separator + + + {{ option.label | translate }} + + + +
+
+
+ + +
+
+ + + +
+
+ + Paste/write your topics in this textbox. + + Keep each item in a single line. + +
+ + + +
+
+ +
+ + + + + + +

Preview

+
+ +
+   + playlist_add +  {{ newCount }}  Topics(s) will be imported. +
+ +
+   + warning +  {{ nonImportableCount }}  entries will be ommitted. +
+ +
+   + done +  {{ doneCount }}  Topics have been imported. +
+
+
+ Click on 'import' (right top corner) to import the new topics. + +
+
+ + Show all + Show errors only + Show correct entries + +
+
+ + + + + +
+ + {{ getActionIcon(entry) }} + +
+
+ + {{ getActionIcon(entry) }} + +
+
+ + {{ getActionIcon(entry) }} + +
+ + + + + + Title + + + warning + + + warning + + {{ entry.newEntry.title }} + + + + + Item text + + {{ getShortPreview(entry.newEntry.text) }} + + + + + + Duration + + {{ getDuration(entry.newEntry.duration) }} + + + + + + Comment + + {{ entry.newEntry.comment }} + + + + + + Type + + {{ getTypeString(entry.newEntry.type) | translate }} + + + + + +
+
+ diff --git a/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.spec.ts b/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.spec.ts new file mode 100644 index 000000000..1194ad82f --- /dev/null +++ b/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.spec.ts @@ -0,0 +1,26 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { AgendaImportListComponent } from './agenda-import-list.component'; +import { E2EImportsModule } from 'e2e-imports.module'; + +describe('AgendaImportListComponent', () => { + let component: AgendaImportListComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [AgendaImportListComponent], + imports: [E2EImportsModule] + }).compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(AgendaImportListComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.ts b/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.ts new file mode 100644 index 000000000..d679a22f8 --- /dev/null +++ b/client/src/app/site/agenda/components/agenda-import-list/agenda-import-list.component.ts @@ -0,0 +1,156 @@ +import { Component } from '@angular/core'; +import { FormGroup, FormBuilder } from '@angular/forms'; +import { MatSnackBar } from '@angular/material'; +import { Title } from '@angular/platform-browser'; +import { TranslateService } from '@ngx-translate/core'; + +import { AgendaImportService } from '../../agenda-import.service'; +import { BaseImportListComponent } from 'app/site/base/base-import-list'; +import { DurationService } from 'app/core/services/duration.service'; +import { FileExportService } from 'app/core/services/file-export.service'; +import { itemVisibilityChoices } from 'app/shared/models/agenda/item'; +import { ViewCreateTopic } from '../../models/view-create-topic'; + +/** + * Component for the agenda import list view. + */ +@Component({ + selector: 'os-agenda-import-list', + templateUrl: './agenda-import-list.component.html' +}) +export class AgendaImportListComponent extends BaseImportListComponent { + /** + * A form for text input + */ + public textAreaForm: FormGroup; + + /** + * Constructor for list view bases + * + * @param titleService the title serivce + * @param matSnackBar snackbar for displaying errors + * @param translate the translate service + * @param importer: The agenda csv import service + * @param formBuilder: FormBuilder for the textarea + * @param exporter: ExportService for example download + * @param durationService: Service converting numbers to time strings + */ + public constructor( + titleService: Title, + matSnackBar: MatSnackBar, + translate: TranslateService, + importer: AgendaImportService, + formBuilder: FormBuilder, + private exporter: FileExportService, + private durationService: DurationService + ) { + super(importer, titleService, translate, matSnackBar); + this.textAreaForm = formBuilder.group({ inputtext: [''] }); + } + + /** + * Get the first characters of a string, for preview purposes + * + * @param input any string + * @returns a string with at most 50 characters + */ + public getShortPreview(input: string): string { + if (!input) { + return ''; + } + if (input.length > 50) { + return this.stripHtmlTags(input.substring(0, 47)) + '...'; + } + return this.stripHtmlTags(input); + } + + /** + * Fetch the first and last 150 characters of a string; used within + * tooltips for the preview + * + * @param input any string + * @returns a string with the first and last 150 characters of the input + * string + */ + public getLongPreview(input: string): string { + if (!input) { + return ''; + } + if (input.length < 300) { + return this.stripHtmlTags(input); + } + return ( + this.stripHtmlTags(input.substring(0, 147)) + + ' [...] ' + + this.stripHtmlTags(input.substring(input.length - 150, input.length)) + ); + } + + /** + * Helper to remove html tags from a string. + * CAUTION: It is just a basic "don't show distracting html tags in a + * preview", not an actual tested sanitizer! + * + * @param inputString + * @returns a string without hatml tags + */ + private stripHtmlTags(inputString: string): string { + const regexp = new RegExp(/<[^ ][^<>]*(>|$)/g); + return inputString.replace(regexp, '').trim(); + } + + /** + * Triggers an example csv download + */ + public downloadCsvExample(): void { + const headerRow = ['Title', 'Text', 'Duration', 'Comment', 'Internal item'] + .map(item => this.translate.instant(item)) + .join(','); + const rows = [ + headerRow, + 'Demo 1,Demo text 1,1:00,test comment,', + 'Break,,0:10,,internal', + 'Demo 2,Demo text 2,1:30,,hidden' + ]; + this.exporter.saveFile(rows.join('\n'), this.translate.instant('Topic example') + '.csv'); + } + + /** + * Fetches the string to a item_type + * + * @param type + * @returns A string, which may be empty if the type is not found in the visibilityChoices + */ + public getTypeString(type: number): string { + const visibility = itemVisibilityChoices.find(choice => choice.key === type); + return visibility ? visibility.name : ''; + } + + /** + * Sends the data in the text field input area to the importer + */ + public parseTextArea(): void { + (this.importer as AgendaImportService).parseTextArea(this.textAreaForm.get('inputtext').value); + } + + /** + * Triggers a change in the tab group: Clearing the preview selection + */ + public onTabChange(): void { + this.importer.clearPreview(); + } + + /** + * Gets a readable string for a duration + * + * @param duration + * @returns a duration string, an empty string if the duration is not set or negative + */ + public getDuration(duration: number): string { + if (duration >= 0) { + return this.durationService.durationToString(duration); + } else { + return ''; + } + } +} diff --git a/client/src/app/site/agenda/components/agenda-list/agenda-list.component.html b/client/src/app/site/agenda/components/agenda-list/agenda-list.component.html index b41ec3364..e9145d822 100644 --- a/client/src/app/site/agenda/components/agenda-list/agenda-list.component.html +++ b/client/src/app/site/agenda/components/agenda-list/agenda-list.component.html @@ -107,6 +107,10 @@ archive Export as CSV +
diff --git a/client/src/app/site/agenda/components/agenda-list/agenda-list.component.ts b/client/src/app/site/agenda/components/agenda-list/agenda-list.component.ts index 9a8ce692d..59ae4b408 100644 --- a/client/src/app/site/agenda/components/agenda-list/agenda-list.component.ts +++ b/client/src/app/site/agenda/components/agenda-list/agenda-list.component.ts @@ -11,10 +11,10 @@ import { PromptService } from '../../../../core/services/prompt.service'; import { ViewItem } from '../../models/view-item'; import { AgendaCsvExportService } from '../../services/agenda-csv-export.service'; +import { ConfigService } from 'app/core/services/config.service'; +import { DurationService } from 'app/core/services/duration.service'; import { ItemInfoDialogComponent } from '../item-info-dialog/item-info-dialog.component'; import { ViewportService } from 'app/core/services/viewport.service'; -import { DurationService } from 'app/site/core/services/duration.service'; -import { ConfigService } from 'app/core/services/config.service'; /** * List view for the agenda. diff --git a/client/src/app/site/agenda/components/item-info-dialog/item-info-dialog.component.ts b/client/src/app/site/agenda/components/item-info-dialog/item-info-dialog.component.ts index 23008664b..76a73bba3 100644 --- a/client/src/app/site/agenda/components/item-info-dialog/item-info-dialog.component.ts +++ b/client/src/app/site/agenda/components/item-info-dialog/item-info-dialog.component.ts @@ -4,7 +4,7 @@ import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material'; import { ViewItem } from '../../models/view-item'; import { itemVisibilityChoices } from 'app/shared/models/agenda/item'; -import { DurationService } from 'app/site/core/services/duration.service'; +import { DurationService } from 'app/core/services/duration.service'; /** * Dialog component to change agenda item details diff --git a/client/src/app/site/agenda/components/topic-detail/topic-detail.component.ts b/client/src/app/site/agenda/components/topic-detail/topic-detail.component.ts index d9a863256..d0a591f91 100644 --- a/client/src/app/site/agenda/components/topic-detail/topic-detail.component.ts +++ b/client/src/app/site/agenda/components/topic-detail/topic-detail.component.ts @@ -15,6 +15,7 @@ import { BehaviorSubject } from 'rxjs'; import { DataStoreService } from 'app/core/services/data-store.service'; import { Mediafile } from 'app/shared/models/mediafiles/mediafile'; import { Item, itemVisibilityChoices } from 'app/shared/models/agenda/item'; +import { CreateTopic } from '../../models/create-topic'; /** * Detail page for topics. @@ -125,7 +126,7 @@ export class TopicDetailComponent extends BaseViewComponent { if (!this.topicForm.value.agenda_parent_id) { delete this.topicForm.value.agenda_parent_id; } - await this.repo.create(this.topicForm.value); + await this.repo.create(new CreateTopic(this.topicForm.value)); this.router.navigate([`/agenda/`]); } else { this.setEditMode(false); diff --git a/client/src/app/site/agenda/models/create-topic.ts b/client/src/app/site/agenda/models/create-topic.ts new file mode 100644 index 000000000..0e398cd49 --- /dev/null +++ b/client/src/app/site/agenda/models/create-topic.ts @@ -0,0 +1,17 @@ +import { Topic } from 'app/shared/models/topics/topic'; + +/** + * Representation of Topic during creation. + */ +export class CreateTopic extends Topic { + public attachments_id: number[]; + public agenda_type: number; + public agenda_parent_id: number; + public agenda_comment: string; + public agenda_duration: number; + public agenda_weight: number; + + public constructor(input?: any) { + super(input); + } +} diff --git a/client/src/app/site/agenda/models/view-create-topic.ts b/client/src/app/site/agenda/models/view-create-topic.ts new file mode 100644 index 000000000..70766167e --- /dev/null +++ b/client/src/app/site/agenda/models/view-create-topic.ts @@ -0,0 +1,110 @@ +import { CreateTopic } from './create-topic'; +import { ViewTopic } from './view-topic'; + +/** + * View model for Topic('Agenda item') creation. + * + */ +export class ViewCreateTopic extends ViewTopic { + public get topic(): CreateTopic { + return this._topic as CreateTopic; + } + + /** + * Fetches the field representing the new title + * + * @returns title string as set during import (may be different from getTitle) + */ + public get title(): string { + return this.topic.title; + } + + /** + * Setter for the title. Sets the title of the underlying CreateTopic + * + * @param title + */ + public set title(title: string) { + this.topic.title = title; + } + + /** + * @returns the duration in minutes + */ + public get duration(): number { + return this.topic.agenda_duration; + } + + /** + * Setter for the duration. Expects values as in {@link DurationService} + */ + public set duration(duration: number) { + this.topic.agenda_duration = duration; + } + + /** + * @returns the comment string as set during the import + */ + public get comment(): string { + return this.topic.agenda_comment; + } + + /** + * Sets the comment string of the underlying topic + * @param comment A string to set as comment + */ + public set comment(comment: string) { + this.topic.agenda_comment = comment; + } + + /** + * @returns a number representing the item type + */ + public get type(): number { + return this.topic.agenda_type; + } + + /** + * sets the item type for the topic's agenda entry. No validation is done here. + * + * @param A number representing the item's type. See {@link itemVisibilityChoices} + * for the interpretation of type numbers. + */ + public set type(type: number) { + this.topic.agenda_type = type; + } + + /** + * Sets the text string of the underlying topic + * + * @param text A string. + */ + public set text(text: string) { + this.topic.text = text; + } + + /** + * @returns the comment string of the underlying topic + */ + public get text(): string { + return this.topic.text; + } + + /** + * Checks if the CreateTopic is valid. Currently only requires an existing title + * + * @returns true if it is a valid Topic + */ + public get isValid(): boolean { + return this.title ? true : false; + } + + /** + * Constructor. Empty + * + * @param topic A CreateTopic + */ + public constructor(topic: CreateTopic) { + super(topic); + } +} diff --git a/client/src/app/site/agenda/models/view-topic.ts b/client/src/app/site/agenda/models/view-topic.ts index 743f6f1b9..a44c68cfb 100644 --- a/client/src/app/site/agenda/models/view-topic.ts +++ b/client/src/app/site/agenda/models/view-topic.ts @@ -9,7 +9,7 @@ import { BaseModel } from 'app/shared/models/base/base-model'; * @ignore */ export class ViewTopic extends BaseViewModel { - private _topic: Topic; + protected _topic: Topic; private _attachments: Mediafile[]; private _agenda_item: Item; diff --git a/client/src/app/site/agenda/services/topic-repository.service.ts b/client/src/app/site/agenda/services/topic-repository.service.ts index 29630e257..842e796de 100644 --- a/client/src/app/site/agenda/services/topic-repository.service.ts +++ b/client/src/app/site/agenda/services/topic-repository.service.ts @@ -9,6 +9,7 @@ import { DataSendService } from 'app/core/services/data-send.service'; import { ViewTopic } from '../models/view-topic'; import { Identifiable } from 'app/shared/models/base/identifiable'; import { CollectionStringModelMapperService } from 'app/core/services/collectionStringModelMapper.service'; +import { CreateTopic } from '../models/create-topic'; /** * Repository for topics @@ -61,10 +62,8 @@ export class TopicRepositoryService extends BaseRepository { * @param topicData Partial topic data to be created * @returns an Identifiable (usually id) as promise */ - public async create(topicData: Partial): Promise { - const newTopic = new Topic(); - newTopic.patchValues(topicData); - return await this.dataSend.createModel(newTopic); + public async create(topic: CreateTopic): Promise { + return await this.dataSend.createModel(topic); } /** @@ -89,4 +88,16 @@ export class TopicRepositoryService extends BaseRepository { public async delete(viewTopic: ViewTopic): Promise { return await this.dataSend.deleteModel(viewTopic.topic); } + + /** + * Returns an array of all duplicates for a topic + * + * @param topic + */ + public getTopicDuplicates(topic: ViewTopic): ViewTopic[] { + const duplicates = this.DS.filter(Topic, item => topic.title === item.title); + const viewTopics: ViewTopic[] = []; + duplicates.forEach(item => viewTopics.push(this.createViewModel(item))); + return viewTopics; + } } diff --git a/client/src/app/site/base/base-import-list.ts b/client/src/app/site/base/base-import-list.ts new file mode 100644 index 000000000..7a448917d --- /dev/null +++ b/client/src/app/site/base/base-import-list.ts @@ -0,0 +1,276 @@ +import { MatTableDataSource, MatTable, MatSnackBar, MatSelectChange } from '@angular/material'; +import { ViewChild, OnInit } from '@angular/core'; + +import { BaseViewComponent } from './base-view'; +import { BaseViewModel } from './base-view-model'; +import { NewEntry, ValueLabelCombination, BaseImportService } from 'app/core/services/base-import.service'; +import { Title } from '@angular/platform-browser'; +import { TranslateService } from '@ngx-translate/core'; + +export abstract class BaseImportListComponent extends BaseViewComponent implements OnInit { + /** + * The data source for a table. Requires to be initialised with a BaseViewModel + */ + public dataSource: MatTableDataSource>; + + /** + * Switch that turns true if a file has been selected in the input + */ + public hasFile = false; + + /** + * Currently selected encoding. Is set and changed by the config's available + * encodings and user mat-select input + */ + public selectedEncoding = 'utf-8'; + + /** + * indicator on which elements to display + */ + public shown: 'all' | 'error' | 'noerror' = 'all'; + + /** + * The table itself + */ + @ViewChild(MatTable) + protected table: MatTable>; + + /** + * @returns the amount of total item successfully parsed + */ + public get totalCount(): number { + return this.importer && this.hasFile ? this.importer.summary.total : null; + } + + /** + * @returns the encodings available and their labels + */ + public get encodings(): ValueLabelCombination[] { + return this.importer.encodings; + } + + /** + * @returns the available column separators and their labels + */ + public get columnSeparators(): ValueLabelCombination[] { + return this.importer.columnSeparators; + } + + /** + * @eturns the available text separators and their labels + */ + public get textSeparators(): ValueLabelCombination[] { + return this.importer.textSeparators; + } + + /** + * @returns the amount of import items that will be imported + */ + public get newCount(): number { + return this.importer && this.hasFile ? this.importer.summary.new : 0; + } + + /** + * @returns the number of import items that cannot be imported + */ + public get nonImportableCount(): number { + if (this.importer && this.hasFile) { + return this.importer.summary.errors + this.importer.summary.duplicates; + } + return 0; + } + + /** + * @returns the number of import items that have been successfully imported + */ + public get doneCount(): number { + return this.importer && this.hasFile ? this.importer.summary.done : 0; + } + + /** + * Constructor. Initializes the table and subscribes to import errors + * + * @param importer The import service, depending on the implementation + * @param titleService A title service + * @param translate TranslationService for translating strings + * @param matSnackBar MatSnackBar for displaying errors + */ + + public constructor( + protected importer: BaseImportService, + titleService: Title, + translate: TranslateService, + matSnackBar: MatSnackBar + ) { + super(titleService, translate, matSnackBar); + this.initTable(); + this.importer.errorEvent.subscribe(this.raiseError); + } + + /** + * Starts with a clean preview (removing any previously existing import previews) + */ + public ngOnInit(): void { + this.importer.clearPreview(); + } + + /** + * Initializes the table + */ + public initTable(): void { + this.dataSource = new MatTableDataSource(); + this.setFilter(); + this.importer.getNewEntries().subscribe(newEntries => { + this.dataSource.data = newEntries; + this.hasFile = newEntries.length > 0; + }); + } + + /** + * Returns the table column definition. Fetches all headers from + * {@link MotionImportService} and an additional status column + * + * @returns An array of the columns forming the import header, and an additional 'status' bar on the front + */ + public getColumnDefinition(): string[] { + return ['status'].concat(this.importer.expectedHeader); + } + + /** + * triggers the importer's onSelectFile after a file has been chosen + */ + public onSelectFile(event: any): void { + this.importer.onSelectFile(event); + } + + /** + * Triggers the importer's import + * + */ + public async doImport(): Promise { + await this.importer.doImport(); + this.setFilter(); + } + + /** + * Updates and manually triggers the filter function. + * See {@link hidden} for options + * (changed from default mat-table filter) + */ + public setFilter(): void { + this.dataSource.filter = ''; + if (this.shown === 'all') { + this.dataSource.filterPredicate = (data, filter) => { + return true; + }; + } else if (this.shown === 'noerror') { + this.dataSource.filterPredicate = (data, filter) => { + if (data.status === 'done') { + return true; + } else if (data.status !== 'error') { + return true; + } + }; + } else if (this.shown === 'error') { + this.dataSource.filterPredicate = (data, filter) => { + if (data.errors.length || data.duplicates.length) { + return true; + } + return false; + }; + } + this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter + } + + /** + * Get the appropiate css class for a row according to the import state + * + * @param row a newEntry object with a current status + * @returns a css class name + */ + public getStateClass(row: NewEntry): string { + switch (row.status) { + case 'done': + return 'import-done import-decided'; + case 'error': + return 'import-error'; + default: + return ''; + } + } + + /** + * Get the icon for the action of the item + * @param entry a newEntry object with a current status + * @eturn the icon for the action of the item + */ + public getActionIcon(entry: NewEntry): string { + switch (entry.status) { + case 'error': // no import possible + return 'block'; + case 'new': + return ''; + case 'done': // item has been imported + return 'done'; + default: + // fallback: Error + return 'block'; + } + } + + /** + * A function to trigger the csv example download. + */ + public abstract downloadCsvExample(): void; + + /** + * Trigger for the column separator selection. + * + * @param event + */ + public selectColSep(event: MatSelectChange): void { + this.importer.columnSeparator = event.value; + this.importer.refreshFile(); + } + + /** + * Trigger for the column separator selection + * + * @param event + */ + public selectTextSep(event: MatSelectChange): void { + this.importer.textSeparator = event.value; + this.importer.refreshFile(); + } + + /** + * Trigger for the encoding selection + * + * @param event + */ + public selectEncoding(event: MatSelectChange): void { + this.importer.encoding = event.value; + this.importer.refreshFile(); + } + + /** + * Returns a descriptive string for an import error + * + * @param error The short string for the error as listed in the {@lilnk errorList} + * @returns a predefined descriptive error string from the importer + */ + public getVerboseError(error: string): string { + return this.importer.verbose(error); + } + + /** + * Checks if an error is present in a new entry + * + * @param row the NewEntry + * @param error An error as defined as key of {@link errorList} + * @returns true if the error is present in the entry described in the row + */ + public hasError(row: NewEntry, error: string): boolean { + return this.importer.hasError(row, error); + } +} diff --git a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html index d341cfa94..6a28dee39 100644 --- a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html +++ b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html @@ -9,11 +9,10 @@
- + Requires comma or semicolon separated values with these column header names in the first row:
- Identifier, Title, Text, Reason, Submitter, Category, Origin, Motion block @@ -23,7 +22,6 @@ Identifier, reason, submitter, category, origin and motion block are optional and may be empty.
  • Additional columns after the required ones may be present and won't affect the import.
    • -
  • Only double quotes are accepted as text delimiter (no single quotes).
    • @@ -71,12 +69,12 @@ Select file
    -   {{ totalCount }} entries found.
    - + +

    Preview

    @@ -97,35 +95,38 @@  {{ doneCount }}  Motions have been imported.
    +
    + Click on 'import' (right top corner) to import the new motions. + +
    - - + Show all - Show errors only - Show correct entries + Show errors only + Show correct entries -
    + - - -
    + + +
    {{ getActionIcon(entry) }}
    -
    +
    {{ getActionIcon(entry) }}
    -
    +
    {{ getActionIcon(entry) }} @@ -140,12 +141,12 @@ warning -  {{ entry.newMotion.identifier }} + {{ entry.newEntry.identifier }} @@ -155,35 +156,35 @@ warning -  {{ entry.newMotion.title }} + {{ entry.newEntry.title }} - + Motion text - + warning -  {{ getShortPreview(entry.newMotion.text) }} + {{ getShortPreview(entry.newEntry.text) }} Reason - - {{ getShortPreview(entry.newMotion.reason) }} + + {{ getShortPreview(entry.newEntry.reason) }} @@ -191,15 +192,15 @@ Submitters -
    +
    warning - + {{ submitter.name }} add   @@ -212,16 +213,16 @@ Category -
    +
    warning - {{ entry.newMotion.csvCategory.name }} - add 
    @@ -231,23 +232,23 @@ Origin - {{ entry.newMotion.origin }} + {{ entry.newEntry.origin }} - + Motion block -
    +
    warning - {{ entry.newMotion.csvMotionblock.name }} - + {{ entry.newEntry.csvMotionblock.name }} + add   @@ -256,7 +257,6 @@ -
    diff --git a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss deleted file mode 100644 index bfd5a6bf0..000000000 --- a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss +++ /dev/null @@ -1,53 +0,0 @@ -.table-container { - width: 100%; - overflow-x: scroll; - margin-top: 5px; -} - -table { - width: 100%; - overflow: scroll; -} - -.mat-header-cell { - min-width: 100px; - padding-right: 8px; -} -.mat-cell { - min-width: 100px; - padding-top: 2px; - padding-right: 8px; -} -.selection { - min-width: 80px; -} - -.import-done { - background-color: #cfc; -} -.import-error { - background-color: #fcc; -} - -.code { - padding-left: 1em; - font-style: italic; -} - -div.wrapper { - display: flex; - vertical-align: bottom; - padding: 10px; -} - -div.summary { - display: flex; -} - -.hidden-input { - display: none; -} - -.newBadge { - margin-left: -3px; -} diff --git a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts index 9b14d777d..8a75ef0a1 100644 --- a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts +++ b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts @@ -1,100 +1,21 @@ -import { MatTableDataSource, MatTable, MatSnackBar, MatSelectChange } from '@angular/material'; +import { Component } from '@angular/core'; +import { MatSnackBar } from '@angular/material'; import { Title } from '@angular/platform-browser'; import { TranslateService } from '@ngx-translate/core'; -import { ViewChild, Component, OnInit } from '@angular/core'; -import { BaseViewComponent } from 'app/site/base/base-view'; +import { BaseImportListComponent } from 'app/site/base/base-import-list'; import { MotionCsvExportService } from '../../services/motion-csv-export.service'; -import { MotionImportService, NewMotionEntry, ValueLabelCombination } from '../../services/motion-import.service'; +import { MotionImportService } from '../../services/motion-import.service'; +import { ViewMotion } from '../../models/view-motion'; /** * Component for the motion import list view. */ @Component({ selector: 'os-motion-import-list', - templateUrl: './motion-import-list.component.html', - styleUrls: ['./motion-import-list.component.scss'] + templateUrl: './motion-import-list.component.html' }) -export class MotionImportListComponent extends BaseViewComponent implements OnInit { - /** - * The data source for a table. Requires to be initialised with a BaseViewModel - */ - public dataSource: MatTableDataSource; - - /** - * Switch that turns true if a file has been selected in the input - */ - public hasFile = false; - - /** - * Currently selected encoding. Is set and changed by the config's available - * encodings and user mat-select input - */ - public selectedEncoding = 'utf-8'; - - /** - * indicator on which elements to display - */ - public shown: 'all' | 'error' | 'noerror' = 'all'; - - /** - * The table itself - */ - @ViewChild(MatTable) - protected table: MatTable; - - /** - * Returns the amount of total item successfully parsed - */ - public get totalCount(): number { - return this.importer && this.hasFile ? this.importer.summary.total : null; - } - - /** - * Returns the encodings available and their labels - */ - public get encodings(): ValueLabelCombination[] { - return this.importer.encodings; - } - - /** - * Returns the available column separators and their labels - */ - public get columnSeparators(): ValueLabelCombination[] { - return this.importer.columnSeparators; - } - - /** - * Returns the available text separators and their labels - */ - public get textSeparators(): ValueLabelCombination[] { - return this.importer.textSeparators; - } - - /** - * Returns the amount of import items that will be imported - */ - public get newCount(): number { - return this.importer && this.hasFile ? this.importer.summary.new : 0; - } - - /** - * Returns the number of import items that cannot be imported - */ - public get nonImportableCount(): number { - if (this.importer && this.hasFile) { - return this.importer.summary.errors + this.importer.summary.duplicates; - } - return 0; - } - - /** - * Returns the number of import items that have been successfully imported - */ - public get doneCount(): number { - return this.importer && this.hasFile ? this.importer.summary.done : 0; - } - +export class MotionImportListComponent extends BaseImportListComponent { /** * Constructor for list view bases * @@ -107,101 +28,11 @@ export class MotionImportListComponent extends BaseViewComponent implements OnIn public constructor( titleService: Title, matSnackBar: MatSnackBar, - public translate: TranslateService, - private importer: MotionImportService, + translate: TranslateService, + importer: MotionImportService, private motionCSVExport: MotionCsvExportService ) { - super(titleService, translate, matSnackBar); - this.initTable(); - this.importer.errorEvent.subscribe(this.raiseError); - } - - /** - * Starts with a clean preview (removing any previously existing import previews) - */ - public ngOnInit(): void { - this.importer.clearPreview(); - } - - /** - * Initializes the table - */ - public initTable(): void { - this.dataSource = new MatTableDataSource(); - this.setFilter(); - this.importer.getNewEntries().subscribe(newEntries => { - this.dataSource.data = newEntries; - this.hasFile = newEntries.length > 0; - }); - } - - /** - * Returns the table column definition. Fetches all headers from - * {@link MotionImportService} and an additional status column - */ - public getColumnDefinition(): string[] { - return ['status'].concat(this.importer.expectedHeader); - } - - /** - * triggers the importer's onSelectFile after a file has been chosen - */ - public onSelectFile(event: any): void { - this.importer.onSelectFile(event); - } - - /** - * Triggers the importer's import - */ - public async doImport(): Promise { - await this.importer.doImport(); - this.setFilter(); - } - - /** - * Updates and manually triggers the filter function. - * See {@link hidden} for options - * (changed from default mat-table filter) - */ - public setFilter(): void { - this.dataSource.filter = ''; - if (this.shown === 'all') { - this.dataSource.filterPredicate = (data, filter) => { - return true; - }; - } else if (this.shown === 'noerror') { - this.dataSource.filterPredicate = (data, filter) => { - if (data.newMotion.status === 'done') { - return true; - } else if (!(data.newMotion.status !== 'error') && !data.duplicates.length) { - return true; - } - }; - } else if (this.shown === 'error') { - this.dataSource.filterPredicate = (data, filter) => { - if (data.newMotion.errors.length || data.duplicates.length) { - return true; - } - return false; - }; - } - this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter - } - - /** - * Returns the appropiate css class for a row according to the import state - * - * @param row - */ - public getStateClass(row: NewMotionEntry): string { - switch (row.newMotion.status) { - case 'done': - return 'import-done import-decided'; - case 'error': - return 'import-error'; - default: - return ''; - } + super(importer, titleService, translate, matSnackBar); } /** @@ -233,24 +64,6 @@ export class MotionImportListComponent extends BaseViewComponent implements OnIn ); } - /** - * Return the icon for the action of the item - * @param entry - */ - public getActionIcon(entry: NewMotionEntry): string { - switch (entry.newMotion.status) { - case 'error': // no import possible - return 'block'; - case 'new': // new item, will be imported - return 'playlist_add'; - case 'done': // item has been imported - return 'done'; - default: - // fallback: Error - return 'block'; - } - } - /** * Helper to remove html tags from a string. * CAUTION: It is just a basic "don't show distracting html tags in a @@ -268,41 +81,4 @@ export class MotionImportListComponent extends BaseViewComponent implements OnIn public downloadCsvExample(): void { this.motionCSVExport.exportDummyMotion(); } - - /** - * Trigger for the column separator selection - * - * @param event - */ - public selectColSep(event: MatSelectChange): void { - this.importer.columnSeparator = event.value; - this.importer.refreshFile(); - } - - /** - * Trigger for the column separator selection - * - * @param event - */ - public selectTextSep(event: MatSelectChange): void { - this.importer.textSeparator = event.value; - this.importer.refreshFile(); - } - - /** - * Trigger for the encoding selection - * - * @param event - */ - public selectEncoding(event: MatSelectChange): void { - this.importer.encoding = event.value; - this.importer.refreshFile(); - } - - /** - * Returns a descriptive string for an import error - */ - public getVerboseError(error: string): string { - return this.importer.verbose(error); - } } diff --git a/client/src/app/site/motions/models/view-csv-create-motion.ts b/client/src/app/site/motions/models/view-csv-create-motion.ts index 98ce8904c..12c794856 100644 --- a/client/src/app/site/motions/models/view-csv-create-motion.ts +++ b/client/src/app/site/motions/models/view-csv-create-motion.ts @@ -2,9 +2,8 @@ import { ViewCreateMotion } from './view-create-motion'; import { CreateMotion } from './create-motion'; /** - * Interface for imported secondary data. A name can be matched to an existing - * model instance by the solve... functions. - * TODO MultiId will be filled if there is more than one match (to be used in case of 'I want to select one of these matches) + * Interface for correlating between strings representing BaseModels and existing + * BaseModels. */ export interface CsvMapping { name: string; @@ -12,8 +11,6 @@ export interface CsvMapping { multiId?: number[]; } -type CsvImportStatus = 'new' | 'error' | 'done'; - /** * Create motion class for the View. Its different to ViewMotion in fact that the submitter handling is different * on motion creation. @@ -38,68 +35,10 @@ export class ViewCsvCreateMotion extends ViewCreateMotion { */ public csvSubmitters: CsvMapping[]; - /** - * The current import status of this motion. - * Starts as 'new', if set to 'done', a proper {@link Motion} model will - * probably exist in the dataStore. error status will be set if the import - * cannot be done - */ - private _status: CsvImportStatus = 'new'; - - /** - * list of import errors See {@link MotionImportService} - */ - public errors: string[] = []; - - /** - * Returns the current status. - */ - public get status(): CsvImportStatus { - return this._status; - } - - public set status(status: CsvImportStatus) { - this._status = status; - } - - public get motion(): CreateMotion { - return this._motion; - } - public constructor(motion?: CreateMotion) { super(motion); } - /** - * Duplicate this motion into a copy of itself - */ - public copy(): ViewCreateMotion { - return new ViewCreateMotion( - this._motion, - this._category, - this._submitters, - this._supporters, - this._workflow, - this._state - ); - } - - /** - * Checks if a given error is present. TODO: Is more a ViewModel option - * - * @param error - */ - public hasError(error: string): boolean { - return this.errors.includes(error); - } - - /** - * Toggle to set a CreateMotion to a 'successfully parsed' status - */ - public done(): void { - this._status = 'done'; - } - /** * takes a list of motion block mappings to update the current csvMotionblock. * Returns the amount of entries that remain unmatched diff --git a/client/src/app/site/motions/services/motion-import.service.ts b/client/src/app/site/motions/services/motion-import.service.ts index 4c385e0b3..5a1432cbf 100644 --- a/client/src/app/site/motions/services/motion-import.service.ts +++ b/client/src/app/site/motions/services/motion-import.service.ts @@ -1,7 +1,6 @@ -import { BehaviorSubject, Observable } from 'rxjs'; -import { Injectable, EventEmitter } from '@angular/core'; +import { Injectable } from '@angular/core'; import { MatSnackBar } from '@angular/material'; -import { Papa, PapaParseConfig } from 'ngx-papaparse'; +import { Papa } from 'ngx-papaparse'; import { TranslateService } from '@ngx-translate/core'; import { Category } from 'app/shared/models/motions/category'; @@ -11,49 +10,9 @@ import { MotionBlock } from 'app/shared/models/motions/motion-block'; import { MotionBlockRepositoryService } from './motion-block-repository.service'; import { MotionRepositoryService } from './motion-repository.service'; import { UserRepositoryService } from '../../users/services/user-repository.service'; -import { ViewMotion } from '../models/view-motion'; import { ViewCsvCreateMotion, CsvMapping } from '../models/view-csv-create-motion'; - -/** - * Interface for value- Label combinations. - * Map objects didn't work, TODO: Use map objects (needs iterating through all objects of a map) - */ -export interface ValueLabelCombination { - value: string; - label: string; -} - -/** - * Interface for a new Motion and their (if any) duplicates - */ -export interface NewMotionEntry { - newMotion: ViewCsvCreateMotion; - duplicates: ViewMotion[]; -} - -/** - * interface for a preview summary - */ -interface ImportMotionCSVPreview { - total: number; - duplicates: number; - errors: number; - new: number; - done: number; -} - -/** - * List of possible import errors specific for motion imports. - */ -const errorList = { - MotionBlock: 'Could not resolve the motion block', - Category: 'Could not resolve the category', - Submitters: 'Could not resolve the submitters', - Title: 'A title is required', - Text: "A content in the 'text' column is required", - Duplicates: 'A motion with this identifier already exists.', - generic: 'Server upload failed' // TODO -}; +import { BaseImportService, NewEntry } from 'app/core/services/base-import.service'; +import { ViewMotion } from '../models/view-motion'; /** * Service for motion imports @@ -61,59 +20,23 @@ const errorList = { @Injectable({ providedIn: 'root' }) -export class MotionImportService { - /** The header (order and items) that is expected from the imported file - * +export class MotionImportService extends BaseImportService { + /** + * List of possible errors and their verbose explanation */ - public expectedHeader = [ - 'identifier', - 'title', - 'text', - 'reason', - 'submitters', - 'category', - 'origin', - 'motion block' - ]; + public errorList = { + MotionBlock: 'Could not resolve the motion block', + Category: 'Could not resolve the category', + Submitters: 'Could not resolve the submitters', + Title: 'A title is required', + Text: "A content in the 'text' column is required", + Duplicates: 'A motion with this identifier already exists.' + }; /** - * The last parsed file object (may be reparsed with new encoding, thus kept in memory) + * The minimimal number of header entries needed to successfully create an entry */ - private _rawFile: File; - - /** - * The used column Separator. If left on an empty string (default), - * the papaparse parser will automatically decide on separators. - */ - public columnSeparator = ''; - - public textSeparator = '"'; - - public encoding = 'utf-8'; - - /** - * List of possible encodings and their label - */ - public encodings: ValueLabelCombination[] = [ - { value: 'utf-8', label: 'UTF 8 - Unicode' }, - { value: 'iso-8859-1', label: 'ISO 8859-1 - West European' }, - { value: 'iso-8859-15', label: 'ISO 8859-15 - West European (with €)' } - ]; - - /** - * List of possible column separators - */ - public columnSeparators: ValueLabelCombination[] = [ - { label: 'Comma', value: ',' }, - { label: 'Semicolon', value: ';' }, - // {label: 'Tabulator', value: '\t'}, - { label: 'Automatic', value: '' } - ]; - - public textSeparators: ValueLabelCombination[] = [ - { label: 'Double quotes (")', value: '"' }, - { label: "Single quotes (')", value: "'" } - ]; + public requiredHeaderLength = 3; /** * submitters that need to be created prior to importing @@ -131,44 +54,8 @@ export class MotionImportService { public newMotionBlocks: CsvMapping[] = []; /** - * FileReader object for file import - */ - private reader = new FileReader(); - - /** - * the list of parsed models that have been extracted from the opened file - */ - private _entries: NewMotionEntry[] = []; - - /** - * BehaviorSubject for displaying a preview for the currently selected entries - */ - public newEntries = new BehaviorSubject([]); - - /** - * Emits an error string to display if a file import cannot be done - */ - public errorEvent = new EventEmitter(); - - /** - * storing the summary preview for the import, to avoid recalculating it - * at each display change. - */ - private _preview: ImportMotionCSVPreview; - - /** - * Returns a summary on actions that will be taken/not taken. - */ - public get summary(): ImportMotionCSVPreview { - if (!this._preview) { - this.updatePreview(); - } - return this._preview; - } - - /** - * Constructor. Creates a fileReader to subscribe to it for incoming parsed - * strings + * Constructor. Defines the headers expected and calls the abstract class + * @param repo: The repository for motions. * @param categoryRepo Repository to fetch pre-existing categories * @param motionBlockRepo Repository to fetch pre-existing motionBlocks * @param userRepo Repository to query/ create users @@ -181,158 +68,112 @@ export class MotionImportService { private categoryRepo: CategoryRepositoryService, private motionBlockRepo: MotionBlockRepositoryService, private userRepo: UserRepositoryService, - private translate: TranslateService, - private papa: Papa, - private matSnackbar: MatSnackBar + translate: TranslateService, + papa: Papa, + matSnackbar: MatSnackBar ) { - this.reader.onload = (event: any) => { - // TODO type: event is a progressEvent, - // but has a property target.result, which typescript doesn't recognize - this.parseInput(event.target.result); - }; + super(translate, papa, matSnackbar); + + this.expectedHeader = [ + 'identifier', + 'title', + 'text', + 'reason', + 'submitters', + 'category', + 'origin', + 'motion_block' + ]; } /** - * Parses the data input. Expects a string as returned by via a - * File.readAsText() operation - * - * @param file + * Clears all temporary data specific to this importer. */ - public parseInput(file: string): void { - this._entries = []; + public clearData(): void { this.newSubmitters = []; this.newCategories = []; this.newMotionBlocks = []; - const papaConfig: PapaParseConfig = { - header: false, - skipEmptyLines: true, - quoteChar: this.textSeparator - }; - if (this.columnSeparator) { - papaConfig.delimiter = this.columnSeparator; - } - const entryLines = this.papa.parse(file, papaConfig).data; - const valid = this.checkHeader(entryLines.shift()); - if (!valid) { - return; - } - entryLines.forEach(line => { - const newMotion = new ViewCsvCreateMotion(new CreateMotion()); - const headerLength = Math.min(this.expectedHeader.length, line.length); - for (let idx = 0; idx < headerLength; idx++) { - // iterate over items, find existing ones (thier id) and collect new entries - switch (this.expectedHeader[idx]) { - case 'submitters': - newMotion.csvSubmitters = this.getSubmitters(line[idx]); - break; - case 'category': - newMotion.csvCategory = this.getCategory(line[idx]); - break; - case 'motion block': - newMotion.csvMotionblock = this.getMotionBlock(line[idx]); - break; - default: - newMotion.motion[this.expectedHeader[idx]] = line[idx]; - } - } - const updateModels = this.getDuplicates(newMotion.motion); - if (updateModels.length) { - this.setError(newMotion, 'Duplicates'); - } - this._entries.push({ newMotion: newMotion, duplicates: updateModels }); - }); - this.newEntries.next(this._entries); - this.updatePreview(); } /** - * Triggers the import. + * Parses a string representing an entry, extracting secondary data, appending + * the array of secondary imports as needed + * + * @param line + * @returns a new Entry representing a Motion + */ + public mapData(line: string): NewEntry { + const newEntry = new ViewCsvCreateMotion(new CreateMotion()); + const headerLength = Math.min(this.expectedHeader.length, line.length); + for (let idx = 0; idx < headerLength; idx++) { + switch (this.expectedHeader[idx]) { + case 'submitters': + newEntry.csvSubmitters = this.getSubmitters(line[idx]); + break; + case 'category': + newEntry.csvCategory = this.getCategory(line[idx]); + break; + case 'motion_block': + newEntry.csvMotionblock = this.getMotionBlock(line[idx]); + break; + default: + newEntry.motion[this.expectedHeader[idx]] = line[idx]; + } + } + const updateModels = this.repo.getMotionDuplicates(newEntry); + return { + newEntry: newEntry, + duplicates: updateModels, + status: updateModels.length ? 'error' : 'new', + errors: updateModels.length ? ['Duplicates'] : [] + }; + } + + /** + * Executes the import. Creates all secondary data, maps the newly created + * secondary data to the new entries, then creates all entries without errors + * by submitting them to the server. The entries will receive the status + * 'done' on success. */ public async doImport(): Promise { this.newMotionBlocks = await this.createNewMotionBlocks(); this.newCategories = await this.createNewCategories(); this.newSubmitters = await this.createNewUsers(); - for (const entry of this._entries) { - if (entry.newMotion.status !== 'new') { + for (const entry of this.entries) { + if (entry.status !== 'new') { continue; } - const openBlocks = entry.newMotion.solveMotionBlocks(this.newMotionBlocks); + const openBlocks = (entry.newEntry as ViewCsvCreateMotion).solveMotionBlocks(this.newMotionBlocks); if (openBlocks) { - this.setError(entry.newMotion, 'MotionBlock'); - // TODO error handling if not all submitters could be matched + this.setError(entry, 'MotionBlock'); this.updatePreview(); continue; } - const openCategories = entry.newMotion.solveCategory(this.newCategories); + const openCategories = (entry.newEntry as ViewCsvCreateMotion).solveCategory(this.newCategories); if (openCategories) { - this.setError(entry.newMotion, 'Category'); + this.setError(entry, 'Category'); this.updatePreview(); continue; } - const openUsers = entry.newMotion.solveSubmitters(this.newSubmitters); + const openUsers = (entry.newEntry as ViewCsvCreateMotion).solveSubmitters(this.newSubmitters); if (openUsers) { - this.setError(entry.newMotion, 'Submitters'); + this.setError(entry, 'Submitters'); this.updatePreview(); continue; } - await this.repo.create(entry.newMotion.motion); - entry.newMotion.done(); + await this.repo.create((entry.newEntry as ViewCsvCreateMotion).motion); + entry.status = 'done'; } this.updatePreview(); } - /** - * Checks the dataStore for duplicates - * @returns an array of duplicates with the same identifier. - * @param motion - */ - public getDuplicates(motion: CreateMotion): ViewMotion[] { - return this.repo.getMotionDuplicates(motion); - } - - /** - * counts the amount of duplicates that have no decision on the action to - * be taken - */ - public updatePreview(): void { - const summary = { - total: 0, - new: 0, - duplicates: 0, - errors: 0, - done: 0 - }; - this._entries.forEach(entry => { - summary.total += 1; - if (entry.newMotion.status === 'done') { - summary.done += 1; - return; - } else if (entry.newMotion.status === 'error' && !entry.duplicates.length) { - // errors that are not due to duplicates - summary.errors += 1; - return; - } else if (entry.duplicates.length) { - summary.duplicates += 1; - return; - } else if (entry.newMotion.status === 'new') { - summary.new += 1; - } - }); - this._preview = summary; - } - - /** - * returns a subscribable representation of the new Users to be imported - */ - public getNewEntries(): Observable { - return this.newEntries.asObservable(); - } - /** * Checks the provided submitter(s) and returns an object with mapping of * existing users and of users that need to be created + * * @param submitterlist + * @returns a list of submitters mapped with (if already existing) their id */ public getSubmitters(submitterlist: string): CsvMapping[] { const result: CsvMapping[] = []; @@ -375,7 +216,9 @@ export class MotionImportService { * characters at the beginning, separated by ' - ' from the name. * It will also accept a registered translation between the current user's * language and english + * * @param categoryString + * @returns categories mapped to existing categories */ public getCategory(categoryString: string): CsvMapping { if (!categoryString) { @@ -411,7 +254,9 @@ export class MotionImportService { * Checks the motionBlock provided in the string for existance, expands newMotionBlocks * if needed. Note that it will also check for translation between the current * user's language and english + * * @param blockString + * @returns a CSVMap with the MotionBlock and an id (if the motionBlock is already in the dataStore) */ public getMotionBlock(blockString: string): CsvMapping { if (!blockString) { @@ -434,6 +279,8 @@ export class MotionImportService { /** * Creates all new Users needed for the import. + * + * @returns a promise with list of new Submitters, updated with newly created ids */ private async createNewUsers(): Promise { const promises: Promise[] = []; @@ -445,6 +292,8 @@ export class MotionImportService { /** * Creates all new Motion Blocks needed for the import. + * + * @returns a promise with list of new MotionBlocks, updated with newly created ids */ private async createNewMotionBlocks(): Promise { const promises: Promise[] = []; @@ -460,6 +309,8 @@ export class MotionImportService { /** * Creates all new Categories needed for the import. + * + * @returns a promise with list of new Categories, updated with newly created ids */ private async createNewCategories(): Promise { const promises: Promise[] = []; @@ -481,105 +332,12 @@ export class MotionImportService { return await Promise.all(promises); } - /** - * Handler after a file was selected. Basic checking for type, then hand - * over to parsing - * - * @param event type is Event, but has target.files, which typescript doesn't seem to recognize - */ - public onSelectFile(event: any): void { - // TODO type - if (event.target.files && event.target.files.length === 1) { - if (event.target.files[0].type === 'text/csv') { - this._rawFile = event.target.files[0]; - this.readFile(event.target.files[0]); - } else { - this.matSnackbar.open(this.translate.instant('Wrong file type detected. Import failed.'), '', { - duration: 3000 - }); - this.clearPreview(); - this._rawFile = null; - } - } - } - - /** - * Rereads the (previously selected) file, if present. Thought to be triggered - * by parameter changes on encoding, column, text separators - */ - public refreshFile(): void { - if (this._rawFile) { - this.readFile(this._rawFile); - } - } - - /** - * (re)-reads a given file with the current parameter - */ - private readFile(file: File): void { - this.reader.readAsText(file, this.encoding); - } - - /** - * Checks the first line of the csv (the header) for consistency (length) - * @param row expected to be an array parsed from the first line of a csv file - */ - private checkHeader(row: string[]): boolean { - const snackbarDuration = 3000; - if (row.length < 4) { - this.matSnackbar.open(this.translate.instant('The file has too few columns to be parsed properly.'), '', { - duration: snackbarDuration - }); - - this.clearPreview(); - return false; - } else if (row.length < this.expectedHeader.length) { - this.matSnackbar.open( - this.translate.instant('The file seems to have some ommitted columns. They will be considered empty.'), - '', - { duration: snackbarDuration } - ); - } else if (row.length > this.expectedHeader.length) { - this.matSnackbar.open( - this.translate.instant('The file seems to have additional columns. They will be ignored.'), - '', - { duration: snackbarDuration } - ); - } - return true; - } - - /** - * Resets the data and preview (triggered upon selecting an invalid file) - */ - public clearPreview(): void { - this._entries = []; - this.newEntries.next([]); - this._preview = null; - } - - /** - * set a list of short names for error, indicating which column failed - */ - public setError(motion: ViewCsvCreateMotion, error: string): void { - if (errorList.hasOwnProperty(error) && !motion.errors.includes(error)) { - motion.errors.push(error); - motion.status = 'error'; - } - } - - /** - * Get an extended error description. - * @param error - */ - public verbose(error: string): string { - return errorList[error]; - } - /** * Helper to separate a category string from its' prefix. Assumes that a prefix is no longer * than 5 chars and separated by a ' - ' + * * @param categoryString the string to parse + * @returns an object with .prefix and .name strings */ private splitCategoryString(categoryString: string): { prefix: string; name: string } { let prefixSeparator = ' - '; diff --git a/client/src/app/site/motions/services/motion-repository.service.ts b/client/src/app/site/motions/services/motion-repository.service.ts index fab18d895..198db6a32 100644 --- a/client/src/app/site/motions/services/motion-repository.service.ts +++ b/client/src/app/site/motions/services/motion-repository.service.ts @@ -607,12 +607,13 @@ export class MotionRepositoryService extends BaseRepository } /** - * Returns all Motion duplicates (sharing specific values given in input) + * Returns motion duplicates (sharing the identifier) + * * @param viewMotion the ViewMotion to compare against the list of Motions * in the data - * @param sharedValues properties that must be equal to consider it a duplicate + * @returns An Array of ViewMotions with the same identifier of the input, or an empty array */ - public getMotionDuplicates(motion: Motion): ViewMotion[] { + public getMotionDuplicates(motion: ViewMotion): ViewMotion[] { const duplicates = this.DS.filter(Motion, item => motion.identifier === item.identifier); const viewMotions: ViewMotion[] = []; duplicates.forEach(item => viewMotions.push(this.createViewModel(item))); diff --git a/client/src/app/site/users/components/user-import/user-import-list.component.html b/client/src/app/site/users/components/user-import/user-import-list.component.html new file mode 100644 index 000000000..8d5ce6d26 --- /dev/null +++ b/client/src/app/site/users/components/user-import/user-import-list.component.html @@ -0,0 +1,283 @@ + + +

    Import users

    + +
    + +
    +     + + + + Requires comma or semicolon separated values with these column header names in the first row:
    +
    + Title, Given name, Surname , + Structure level, Participant number, + Groups , Comment, Is active, + Is present , Is committee, + Initial password, Email +
    +
      +
    • + At least given name or surname have to be filled in. All other fields are optional and may be empty. +
      • +
    • + Additional columns after the required ones may be present and won't affect the import. +
      • +
    + +
    + + Encoding of the file + + + {{ option.label | translate }} + + + + + Column Separator + + + {{ option.label | translate }} + + + + + Text separator + + + {{ option.label | translate }} + + + +
    +
    +
    + + +
    +
    +     + +
    +
    + + Copy and paste your participant names in this textbox. + + Keep each person in a single line. +
    + Comma separated names will be read as 'Surname(s), given name(s)'. +
    + + + +
    +
    + +
    +     +     +     + + + +

    Preview

    +
    + +
    +   + playlist_add +  {{ newCount }}  User(s) will be imported. +
    + +
    +   + warning +  {{ nonImportableCount }}  entries will be ommitted. +
    + +
    +   + done +  {{ doneCount }}  Users have been imported. +
    +
    +
    + Click on 'import' (right top corner) to import the new users. + +
    + + Show all + Show errors only + Show correct entries + +
    + + + + + + +
    + + {{ getActionIcon(entry) }} + + + warning + +
    +
    + + {{ getActionIcon(entry) }} + +
    +
    + + {{ getActionIcon(entry) }} + +
    +     +     + + + + Title + + + + warning + +   + + {{ entry.newEntry.title }} + + + + + + Given name + + + + warning + +   + + {{ entry.newEntry.first_name }} + + + + + Surname + + + + warning + +   + + {{ entry.newEntry.last_name }} + + + + + Structure level + {{ entry.newEntry.structure_level }} + + + + Participant number + {{ entry.newEntry.user.number }} + + + + + Groups + +
    + + + warning + + + + {{ group.name }} + add +   + +
    +     +     + + + Comment + {{ entry.newEntry.comment }} + + + + Is Active + + + + + + Is Present + + + + + + Is Committee + + + + + + Initial password + {{ entry.newEntry.default_password }} + + + Email + {{ entry.newEntry.email }} + + + + +
    +
    +     diff --git a/client/src/app/site/users/components/user-import/user-import-list.component.spec.ts b/client/src/app/site/users/components/user-import/user-import-list.component.spec.ts new file mode 100644 index 000000000..bddd6128c --- /dev/null +++ b/client/src/app/site/users/components/user-import/user-import-list.component.spec.ts @@ -0,0 +1,26 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { UserImportListComponent } from './user-import-list.component'; +import { E2EImportsModule } from 'e2e-imports.module'; + +describe('UserImportListComponent', () => { + let component: UserImportListComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [UserImportListComponent], + imports: [E2EImportsModule] + }).compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(UserImportListComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/client/src/app/site/users/components/user-import/user-import-list.component.ts b/client/src/app/site/users/components/user-import/user-import-list.component.ts new file mode 100644 index 000000000..c36b762c1 --- /dev/null +++ b/client/src/app/site/users/components/user-import/user-import-list.component.ts @@ -0,0 +1,103 @@ +import { Component } from '@angular/core'; +import { MatSnackBar } from '@angular/material'; +import { Title } from '@angular/platform-browser'; +import { TranslateService } from '@ngx-translate/core'; + +import { BaseImportListComponent } from 'app/site/base/base-import-list'; +import { FileExportService } from 'app/core/services/file-export.service'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { NewEntry } from 'app/core/services/base-import.service'; +import { UserImportService } from '../../services/user-import.service'; +import { ViewUser } from '../../models/view-user'; + +/** + * Component for the user import list view. + */ +@Component({ + selector: 'os-user-import-list', + templateUrl: './user-import-list.component.html' +}) +export class UserImportListComponent extends BaseImportListComponent { + public textAreaForm: FormGroup; + + /** + * Constructor for list view bases + * + * @param titleService the title serivce + * @param matSnackBar snackbar for displaying errors + * @param formBuilder: FormBuilder for the textArea + * @param translate the translate service + * @param exporter: csv export service for dummy dat + * @param importer: The motion csv import service + */ + public constructor( + titleService: Title, + matSnackBar: MatSnackBar, + formBuilder: FormBuilder, + public translate: TranslateService, + private exporter: FileExportService, + importer: UserImportService + ) { + super(importer, titleService, translate, matSnackBar); + this.textAreaForm = formBuilder.group({ inputtext: [''] }); + } + + /** + * Triggers an example csv download + */ + public downloadCsvExample(): void { + const headerRow = [ + 'Title', + 'Given name', + 'Surname', + 'Structure level', + 'Participant number', + 'Groups', + 'Comment', + 'Is active', + 'Is present', + 'Is a committee', + 'Initial password', + 'Email' + ] + .map(item => this.translate.instant(item)) + .join(','); + const rows = [ + headerRow, + 'Dr.,Max,Mustermann,"Berlin",1234567890,"Delegates, Staff",xyz,1,1,,initialPassword,', + ',John,Doe,Washington,75/99/8-2,Committees,"This is a comment, without doubt",1,1,,,john.doe@email.com', + ',Fred,Bloggs,London,,,,,,,,', + ',,Executive Board,,,,,,,1,,' + ]; + this.exporter.saveFile(rows.join('\n'), this.translate.instant('User example') + '.csv'); + } + + /** + * Shorthand for getVerboseError on name fields checking for duplicates and invalid fields + * + * @param row + * @returns an error string similar to getVerboseError + */ + public nameErrors(row: NewEntry): string { + for (const name of ['NoName', 'Duplicates', 'DuplicateImport']) { + if (this.importer.hasError(row, name)) { + return this.importer.verbose(name); + } + } + return ''; + } + + /** + * Sends the data in the text field input area to the importer + */ + public parseTextArea(): void { + (this.importer as UserImportService).parseTextArea(this.textAreaForm.get('inputtext').value); + } + + /** + * Triggers a change in the tab group: Clearing the preview selection + */ + public onTabChange(): void { + this.importer.clearPreview(); + } +} diff --git a/client/src/app/site/users/components/user-list/user-list.component.html b/client/src/app/site/users/components/user-list/user-list.component.html index 9dc6e672b..322226a27 100644 --- a/client/src/app/site/users/components/user-list/user-list.component.html +++ b/client/src/app/site/users/components/user-list/user-list.component.html @@ -90,6 +90,12 @@ archive Export as CSV + + +