diff --git a/client/package.json b/client/package.json index 73f6b8533..27dccb4c4 100644 --- a/client/package.json +++ b/client/package.json @@ -43,6 +43,7 @@ "material-design-icons": "^3.0.1", "ngx-file-drop": "^5.0.0", "ngx-mat-select-search": "^1.4.2", + "ngx-papaparse": "^3.0.2", "po2json": "^1.0.0-alpha", "roboto-fontface": "^0.10.0", "rxjs": "^6.3.3", diff --git a/client/src/app/app.module.ts b/client/src/app/app.module.ts index d6b279638..b7c5bc91f 100644 --- a/client/src/app/app.module.ts +++ b/client/src/app/app.module.ts @@ -3,6 +3,7 @@ import { BrowserModule } from '@angular/platform-browser'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { NgModule, APP_INITIALIZER } from '@angular/core'; import { HttpClientModule, HttpClient, HttpClientXsrfModule } from '@angular/common/http'; +import { PapaParseModule } from 'ngx-papaparse'; // Elementary App Components import { AppRoutingModule } from './app-routing.module'; @@ -53,7 +54,8 @@ export function AppLoaderFactory(appLoadService: AppLoadService): () => Promise< }), AppRoutingModule, CoreModule, - LoginModule + LoginModule, + PapaParseModule ], providers: [{ provide: APP_INITIALIZER, useFactory: AppLoaderFactory, deps: [AppLoadService], multi: true }], bootstrap: [AppComponent] diff --git a/client/src/app/core/services/csv-export.service.ts b/client/src/app/core/services/csv-export.service.ts index b3022d91d..c7f6e85b8 100644 --- a/client/src/app/core/services/csv-export.service.ts +++ b/client/src/app/core/services/csv-export.service.ts @@ -147,12 +147,12 @@ export class CsvExportService { if (!tsList.length) { throw new Error('no usable text separator left for valid csv text'); } - const csvContentAsString: string = csvContent .map(line => { return line.map(entry => tsList[0] + entry + tsList[0]).join(columnSeparator); }) .join(lineSeparator); + this.exporter.saveFile(csvContentAsString, filename); } @@ -163,7 +163,7 @@ export class CsvExportService { * * @param input any input to be sent to CSV * @param tsList The list of special characters to check. - * @returns the cleand CSV String list + * @returns the cleaned CSV String list */ public checkCsvTextSafety(input: string, tsList: string[]): string[] { if (input === null || input === undefined) { diff --git a/client/src/app/core/services/data-store.service.ts b/client/src/app/core/services/data-store.service.ts index 3242ad9ac..43594e702 100644 --- a/client/src/app/core/services/data-store.service.ts +++ b/client/src/app/core/services/data-store.service.ts @@ -287,6 +287,21 @@ export class DataStoreService { return this.getAll(collectionType).filter(callback); } + /** + * Finds a model item in the dataStore by type. + * + * @param collectionType The desired BaseModel type to be read from the dataStore + * @param callback a find function + * @return The first BaseModel item matching the filter function + * @example this.DS.find(User, myUser => myUser.first_name === "Jenny") + */ + public find>( + collectionType: ModelConstructor | string, + callback: (model: T) => boolean + ): T { + return this.getAll(collectionType).find(callback); + } + /** * Add one or multiple models to dataStore. * diff --git a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html new file mode 100644 index 000000000..d341cfa94 --- /dev/null +++ b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.html @@ -0,0 +1,263 @@ + + +

Import motions

+ +
+ +
+ + + + Requires comma or semicolon separated values with these column header names in the first row: +
+
+ + Identifier, Title, Text, + Reason, Submitter, Category, + Origin, Motion block +
+
    +
  • + Identifier, reason, submitter, category, origin and motion block are optional and may be empty. +
    • +
  • Additional columns after the required ones may be present and won't affect the import.
    • +
  • Only double quotes are accepted as text delimiter (no single quotes).
    • +
+ +
+ + Encoding of the file + + + {{ option.label | translate }} + + + + + Column Separator + + + {{ option.label | translate }} + + + + + Text separator + + + {{ option.label | translate }} + + + +
+
+
+ + +
+   {{ totalCount }} entries found. +
+ + + + +
+ +
+   + playlist_add +  {{ newCount }}  Motion(s) will be imported. +
+ +
+   + warning +  {{ nonImportableCount }}  entries will be ommitted. +
+ +
+   + done +  {{ doneCount }}  Motions have been imported. +
+
+
+ + + Show all + Show errors only + Show correct entries + + +
+
+ + + + + +
+ + {{ getActionIcon(entry) }} + +
+
+ + {{ getActionIcon(entry) }} + +
+
+ + {{ getActionIcon(entry) }} + +
+ + + + + + Identifier + + + warning + +  {{ entry.newMotion.identifier }} + + + + + + Title + + + warning + +  {{ entry.newMotion.title }} + + + + + + Motion text + + + warning + +  {{ getShortPreview(entry.newMotion.text) }} + + + + + + Reason + + {{ getShortPreview(entry.newMotion.reason) }} + + + + + + Submitters + +
+ + warning + + + {{ submitter.name }} + add +   + +
+ + + + + + Category + +
+ + warning + + {{ entry.newMotion.csvCategory.name }} + add  +
+ + + + + + Origin + {{ entry.newMotion.origin }} + + + + + Motion block + +
+ + warning + + {{ entry.newMotion.csvMotionblock.name }} + + add + +   +
+ + + + + + +
+
+ diff --git a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss new file mode 100644 index 000000000..bfd5a6bf0 --- /dev/null +++ b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.scss @@ -0,0 +1,53 @@ +.table-container { + width: 100%; + overflow-x: scroll; + margin-top: 5px; +} + +table { + width: 100%; + overflow: scroll; +} + +.mat-header-cell { + min-width: 100px; + padding-right: 8px; +} +.mat-cell { + min-width: 100px; + padding-top: 2px; + padding-right: 8px; +} +.selection { + min-width: 80px; +} + +.import-done { + background-color: #cfc; +} +.import-error { + background-color: #fcc; +} + +.code { + padding-left: 1em; + font-style: italic; +} + +div.wrapper { + display: flex; + vertical-align: bottom; + padding: 10px; +} + +div.summary { + display: flex; +} + +.hidden-input { + display: none; +} + +.newBadge { + margin-left: -3px; +} diff --git a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.spec.ts b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.spec.ts new file mode 100644 index 000000000..da4df5d01 --- /dev/null +++ b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.spec.ts @@ -0,0 +1,26 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { MotionImportListComponent } from './motion-import-list.component'; +import { E2EImportsModule } from 'e2e-imports.module'; + +describe('MotionImportListComponent', () => { + let component: MotionImportListComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [MotionImportListComponent], + imports: [E2EImportsModule] + }).compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(MotionImportListComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts new file mode 100644 index 000000000..9b14d777d --- /dev/null +++ b/client/src/app/site/motions/components/motion-import-list/motion-import-list.component.ts @@ -0,0 +1,308 @@ +import { MatTableDataSource, MatTable, MatSnackBar, MatSelectChange } from '@angular/material'; +import { Title } from '@angular/platform-browser'; +import { TranslateService } from '@ngx-translate/core'; +import { ViewChild, Component, OnInit } from '@angular/core'; + +import { BaseViewComponent } from 'app/site/base/base-view'; +import { MotionCsvExportService } from '../../services/motion-csv-export.service'; +import { MotionImportService, NewMotionEntry, ValueLabelCombination } from '../../services/motion-import.service'; + +/** + * Component for the motion import list view. + */ +@Component({ + selector: 'os-motion-import-list', + templateUrl: './motion-import-list.component.html', + styleUrls: ['./motion-import-list.component.scss'] +}) +export class MotionImportListComponent extends BaseViewComponent implements OnInit { + /** + * The data source for a table. Requires to be initialised with a BaseViewModel + */ + public dataSource: MatTableDataSource; + + /** + * Switch that turns true if a file has been selected in the input + */ + public hasFile = false; + + /** + * Currently selected encoding. Is set and changed by the config's available + * encodings and user mat-select input + */ + public selectedEncoding = 'utf-8'; + + /** + * indicator on which elements to display + */ + public shown: 'all' | 'error' | 'noerror' = 'all'; + + /** + * The table itself + */ + @ViewChild(MatTable) + protected table: MatTable; + + /** + * Returns the amount of total item successfully parsed + */ + public get totalCount(): number { + return this.importer && this.hasFile ? this.importer.summary.total : null; + } + + /** + * Returns the encodings available and their labels + */ + public get encodings(): ValueLabelCombination[] { + return this.importer.encodings; + } + + /** + * Returns the available column separators and their labels + */ + public get columnSeparators(): ValueLabelCombination[] { + return this.importer.columnSeparators; + } + + /** + * Returns the available text separators and their labels + */ + public get textSeparators(): ValueLabelCombination[] { + return this.importer.textSeparators; + } + + /** + * Returns the amount of import items that will be imported + */ + public get newCount(): number { + return this.importer && this.hasFile ? this.importer.summary.new : 0; + } + + /** + * Returns the number of import items that cannot be imported + */ + public get nonImportableCount(): number { + if (this.importer && this.hasFile) { + return this.importer.summary.errors + this.importer.summary.duplicates; + } + return 0; + } + + /** + * Returns the number of import items that have been successfully imported + */ + public get doneCount(): number { + return this.importer && this.hasFile ? this.importer.summary.done : 0; + } + + /** + * Constructor for list view bases + * + * @param titleService the title serivce + * @param matSnackBar snackbar for displaying errors + * @param translate the translate service + * @param importer: The motion csv import service + * @param motionCSVExport: service for exporting example data + */ + public constructor( + titleService: Title, + matSnackBar: MatSnackBar, + public translate: TranslateService, + private importer: MotionImportService, + private motionCSVExport: MotionCsvExportService + ) { + super(titleService, translate, matSnackBar); + this.initTable(); + this.importer.errorEvent.subscribe(this.raiseError); + } + + /** + * Starts with a clean preview (removing any previously existing import previews) + */ + public ngOnInit(): void { + this.importer.clearPreview(); + } + + /** + * Initializes the table + */ + public initTable(): void { + this.dataSource = new MatTableDataSource(); + this.setFilter(); + this.importer.getNewEntries().subscribe(newEntries => { + this.dataSource.data = newEntries; + this.hasFile = newEntries.length > 0; + }); + } + + /** + * Returns the table column definition. Fetches all headers from + * {@link MotionImportService} and an additional status column + */ + public getColumnDefinition(): string[] { + return ['status'].concat(this.importer.expectedHeader); + } + + /** + * triggers the importer's onSelectFile after a file has been chosen + */ + public onSelectFile(event: any): void { + this.importer.onSelectFile(event); + } + + /** + * Triggers the importer's import + */ + public async doImport(): Promise { + await this.importer.doImport(); + this.setFilter(); + } + + /** + * Updates and manually triggers the filter function. + * See {@link hidden} for options + * (changed from default mat-table filter) + */ + public setFilter(): void { + this.dataSource.filter = ''; + if (this.shown === 'all') { + this.dataSource.filterPredicate = (data, filter) => { + return true; + }; + } else if (this.shown === 'noerror') { + this.dataSource.filterPredicate = (data, filter) => { + if (data.newMotion.status === 'done') { + return true; + } else if (!(data.newMotion.status !== 'error') && !data.duplicates.length) { + return true; + } + }; + } else if (this.shown === 'error') { + this.dataSource.filterPredicate = (data, filter) => { + if (data.newMotion.errors.length || data.duplicates.length) { + return true; + } + return false; + }; + } + this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter + } + + /** + * Returns the appropiate css class for a row according to the import state + * + * @param row + */ + public getStateClass(row: NewMotionEntry): string { + switch (row.newMotion.status) { + case 'done': + return 'import-done import-decided'; + case 'error': + return 'import-error'; + default: + return ''; + } + } + + /** + * Returns the first characters of a string, for preview purposes + * + * @param input + */ + public getShortPreview(input: string): string { + if (input.length > 50) { + return this.stripHtmlTags(input.substring(0, 47)) + '...'; + } + return this.stripHtmlTags(input); + } + + /** + * Returns the first and last 150 characters of a string; used within + * tooltips for the preview + * + * @param input + */ + public getLongPreview(input: string): string { + if (input.length < 300) { + return this.stripHtmlTags(input); + } + return ( + this.stripHtmlTags(input.substring(0, 147)) + + ' [...] ' + + this.stripHtmlTags(input.substring(input.length - 150, input.length)) + ); + } + + /** + * Return the icon for the action of the item + * @param entry + */ + public getActionIcon(entry: NewMotionEntry): string { + switch (entry.newMotion.status) { + case 'error': // no import possible + return 'block'; + case 'new': // new item, will be imported + return 'playlist_add'; + case 'done': // item has been imported + return 'done'; + default: + // fallback: Error + return 'block'; + } + } + + /** + * Helper to remove html tags from a string. + * CAUTION: It is just a basic "don't show distracting html tags in a + * preview", not an actual tested sanitizer! + * @param inputString + */ + private stripHtmlTags(inputString: string): string { + const regexp = new RegExp(/<[^ ][^<>]*(>|$)/g); + return inputString.replace(regexp, '').trim(); + } + + /** + * Triggers an example csv download + */ + public downloadCsvExample(): void { + this.motionCSVExport.exportDummyMotion(); + } + + /** + * Trigger for the column separator selection + * + * @param event + */ + public selectColSep(event: MatSelectChange): void { + this.importer.columnSeparator = event.value; + this.importer.refreshFile(); + } + + /** + * Trigger for the column separator selection + * + * @param event + */ + public selectTextSep(event: MatSelectChange): void { + this.importer.textSeparator = event.value; + this.importer.refreshFile(); + } + + /** + * Trigger for the encoding selection + * + * @param event + */ + public selectEncoding(event: MatSelectChange): void { + this.importer.encoding = event.value; + this.importer.refreshFile(); + } + + /** + * Returns a descriptive string for an import error + */ + public getVerboseError(error: string): string { + return this.importer.verbose(error); + } +} diff --git a/client/src/app/site/motions/components/motion-list/motion-list.component.html b/client/src/app/site/motions/components/motion-list/motion-list.component.html index 290883a82..f1b8bfd50 100644 --- a/client/src/app/site/motions/components/motion-list/motion-list.component.html +++ b/client/src/app/site/motions/components/motion-list/motion-list.component.html @@ -153,10 +153,15 @@ speaker_notes Comment fields + +
- -