motion csv import

This commit is contained in:
Maximilian Krambach 2018-12-04 19:31:24 +01:00
parent 2f7336b257
commit 060d8c8324
20 changed files with 1566 additions and 16 deletions

View File

@ -43,6 +43,7 @@
"material-design-icons": "^3.0.1",
"ngx-file-drop": "^5.0.0",
"ngx-mat-select-search": "^1.4.2",
"ngx-papaparse": "^3.0.2",
"po2json": "^1.0.0-alpha",
"roboto-fontface": "^0.10.0",
"rxjs": "^6.3.3",

View File

@ -3,6 +3,7 @@ import { BrowserModule } from '@angular/platform-browser';
import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
import { NgModule, APP_INITIALIZER } from '@angular/core';
import { HttpClientModule, HttpClient, HttpClientXsrfModule } from '@angular/common/http';
import { PapaParseModule } from 'ngx-papaparse';
// Elementary App Components
import { AppRoutingModule } from './app-routing.module';
@ -53,7 +54,8 @@ export function AppLoaderFactory(appLoadService: AppLoadService): () => Promise<
}),
AppRoutingModule,
CoreModule,
LoginModule
LoginModule,
PapaParseModule
],
providers: [{ provide: APP_INITIALIZER, useFactory: AppLoaderFactory, deps: [AppLoadService], multi: true }],
bootstrap: [AppComponent]

View File

@ -147,12 +147,12 @@ export class CsvExportService {
if (!tsList.length) {
throw new Error('no usable text separator left for valid csv text');
}
const csvContentAsString: string = csvContent
.map(line => {
return line.map(entry => tsList[0] + entry + tsList[0]).join(columnSeparator);
})
.join(lineSeparator);
this.exporter.saveFile(csvContentAsString, filename);
}
@ -163,7 +163,7 @@ export class CsvExportService {
*
* @param input any input to be sent to CSV
* @param tsList The list of special characters to check.
* @returns the cleand CSV String list
* @returns the cleaned CSV String list
*/
public checkCsvTextSafety(input: string, tsList: string[]): string[] {
if (input === null || input === undefined) {

View File

@ -287,6 +287,21 @@ export class DataStoreService {
return this.getAll<T>(collectionType).filter(callback);
}
/**
* Finds a model item in the dataStore by type.
*
* @param collectionType The desired BaseModel type to be read from the dataStore
* @param callback a find function
* @return The first BaseModel item matching the filter function
* @example this.DS.find<User>(User, myUser => myUser.first_name === "Jenny")
*/
public find<T extends BaseModel<T>>(
collectionType: ModelConstructor<T> | string,
callback: (model: T) => boolean
): T {
return this.getAll<T>(collectionType).find(callback);
}
/**
* Add one or multiple models to dataStore.
*

View File

@ -0,0 +1,263 @@
<os-head-bar [nav]="false">
<!-- Title -->
<div class="title-slot"><h2 translate>Import motions</h2></div>
<div class="menu-slot">
<button *ngIf="hasFile && newCount" mat-button (click)="doImport()">
<span class="upper" translate> Import</span>
</button>
</div>
</os-head-bar>
<mat-card>
<span translate>Requires comma or semicolon separated values with these column header names in the first row</span>:
<br />
<div class="code red-warning-text">
<!-- TODO: class : indent, warning color -->
<span translate>Identifier</span>, <span translate>Title</span>, <span translate>Text</span>,
<span translate>Reason</span>, <span translate>Submitter</span>, <span translate>Category</span>,
<span translate>Origin</span>, <span translate>Motion block</span>
</div>
<ul>
<li translate>
Identifier, reason, submitter, category, origin and motion block are optional and may be empty.
</li>
<li translate>Additional columns after the required ones may be present and won't affect the import.</li>
<li translate>Only double quotes are accepted as text delimiter (no single quotes).</li>
</ul>
<button mat-button color="accent" (click)="downloadCsvExample()" translate>Download CSV example file</button>
<div class="wrapper">
<mat-form-field>
<mat-label translate>Encoding of the file</mat-label>
<mat-select
class="selection"
placeholder="translate.instant('Select encoding')"
(selectionChange)="selectEncoding($event)"
[value]="encodings[0].value"
>
<mat-option *ngFor="let option of encodings" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field>
<mat-label translate> Column Separator</mat-label>
<mat-select class="selection" (selectionChange)="selectColSep($event)" value="">
<mat-option *ngFor="let option of columnSeparators" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
<mat-form-field>
<mat-label translate>Text separator</mat-label>
<mat-select class="selection" (selectionChange)="selectTextSep($event)" value="&quot;">
<mat-option *ngFor="let option of textSeparators" [value]="option.value">
{{ option.label | translate }}
</mat-option>
</mat-select>
</mat-form-field>
</div>
<div>
<div>
<input
id="motion-import-file-input"
type="file"
class="hidden-input"
accept="text"
#fileInput
(change)="onSelectFile($event)"
/>
<button mat-button onclick="document.getElementById('motion-import-file-input').click()">
<span translate> Select file</span>
</button>
</div>
&nbsp; <span *ngIf="hasFile">{{ totalCount }}&nbsp;<span translate>entries found.</span></span>
</div>
</mat-card>
<!-- preview table -->
<mat-card *ngIf="hasFile">
<div class="summary">
<!-- new entries -->
<div *ngIf="newCount">
&nbsp;
<mat-icon inline>playlist_add</mat-icon>
<span>&nbsp;{{ newCount }}&nbsp;</span> <span translate>Motion(s) will be imported.</span>
</div>
<!-- errors/duplicates -->
<div *ngIf="nonImportableCount" class="red-warning-text">
&nbsp;
<mat-icon inline>warning</mat-icon>
<span>&nbsp;{{ nonImportableCount }}&nbsp;</span> <span translate>entries will be ommitted.</span>
</div>
<!-- have been imported -->
<div *ngIf="doneCount" class="green-text">
&nbsp;
<mat-icon inline>done</mat-icon>
<span>&nbsp;{{ doneCount }}&nbsp;</span> <span translate>Motions have been imported.</span>
</div>
</div>
<div>
<mat-select [(value)]="shown" (selectionChange)="setFilter()">
<!-- TODO: reduce item width to sane value -->
<mat-option value="all" translate> Show all </mat-option>
<mat-option *ngIf="nonImportableCount" value="error" translate> Show errors only </mat-option>
<mat-option *ngIf="nonImportableCount" value="noerror" translate> Show correct entries </mat-option>
</mat-select>
<!-- TODO: Button to hide imported ones -->
</div>
<div class="table-container">
<table mat-table class="on-transition-fade" [dataSource]="dataSource" matSort>
<!-- Status column -->
<ng-container matColumnDef="status" sticky>
<mat-header-cell *matHeaderCellDef></mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.status === 'error'">
<mat-icon
class="red-warning-text"
matTooltip="{{ entry.newMotion.errors.length }} + {{ 'errors' | translate }}"
>
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.newMotion.status === 'new'">
<mat-icon matTooltip="{{ 'Motion will be imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.newMotion.status === 'done'">
<mat-icon matTooltip="{{ 'Motion has been imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
</mat-cell>
</ng-container>
<!-- identifier column -->
<ng-container matColumnDef="identifier">
<mat-header-cell *matHeaderCellDef translate>Identifier</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-icon
color="warn"
inline
*ngIf="entry.newMotion.hasError('Duplicates')"
matTooltip="{{ getVerboseError('Duplicates') | translate }}"
>
warning
</mat-icon>
&nbsp;{{ entry.newMotion.identifier }}
</mat-cell>
</ng-container>
<!-- title column -->
<ng-container matColumnDef="title">
<mat-header-cell *matHeaderCellDef translate>Title</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Title')"
matTooltip="{{ getVerboseError('Title') | translate }}"
>
warning
</mat-icon>
&nbsp;{{ entry.newMotion.title }}
</mat-cell>
</ng-container>
<!-- tect column TODO: Bigger-->
<ng-container matColumnDef="text">
<mat-header-cell *matHeaderCellDef translate>Motion text</mat-header-cell>
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newMotion.text) }}">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Text')"
matTooltip="{{ getVerboseError('Text') | translate }}"
>
warning
</mat-icon>
&nbsp;{{ getShortPreview(entry.newMotion.text) }}
</mat-cell>
</ng-container>
<!-- reason column -->
<ng-container matColumnDef="reason">
<mat-header-cell *matHeaderCellDef translate>Reason</mat-header-cell>
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newMotion.reason) }}">
{{ getShortPreview(entry.newMotion.reason) }}
</mat-cell>
</ng-container>
<!-- submitters column -->
<ng-container matColumnDef="submitters">
<mat-header-cell *matHeaderCellDef translate>Submitters</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.csvSubmitters.length">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Submitters')"
matTooltip="{{ getVerboseError('Submitters') | translate }}"
>
warning
</mat-icon>
<span *ngFor="let submitter of entry.newMotion.csvSubmitters">
{{ submitter.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!submitter.id">add</mat-icon>
&nbsp;
</span>
</div>
</mat-cell>
</ng-container>
<!-- category column -->
<ng-container matColumnDef="category">
<mat-header-cell *matHeaderCellDef translate>Category</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.csvCategory">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('Category')"
matTooltip="{{ getVerboseError('Category') | translate }}"
>
warning
</mat-icon>
{{ entry.newMotion.csvCategory.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newMotion.csvCategory.id"
>add</mat-icon
>&nbsp;
</div>
</mat-cell>
</ng-container>
<!-- origin column -->
<ng-container matColumnDef="origin">
<mat-header-cell *matHeaderCellDef translate>Origin</mat-header-cell>
<mat-cell *matCellDef="let entry">{{ entry.newMotion.origin }}</mat-cell>
</ng-container>
<!-- motion block column -->
<ng-container matColumnDef="motion block">
<mat-header-cell *matHeaderCellDef translate>Motion block</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newMotion.csvMotionblock">
<mat-icon
color="warn"
*ngIf="entry.newMotion.hasError('MotionBlock')"
matTooltip="{{ getVerboseError('MotionBlock') | translate }}"
>
warning
</mat-icon>
{{ entry.newMotion.csvMotionblock.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newMotion.csvMotionblock.id">
add
</mat-icon>
&nbsp;
</div>
</mat-cell>
</ng-container>
<mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
<!-- TODO: class for invalid/undecided -->
<mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
</table>
</div>
</mat-card>

View File

@ -0,0 +1,53 @@
.table-container {
width: 100%;
overflow-x: scroll;
margin-top: 5px;
}
table {
width: 100%;
overflow: scroll;
}
.mat-header-cell {
min-width: 100px;
padding-right: 8px;
}
.mat-cell {
min-width: 100px;
padding-top: 2px;
padding-right: 8px;
}
.selection {
min-width: 80px;
}
.import-done {
background-color: #cfc;
}
.import-error {
background-color: #fcc;
}
.code {
padding-left: 1em;
font-style: italic;
}
div.wrapper {
display: flex;
vertical-align: bottom;
padding: 10px;
}
div.summary {
display: flex;
}
.hidden-input {
display: none;
}
.newBadge {
margin-left: -3px;
}

View File

@ -0,0 +1,26 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { MotionImportListComponent } from './motion-import-list.component';
import { E2EImportsModule } from 'e2e-imports.module';
describe('MotionImportListComponent', () => {
let component: MotionImportListComponent;
let fixture: ComponentFixture<MotionImportListComponent>;
beforeEach(async(() => {
TestBed.configureTestingModule({
declarations: [MotionImportListComponent],
imports: [E2EImportsModule]
}).compileComponents();
}));
beforeEach(() => {
fixture = TestBed.createComponent(MotionImportListComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

View File

@ -0,0 +1,308 @@
import { MatTableDataSource, MatTable, MatSnackBar, MatSelectChange } from '@angular/material';
import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core';
import { ViewChild, Component, OnInit } from '@angular/core';
import { BaseViewComponent } from 'app/site/base/base-view';
import { MotionCsvExportService } from '../../services/motion-csv-export.service';
import { MotionImportService, NewMotionEntry, ValueLabelCombination } from '../../services/motion-import.service';
/**
* Component for the motion import list view.
*/
@Component({
selector: 'os-motion-import-list',
templateUrl: './motion-import-list.component.html',
styleUrls: ['./motion-import-list.component.scss']
})
export class MotionImportListComponent extends BaseViewComponent implements OnInit {
/**
* The data source for a table. Requires to be initialised with a BaseViewModel
*/
public dataSource: MatTableDataSource<NewMotionEntry>;
/**
* Switch that turns true if a file has been selected in the input
*/
public hasFile = false;
/**
* Currently selected encoding. Is set and changed by the config's available
* encodings and user mat-select input
*/
public selectedEncoding = 'utf-8';
/**
* indicator on which elements to display
*/
public shown: 'all' | 'error' | 'noerror' = 'all';
/**
* The table itself
*/
@ViewChild(MatTable)
protected table: MatTable<NewMotionEntry>;
/**
* Returns the amount of total item successfully parsed
*/
public get totalCount(): number {
return this.importer && this.hasFile ? this.importer.summary.total : null;
}
/**
* Returns the encodings available and their labels
*/
public get encodings(): ValueLabelCombination[] {
return this.importer.encodings;
}
/**
* Returns the available column separators and their labels
*/
public get columnSeparators(): ValueLabelCombination[] {
return this.importer.columnSeparators;
}
/**
* Returns the available text separators and their labels
*/
public get textSeparators(): ValueLabelCombination[] {
return this.importer.textSeparators;
}
/**
* Returns the amount of import items that will be imported
*/
public get newCount(): number {
return this.importer && this.hasFile ? this.importer.summary.new : 0;
}
/**
* Returns the number of import items that cannot be imported
*/
public get nonImportableCount(): number {
if (this.importer && this.hasFile) {
return this.importer.summary.errors + this.importer.summary.duplicates;
}
return 0;
}
/**
* Returns the number of import items that have been successfully imported
*/
public get doneCount(): number {
return this.importer && this.hasFile ? this.importer.summary.done : 0;
}
/**
* Constructor for list view bases
*
* @param titleService the title serivce
* @param matSnackBar snackbar for displaying errors
* @param translate the translate service
* @param importer: The motion csv import service
* @param motionCSVExport: service for exporting example data
*/
public constructor(
titleService: Title,
matSnackBar: MatSnackBar,
public translate: TranslateService,
private importer: MotionImportService,
private motionCSVExport: MotionCsvExportService
) {
super(titleService, translate, matSnackBar);
this.initTable();
this.importer.errorEvent.subscribe(this.raiseError);
}
/**
* Starts with a clean preview (removing any previously existing import previews)
*/
public ngOnInit(): void {
this.importer.clearPreview();
}
/**
* Initializes the table
*/
public initTable(): void {
this.dataSource = new MatTableDataSource();
this.setFilter();
this.importer.getNewEntries().subscribe(newEntries => {
this.dataSource.data = newEntries;
this.hasFile = newEntries.length > 0;
});
}
/**
* Returns the table column definition. Fetches all headers from
* {@link MotionImportService} and an additional status column
*/
public getColumnDefinition(): string[] {
return ['status'].concat(this.importer.expectedHeader);
}
/**
* triggers the importer's onSelectFile after a file has been chosen
*/
public onSelectFile(event: any): void {
this.importer.onSelectFile(event);
}
/**
* Triggers the importer's import
*/
public async doImport(): Promise<void> {
await this.importer.doImport();
this.setFilter();
}
/**
* Updates and manually triggers the filter function.
* See {@link hidden} for options
* (changed from default mat-table filter)
*/
public setFilter(): void {
this.dataSource.filter = '';
if (this.shown === 'all') {
this.dataSource.filterPredicate = (data, filter) => {
return true;
};
} else if (this.shown === 'noerror') {
this.dataSource.filterPredicate = (data, filter) => {
if (data.newMotion.status === 'done') {
return true;
} else if (!(data.newMotion.status !== 'error') && !data.duplicates.length) {
return true;
}
};
} else if (this.shown === 'error') {
this.dataSource.filterPredicate = (data, filter) => {
if (data.newMotion.errors.length || data.duplicates.length) {
return true;
}
return false;
};
}
this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter
}
/**
* Returns the appropiate css class for a row according to the import state
*
* @param row
*/
public getStateClass(row: NewMotionEntry): string {
switch (row.newMotion.status) {
case 'done':
return 'import-done import-decided';
case 'error':
return 'import-error';
default:
return '';
}
}
/**
* Returns the first characters of a string, for preview purposes
*
* @param input
*/
public getShortPreview(input: string): string {
if (input.length > 50) {
return this.stripHtmlTags(input.substring(0, 47)) + '...';
}
return this.stripHtmlTags(input);
}
/**
* Returns the first and last 150 characters of a string; used within
* tooltips for the preview
*
* @param input
*/
public getLongPreview(input: string): string {
if (input.length < 300) {
return this.stripHtmlTags(input);
}
return (
this.stripHtmlTags(input.substring(0, 147)) +
' [...] ' +
this.stripHtmlTags(input.substring(input.length - 150, input.length))
);
}
/**
* Return the icon for the action of the item
* @param entry
*/
public getActionIcon(entry: NewMotionEntry): string {
switch (entry.newMotion.status) {
case 'error': // no import possible
return 'block';
case 'new': // new item, will be imported
return 'playlist_add';
case 'done': // item has been imported
return 'done';
default:
// fallback: Error
return 'block';
}
}
/**
* Helper to remove html tags from a string.
* CAUTION: It is just a basic "don't show distracting html tags in a
* preview", not an actual tested sanitizer!
* @param inputString
*/
private stripHtmlTags(inputString: string): string {
const regexp = new RegExp(/<[^ ][^<>]*(>|$)/g);
return inputString.replace(regexp, '').trim();
}
/**
* Triggers an example csv download
*/
public downloadCsvExample(): void {
this.motionCSVExport.exportDummyMotion();
}
/**
* Trigger for the column separator selection
*
* @param event
*/
public selectColSep(event: MatSelectChange): void {
this.importer.columnSeparator = event.value;
this.importer.refreshFile();
}
/**
* Trigger for the column separator selection
*
* @param event
*/
public selectTextSep(event: MatSelectChange): void {
this.importer.textSeparator = event.value;
this.importer.refreshFile();
}
/**
* Trigger for the encoding selection
*
* @param event
*/
public selectEncoding(event: MatSelectChange): void {
this.importer.encoding = event.value;
this.importer.refreshFile();
}
/**
* Returns a descriptive string for an import error
*/
public getVerboseError(error: string): string {
return this.importer.verbose(error);
}
}

View File

@ -153,10 +153,15 @@
<mat-icon>speaker_notes</mat-icon>
<span translate>Comment fields</span>
</button>
<button mat-menu-item (click)="csvExportMotionList()">
<mat-icon>archive</mat-icon>
<span translate>Export as CSV</span>
</button>
<button mat-menu-item *osPerms="'motions.can_manage'" routerLink="import">
<mat-icon>save_alt</mat-icon>
<span translate>Import</span><span>&nbsp;...</span>
</button>
</div>
<div *ngIf="isMultiSelect">
<button mat-menu-item (click)="selectAll()">

View File

@ -7,6 +7,10 @@ import { Motion } from 'app/shared/models/motions/motion';
export class CreateMotion extends Motion {
public submitters_id: number[];
public category_id: number;
public motion_block_id: number;
public constructor(input?: any) {
super(input);
}

View File

@ -0,0 +1,180 @@
import { ViewCreateMotion } from './view-create-motion';
import { CreateMotion } from './create-motion';
/**
* Interface for imported secondary data. A name can be matched to an existing
* model instance by the solve... functions.
* TODO MultiId will be filled if there is more than one match (to be used in case of 'I want to select one of these matches)
*/
export interface CsvMapping {
name: string;
id?: number;
multiId?: number[];
}
type CsvImportStatus = 'new' | 'error' | 'done';
/**
* Create motion class for the View. Its different to ViewMotion in fact that the submitter handling is different
* on motion creation.
*
* @ignore
*/
export class ViewCsvCreateMotion extends ViewCreateMotion {
protected _motion: CreateMotion;
/**
* Mapping for a new/existing category.
*/
public csvCategory: CsvMapping;
/**
* Mapping for a new/existing motion block.
*/
public csvMotionblock: CsvMapping;
/**
* Mapping for new/existing submitters.
*/
public csvSubmitters: CsvMapping[];
/**
* The current import status of this motion.
* Starts as 'new', if set to 'done', a proper {@link Motion} model will
* probably exist in the dataStore. error status will be set if the import
* cannot be done
*/
private _status: CsvImportStatus = 'new';
/**
* list of import errors See {@link MotionImportService}
*/
public errors: string[] = [];
/**
* Returns the current status.
*/
public get status(): CsvImportStatus {
return this._status;
}
public set status(status: CsvImportStatus) {
this._status = status;
}
public get motion(): CreateMotion {
return this._motion;
}
public constructor(motion?: CreateMotion) {
super(motion);
}
/**
* Duplicate this motion into a copy of itself
*/
public copy(): ViewCreateMotion {
return new ViewCreateMotion(
this._motion,
this._category,
this._submitters,
this._supporters,
this._workflow,
this._state
);
}
/**
* Checks if a given error is present. TODO: Is more a ViewModel option
*
* @param error
*/
public hasError(error: string): boolean {
return this.errors.includes(error);
}
/**
* Toggle to set a CreateMotion to a 'successfully parsed' status
*/
public done(): void {
this._status = 'done';
}
/**
* takes a list of motion block mappings to update the current csvMotionblock.
* Returns the amount of entries that remain unmatched
*
* @param motionBlocks
*/
public solveMotionBlocks(motionBlocks: CsvMapping[]): number {
if (!this.csvMotionblock) {
return 0;
} else if (this.csvMotionblock.id) {
this.motion.motion_block_id = this.csvMotionblock.id;
return 0;
} else {
const newBlock = motionBlocks.find(newMotionBlock => newMotionBlock.name === this.csvMotionblock.name);
if (newBlock) {
this.csvMotionblock = newBlock;
this.motion.motion_block_id = newBlock.id;
return 0;
} else {
return 1;
}
}
}
/**
* takes a list of category maps to update the current csv_category.
* Returns the amount of entries that remain unmatched
*
* @param categories
*/
public solveCategory(categories: CsvMapping[]): number {
if (!this.csvCategory) {
return 0;
} else if (this.csvCategory.id) {
this.motion.category_id = this.csvCategory.id;
return 0;
} else {
const newCat = categories.find(newCategory => newCategory.name === this.csvCategory.name);
if (newCat) {
this.csvCategory = newCat;
this.motion.category_id = newCat.id;
return 0;
} else {
return 1;
}
}
}
/**
* takes a list of solved submitter maps to update. Returns the amount of
* entries that remain unmatched
*
* @param submitters
*/
public solveSubmitters(submitters: CsvMapping[]): number {
let open = 0;
const ids: number[] = [];
this.csvSubmitters.forEach(csvSubmitter => {
if (csvSubmitter.id) {
ids.push(csvSubmitter.id);
return;
}
if (!submitters.length) {
open += 1;
return;
}
const mapped = submitters.find(newSubmitter => newSubmitter.name === csvSubmitter.name);
if (mapped) {
csvSubmitter.id = mapped.id;
ids.push(mapped.id);
} else {
open += 1;
}
});
this.motion.submitters_id = ids;
return open;
}
}

View File

@ -1,15 +1,17 @@
import { NgModule } from '@angular/core';
import { Routes, RouterModule } from '@angular/router';
import { MotionListComponent } from './components/motion-list/motion-list.component';
import { MotionDetailComponent } from './components/motion-detail/motion-detail.component';
import { CategoryListComponent } from './components/category-list/category-list.component';
import { MotionCommentSectionListComponent } from './components/motion-comment-section-list/motion-comment-section-list.component';
import { StatuteParagraphListComponent } from './components/statute-paragraph-list/statute-paragraph-list.component';
import { SpeakerListComponent } from '../agenda/components/speaker-list/speaker-list.component';
import { CallListComponent } from './components/call-list/call-list.component';
import { AmendmentCreateWizardComponent } from './components/amendment-create-wizard/amendment-create-wizard.component';
import { CallListComponent } from './components/call-list/call-list.component';
import { CategoryListComponent } from './components/category-list/category-list.component';
import { MotionBlockListComponent } from './components/motion-block-list/motion-block-list.component';
import { MotionBlockDetailComponent } from './components/motion-block-detail/motion-block-detail.component';
import { MotionCommentSectionListComponent } from './components/motion-comment-section-list/motion-comment-section-list.component';
import { MotionDetailComponent } from './components/motion-detail/motion-detail.component';
import { MotionImportListComponent } from './components/motion-import-list/motion-import-list.component';
import { MotionListComponent } from './components/motion-list/motion-list.component';
import { SpeakerListComponent } from '../agenda/components/speaker-list/speaker-list.component';
import { StatuteParagraphListComponent } from './components/statute-paragraph-list/statute-paragraph-list.component';
const routes: Routes = [
{ path: '', component: MotionListComponent },
@ -20,6 +22,7 @@ const routes: Routes = [
{ path: 'blocks', component: MotionBlockListComponent },
{ path: 'blocks/:id', component: MotionBlockDetailComponent },
{ path: 'new', component: MotionDetailComponent },
{ path: 'import', component: MotionImportListComponent },
{ path: ':id', component: MotionDetailComponent },
{ path: ':id/speakers', component: SpeakerListComponent },
{ path: ':id/create-amendment', component: AmendmentCreateWizardComponent }

View File

@ -18,6 +18,7 @@ import { CallListComponent } from './components/call-list/call-list.component';
import { AmendmentCreateWizardComponent } from './components/amendment-create-wizard/amendment-create-wizard.component';
import { MotionBlockListComponent } from './components/motion-block-list/motion-block-list.component';
import { MotionBlockDetailComponent } from './components/motion-block-detail/motion-block-detail.component';
import { MotionImportListComponent } from './components/motion-import-list/motion-import-list.component';
import { ManageSubmittersComponent } from './components/manage-submitters/manage-submitters.component';
@NgModule({
@ -38,6 +39,7 @@ import { ManageSubmittersComponent } from './components/manage-submitters/manage
AmendmentCreateWizardComponent,
MotionBlockListComponent,
MotionBlockDetailComponent,
MotionImportListComponent,
ManageSubmittersComponent
],
entryComponents: [

View File

@ -112,4 +112,13 @@ export class MotionBlockRepositoryService extends BaseRepository<ViewMotionBlock
.getViewModelListObservable()
.pipe(map(viewMotions => viewMotions.filter(viewMotion => viewMotion.motion_block_id === block.id)));
}
/**
* Retrieves motion block(s) by name
* TODO: check if a title is unique for a motionBlock
* @param title Strign to check for
*/
public getMotionBlockByTitle(title: string): MotionBlock {
return this.DS.find(MotionBlock, block => block.title === title);
}
}

View File

@ -4,6 +4,7 @@ import { TranslateService } from '@ngx-translate/core';
import { CsvExportService } from 'app/core/services/csv-export.service';
import { ViewMotion } from '../models/view-motion';
import { FileExportService } from 'app/core/services/file-export.service';
/**
* Exports CSVs for motions. Collect all CSV types here to have them in one place.
@ -18,7 +19,11 @@ export class MotionCsvExportService {
* @param csvExport CsvExportService
* @param translate TranslateService
*/
public constructor(private csvExport: CsvExportService, private translate: TranslateService) {}
public constructor(
private csvExport: CsvExportService,
private translate: TranslateService,
private fileExport: FileExportService
) {}
/**
* Export all motions as CSV
@ -64,4 +69,17 @@ export class MotionCsvExportService {
this.translate.instant('Call list') + '.csv'
);
}
public exportDummyMotion(): void {
const headerRow = ['Identifier', 'Title', 'Text', 'Reason', 'Submitters', 'Category', 'Origin', 'Motion block']
.map(item => this.translate.instant(item))
.join(',');
const rows = [
headerRow,
'A1,Title 1,Text 1,Reason 1,Submitter A,Category A,"Last Year Conference A", Block A',
'B1,Title 2,Text 2,Reason 2,Submitter B, Category B,, Block A',
',Title 3, Text 3,,,,,'
];
this.fileExport.saveFile(rows.join('\n'), this.translate.instant('Motion example') + '.csv');
}
}

View File

@ -0,0 +1,599 @@
import { BehaviorSubject, Observable } from 'rxjs';
import { Injectable, EventEmitter } from '@angular/core';
import { MatSnackBar } from '@angular/material';
import { Papa, PapaParseConfig } from 'ngx-papaparse';
import { TranslateService } from '@ngx-translate/core';
import { Category } from 'app/shared/models/motions/category';
import { CategoryRepositoryService } from './category-repository.service';
import { CreateMotion } from '../models/create-motion';
import { MotionBlock } from 'app/shared/models/motions/motion-block';
import { MotionBlockRepositoryService } from './motion-block-repository.service';
import { MotionRepositoryService } from './motion-repository.service';
import { UserRepositoryService } from '../../users/services/user-repository.service';
import { ViewMotion } from '../models/view-motion';
import { ViewCsvCreateMotion, CsvMapping } from '../models/view-csv-create-motion';
/**
* Interface for value- Label combinations.
* Map objects didn't work, TODO: Use map objects (needs iterating through all objects of a map)
*/
export interface ValueLabelCombination {
value: string;
label: string;
}
/**
* Interface for a new Motion and their (if any) duplicates
*/
export interface NewMotionEntry {
newMotion: ViewCsvCreateMotion;
duplicates: ViewMotion[];
}
/**
* interface for a preview summary
*/
interface ImportMotionCSVPreview {
total: number;
duplicates: number;
errors: number;
new: number;
done: number;
}
/**
* List of possible import errors specific for motion imports.
*/
const errorList = {
MotionBlock: 'Could not resolve the motion block',
Category: 'Could not resolve the category',
Submitters: 'Could not resolve the submitters',
Title: 'A title is required',
Text: "A content in the 'text' column is required",
Duplicates: 'A motion with this identifier already exists.',
generic: 'Server upload failed' // TODO
};
/**
* Service for motion imports
*/
@Injectable({
providedIn: 'root'
})
export class MotionImportService {
/** The header (order and items) that is expected from the imported file
*
*/
public expectedHeader = [
'identifier',
'title',
'text',
'reason',
'submitters',
'category',
'origin',
'motion block'
];
/**
* The last parsed file object (may be reparsed with new encoding, thus kept in memory)
*/
private _rawFile: File;
/**
* The used column Separator. If left on an empty string (default),
* the papaparse parser will automatically decide on separators.
*/
public columnSeparator = '';
public textSeparator = '"';
public encoding = 'utf-8';
/**
* List of possible encodings and their label
*/
public encodings: ValueLabelCombination[] = [
{ value: 'utf-8', label: 'UTF 8 - Unicode' },
{ value: 'iso-8859-1', label: 'ISO 8859-1 - West European' },
{ value: 'iso-8859-15', label: 'ISO 8859-15 - West European (with €)' }
];
/**
* List of possible column separators
*/
public columnSeparators: ValueLabelCombination[] = [
{ label: 'Comma', value: ',' },
{ label: 'Semicolon', value: ';' },
// {label: 'Tabulator', value: '\t'},
{ label: 'Automatic', value: '' }
];
public textSeparators: ValueLabelCombination[] = [
{ label: 'Double quotes (")', value: '"' },
{ label: "Single quotes (')", value: "'" }
];
/**
* submitters that need to be created prior to importing
*/
public newSubmitters: CsvMapping[] = [];
/**
* Categories that need to be created prior to importing
*/
public newCategories: CsvMapping[] = [];
/**
* MotionBlocks that need to be created prior to importing
*/
public newMotionBlocks: CsvMapping[] = [];
/**
* FileReader object for file import
*/
private reader = new FileReader();
/**
* the list of parsed models that have been extracted from the opened file
*/
private _entries: NewMotionEntry[] = [];
/**
* BehaviorSubject for displaying a preview for the currently selected entries
*/
public newEntries = new BehaviorSubject<NewMotionEntry[]>([]);
/**
* Emits an error string to display if a file import cannot be done
*/
public errorEvent = new EventEmitter<string>();
/**
* storing the summary preview for the import, to avoid recalculating it
* at each display change.
*/
private _preview: ImportMotionCSVPreview;
/**
* Returns a summary on actions that will be taken/not taken.
*/
public get summary(): ImportMotionCSVPreview {
if (!this._preview) {
this.updatePreview();
}
return this._preview;
}
/**
* Constructor. Creates a fileReader to subscribe to it for incoming parsed
* strings
* @param categoryRepo Repository to fetch pre-existing categories
* @param motionBlockRepo Repository to fetch pre-existing motionBlocks
* @param userRepo Repository to query/ create users
* @param translate Translation service
* @param papa External csv parser (ngx-papaparser)
* @param matSnackBar snackBar to display import errors
*/
public constructor(
private repo: MotionRepositoryService,
private categoryRepo: CategoryRepositoryService,
private motionBlockRepo: MotionBlockRepositoryService,
private userRepo: UserRepositoryService,
private translate: TranslateService,
private papa: Papa,
private matSnackbar: MatSnackBar
) {
this.reader.onload = (event: any) => {
// TODO type: event is a progressEvent,
// but has a property target.result, which typescript doesn't recognize
this.parseInput(event.target.result);
};
}
/**
* Parses the data input. Expects a string as returned by via a
* File.readAsText() operation
*
* @param file
*/
public parseInput(file: string): void {
this._entries = [];
this.newSubmitters = [];
this.newCategories = [];
this.newMotionBlocks = [];
const papaConfig: PapaParseConfig = {
header: false,
skipEmptyLines: true,
quoteChar: this.textSeparator
};
if (this.columnSeparator) {
papaConfig.delimiter = this.columnSeparator;
}
const entryLines = this.papa.parse(file, papaConfig).data;
const valid = this.checkHeader(entryLines.shift());
if (!valid) {
return;
}
entryLines.forEach(line => {
const newMotion = new ViewCsvCreateMotion(new CreateMotion());
const headerLength = Math.min(this.expectedHeader.length, line.length);
for (let idx = 0; idx < headerLength; idx++) {
// iterate over items, find existing ones (thier id) and collect new entries
switch (this.expectedHeader[idx]) {
case 'submitters':
newMotion.csvSubmitters = this.getSubmitters(line[idx]);
break;
case 'category':
newMotion.csvCategory = this.getCategory(line[idx]);
break;
case 'motion block':
newMotion.csvMotionblock = this.getMotionBlock(line[idx]);
break;
default:
newMotion.motion[this.expectedHeader[idx]] = line[idx];
}
}
const updateModels = this.getDuplicates(newMotion.motion);
if (updateModels.length) {
this.setError(newMotion, 'Duplicates');
}
this._entries.push({ newMotion: newMotion, duplicates: updateModels });
});
this.newEntries.next(this._entries);
this.updatePreview();
}
/**
* Triggers the import.
*/
public async doImport(): Promise<void> {
this.newMotionBlocks = await this.createNewMotionBlocks();
this.newCategories = await this.createNewCategories();
this.newSubmitters = await this.createNewUsers();
for (const entry of this._entries) {
if (entry.newMotion.status !== 'new') {
continue;
}
const openBlocks = entry.newMotion.solveMotionBlocks(this.newMotionBlocks);
if (openBlocks) {
this.setError(entry.newMotion, 'MotionBlock');
// TODO error handling if not all submitters could be matched
this.updatePreview();
continue;
}
const openCategories = entry.newMotion.solveCategory(this.newCategories);
if (openCategories) {
this.setError(entry.newMotion, 'Category');
this.updatePreview();
continue;
}
const openUsers = entry.newMotion.solveSubmitters(this.newSubmitters);
if (openUsers) {
this.setError(entry.newMotion, 'Submitters');
this.updatePreview();
continue;
}
await this.repo.create(entry.newMotion.motion);
entry.newMotion.done();
}
this.updatePreview();
}
/**
* Checks the dataStore for duplicates
* @returns an array of duplicates with the same identifier.
* @param motion
*/
public getDuplicates(motion: CreateMotion): ViewMotion[] {
return this.repo.getMotionDuplicates(motion);
}
/**
* counts the amount of duplicates that have no decision on the action to
* be taken
*/
public updatePreview(): void {
const summary = {
total: 0,
new: 0,
duplicates: 0,
errors: 0,
done: 0
};
this._entries.forEach(entry => {
summary.total += 1;
if (entry.newMotion.status === 'done') {
summary.done += 1;
return;
} else if (entry.newMotion.status === 'error' && !entry.duplicates.length) {
// errors that are not due to duplicates
summary.errors += 1;
return;
} else if (entry.duplicates.length) {
summary.duplicates += 1;
return;
} else if (entry.newMotion.status === 'new') {
summary.new += 1;
}
});
this._preview = summary;
}
/**
* returns a subscribable representation of the new Users to be imported
*/
public getNewEntries(): Observable<NewMotionEntry[]> {
return this.newEntries.asObservable();
}
/**
* Checks the provided submitter(s) and returns an object with mapping of
* existing users and of users that need to be created
* @param submitterlist
*/
public getSubmitters(submitterlist: string): CsvMapping[] {
const result: CsvMapping[] = [];
if (!submitterlist) {
return result;
}
const submitterArray = submitterlist.split(','); // TODO fails with 'full name'
for (const submitter of submitterArray) {
const existingSubmitters = this.userRepo.getUsersByName(submitter);
if (!existingSubmitters.length) {
if (!this.newSubmitters.find(listedSubmitter => listedSubmitter.name === submitter)) {
this.newSubmitters.push({ name: submitter });
}
result.push({ name: submitter });
}
if (existingSubmitters.length === 1) {
result.push({
name: existingSubmitters[0].short_name,
id: existingSubmitters[0].id
});
}
if (existingSubmitters.length > 1) {
result.push({
name: submitter,
multiId: existingSubmitters.map(ex => ex.id)
});
this.matSnackbar.open('TODO: multiple possible users found for this string', 'ok');
// TODO How to handle several submitters ? Is this possible?
// should have some kind of choice dialog there
}
}
return result;
}
/**
* Checks the provided category/ies and returns a mapping, expands
* newCategories if needed.
*
* The assumption is that there may or not be a prefix wit up to 5
* characters at the beginning, separated by ' - ' from the name.
* It will also accept a registered translation between the current user's
* language and english
* @param categoryString
*/
public getCategory(categoryString: string): CsvMapping {
if (!categoryString) {
return null;
}
const category = this.splitCategoryString(categoryString);
const existingCategory = this.categoryRepo.getViewModelList().find(cat => {
if (category.prefix && cat.prefix !== category.prefix) {
return false;
}
if (cat.name === category.name) {
return true;
}
if (this.translate.instant(cat.name) === category.name) {
return true;
}
return false;
});
if (existingCategory) {
return {
name: existingCategory.prefixedName,
id: existingCategory.id
};
} else {
if (!this.newCategories.find(newCat => newCat.name === categoryString)) {
this.newCategories.push({ name: categoryString });
}
return { name: categoryString };
}
}
/**
* Checks the motionBlock provided in the string for existance, expands newMotionBlocks
* if needed. Note that it will also check for translation between the current
* user's language and english
* @param blockString
*/
public getMotionBlock(blockString: string): CsvMapping {
if (!blockString) {
return null;
}
blockString = blockString.trim();
let existingBlock = this.motionBlockRepo.getMotionBlockByTitle(blockString);
if (!existingBlock) {
existingBlock = this.motionBlockRepo.getMotionBlockByTitle(this.translate.instant(blockString));
}
if (existingBlock) {
return { id: existingBlock.id, name: existingBlock.title };
} else {
if (!this.newMotionBlocks.find(newBlock => newBlock.name === blockString)) {
this.newMotionBlocks.push({ name: blockString });
}
return { name: blockString };
}
}
/**
* Creates all new Users needed for the import.
*/
private async createNewUsers(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
for (const user of this.newSubmitters) {
promises.push(this.userRepo.createFromString(user.name));
}
return await Promise.all(promises);
}
/**
* Creates all new Motion Blocks needed for the import.
*/
private async createNewMotionBlocks(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
for (const block of this.newMotionBlocks) {
promises.push(
this.motionBlockRepo.create(new MotionBlock({ title: block.name })).then(identifiable => {
return { name: block.name, id: identifiable.id };
})
);
}
return await Promise.all(promises);
}
/**
* Creates all new Categories needed for the import.
*/
private async createNewCategories(): Promise<CsvMapping[]> {
const promises: Promise<CsvMapping>[] = [];
for (const category of this.newCategories) {
const cat = this.splitCategoryString(category.name);
promises.push(
this.categoryRepo
.create(
new Category({
name: cat.name,
prefix: cat.prefix ? cat.prefix : null
})
)
.then(identifiable => {
return { name: category.name, id: identifiable.id };
})
);
}
return await Promise.all(promises);
}
/**
* Handler after a file was selected. Basic checking for type, then hand
* over to parsing
*
* @param event type is Event, but has target.files, which typescript doesn't seem to recognize
*/
public onSelectFile(event: any): void {
// TODO type
if (event.target.files && event.target.files.length === 1) {
if (event.target.files[0].type === 'text/csv') {
this._rawFile = event.target.files[0];
this.readFile(event.target.files[0]);
} else {
this.matSnackbar.open(this.translate.instant('Wrong file type detected. Import failed.'), '', {
duration: 3000
});
this.clearPreview();
this._rawFile = null;
}
}
}
/**
* Rereads the (previously selected) file, if present. Thought to be triggered
* by parameter changes on encoding, column, text separators
*/
public refreshFile(): void {
if (this._rawFile) {
this.readFile(this._rawFile);
}
}
/**
* (re)-reads a given file with the current parameter
*/
private readFile(file: File): void {
this.reader.readAsText(file, this.encoding);
}
/**
* Checks the first line of the csv (the header) for consistency (length)
* @param row expected to be an array parsed from the first line of a csv file
*/
private checkHeader(row: string[]): boolean {
const snackbarDuration = 3000;
if (row.length < 4) {
this.matSnackbar.open(this.translate.instant('The file has too few columns to be parsed properly.'), '', {
duration: snackbarDuration
});
this.clearPreview();
return false;
} else if (row.length < this.expectedHeader.length) {
this.matSnackbar.open(
this.translate.instant('The file seems to have some ommitted columns. They will be considered empty.'),
'',
{ duration: snackbarDuration }
);
} else if (row.length > this.expectedHeader.length) {
this.matSnackbar.open(
this.translate.instant('The file seems to have additional columns. They will be ignored.'),
'',
{ duration: snackbarDuration }
);
}
return true;
}
/**
* Resets the data and preview (triggered upon selecting an invalid file)
*/
public clearPreview(): void {
this._entries = [];
this.newEntries.next([]);
this._preview = null;
}
/**
* set a list of short names for error, indicating which column failed
*/
public setError(motion: ViewCsvCreateMotion, error: string): void {
if (errorList.hasOwnProperty(error) && !motion.errors.includes(error)) {
motion.errors.push(error);
motion.status = 'error';
}
}
/**
* Get an extended error description.
* @param error
*/
public verbose(error: string): string {
return errorList[error];
}
/**
* Helper to separate a category string from its' prefix. Assumes that a prefix is no longer
* than 5 chars and separated by a ' - '
* @param categoryString the string to parse
*/
private splitCategoryString(categoryString: string): { prefix: string; name: string } {
let prefixSeparator = ' - ';
if (categoryString.startsWith(prefixSeparator)) {
prefixSeparator = prefixSeparator.substring(1);
}
categoryString = categoryString.trim();
let prefix = '';
const separatorIndex = categoryString.indexOf(prefixSeparator);
if (separatorIndex >= 0 && separatorIndex < 6) {
prefix = categoryString.substring(0, separatorIndex);
categoryString = categoryString.substring(separatorIndex + prefixSeparator.length);
}
return { prefix: prefix, name: categoryString };
}
}

View File

@ -149,6 +149,7 @@ export class MotionRepositoryService extends BaseRepository<ViewMotion, Motion>
* @param viewMotion The View Motion. If not present, a new motion will be created
*/
public async create(motion: CreateMotion): Promise<Identifiable> {
// TODO how to handle category id and motion_block id in CreateMotion?
return await this.dataSend.createModel(motion);
}
@ -618,4 +619,17 @@ export class MotionRepositoryService extends BaseRepository<ViewMotion, Motion>
)
.filter((para: ViewMotionAmendedParagraph) => para !== null);
}
/**
* Returns all Motion duplicates (sharing specific values given in input)
* @param viewMotion the ViewMotion to compare against the list of Motions
* in the data
* @param sharedValues properties that must be equal to consider it a duplicate
*/
public getMotionDuplicates(motion: Motion): ViewMotion[] {
const duplicates = this.DS.filter(Motion, item => motion.identifier === item.identifier);
const viewMotions: ViewMotion[] = [];
duplicates.forEach(item => viewMotions.push(this.createViewModel(item)));
return viewMotions;
}
}

View File

@ -86,11 +86,6 @@
<span translate>Groups</span>
</button>
<button mat-menu-item>
<mat-icon>save_alt</mat-icon>
<span translate>Import ...</span>
</button>
<button mat-menu-item (click)="csvExportUserList()">
<mat-icon>archive</mat-icon>
<span translate>Export as CSV</span>

View File

@ -1,4 +1,5 @@
import { Injectable } from '@angular/core';
import { BaseRepository } from '../../base/base-repository';
import { ViewUser } from '../models/view-user';
import { User } from '../../../shared/models/users/user';
@ -197,4 +198,48 @@ export class UserRepositoryService extends BaseRepository<ViewUser, User> {
return msg;
}
/**
* Searches and returns Users by full name
* @param name
*/
public getUsersByName(name: string): ViewUser[] {
const results: ViewUser[] = [];
const users = this.DS.getAll(User).filter(user => {
if (user.full_name === name || user.short_name === name) {
return true;
}
if (user.number === name) {
return true;
}
return false;
});
users.forEach(user => {
results.push(this.createViewModel(user));
});
return results;
}
/**
* Creates a new User from a string
* @param user: String to create the user from
* TODO: return 'user' + new id
*/
public async createFromString(user: string): Promise<{ id: number; name: string }> {
const splitUser = user.split(' ');
const newUser: Partial<User> = {};
switch (splitUser.length) {
case 1:
newUser.first_name = splitUser[0];
break;
case 2:
newUser.first_name = splitUser[0];
newUser.last_name = splitUser[1];
break;
default:
newUser.first_name = user;
}
const createdUser = await this.create(newUser);
return { id: createdUser.id, name: user };
}
}

View File

@ -94,6 +94,14 @@ body {
}
}
.accent-text {
color: mat-color($openslides-accent);
}
.green-text { // TODO better name/theming
color: #5A5;
}
.icon-text-distance {
margin-left: 5px;
}