Merge pull request #4076 from MaximilianKrambach/csvimport/motions
motion csv import
This commit is contained in:
Sean
GitHub
commit
7226d04106
@ -43,6 +43,7 @@
|
||||
"material-design-icons": "^3.0.1",
|
||||
"ngx-file-drop": "^5.0.0",
|
||||
"ngx-mat-select-search": "^1.4.2",
|
||||
"ngx-papaparse": "^3.0.2",
|
||||
"po2json": "^1.0.0-alpha",
|
||||
"roboto-fontface": "^0.10.0",
|
||||
"rxjs": "^6.3.3",
|
||||
|
||||
@ -3,6 +3,7 @@ import { BrowserModule } from '@angular/platform-browser';
|
||||
import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
|
||||
import { NgModule, APP_INITIALIZER } from '@angular/core';
|
||||
import { HttpClientModule, HttpClient, HttpClientXsrfModule } from '@angular/common/http';
|
||||
import { PapaParseModule } from 'ngx-papaparse';
|
||||
|
||||
// Elementary App Components
|
||||
import { AppRoutingModule } from './app-routing.module';
|
||||
@ -53,7 +54,8 @@ export function AppLoaderFactory(appLoadService: AppLoadService): () => Promise<
|
||||
}),
|
||||
AppRoutingModule,
|
||||
CoreModule,
|
||||
LoginModule
|
||||
LoginModule,
|
||||
PapaParseModule
|
||||
],
|
||||
providers: [{ provide: APP_INITIALIZER, useFactory: AppLoaderFactory, deps: [AppLoadService], multi: true }],
|
||||
bootstrap: [AppComponent]
|
||||
|
||||
@ -147,12 +147,12 @@ export class CsvExportService {
|
||||
if (!tsList.length) {
|
||||
throw new Error('no usable text separator left for valid csv text');
|
||||
}
|
||||
|
||||
const csvContentAsString: string = csvContent
|
||||
.map(line => {
|
||||
return line.map(entry => tsList[0] + entry + tsList[0]).join(columnSeparator);
|
||||
})
|
||||
.join(lineSeparator);
|
||||
|
||||
this.exporter.saveFile(csvContentAsString, filename);
|
||||
}
|
||||
|
||||
@ -163,7 +163,7 @@ export class CsvExportService {
|
||||
*
|
||||
* @param input any input to be sent to CSV
|
||||
* @param tsList The list of special characters to check.
|
||||
* @returns the cleand CSV String list
|
||||
* @returns the cleaned CSV String list
|
||||
*/
|
||||
public checkCsvTextSafety(input: string, tsList: string[]): string[] {
|
||||
if (input === null || input === undefined) {
|
||||
|
||||
@ -287,6 +287,21 @@ export class DataStoreService {
|
||||
return this.getAll<T>(collectionType).filter(callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds a model item in the dataStore by type.
|
||||
*
|
||||
* @param collectionType The desired BaseModel type to be read from the dataStore
|
||||
* @param callback a find function
|
||||
* @return The first BaseModel item matching the filter function
|
||||
* @example this.DS.find<User>(User, myUser => myUser.first_name === "Jenny")
|
||||
*/
|
||||
public find<T extends BaseModel<T>>(
|
||||
collectionType: ModelConstructor<T> | string,
|
||||
callback: (model: T) => boolean
|
||||
): T {
|
||||
return this.getAll<T>(collectionType).find(callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add one or multiple models to dataStore.
|
||||
*
|
||||
|
||||
@ -0,0 +1,263 @@
|
||||
<os-head-bar [nav]="false">
|
||||
<!-- Title -->
|
||||
<div class="title-slot"><h2 translate>Import motions</h2></div>
|
||||
|
||||
<div class="menu-slot">
|
||||
<button *ngIf="hasFile && newCount" mat-button (click)="doImport()">
|
||||
<span class="upper" translate> Import</span>
|
||||
</button>
|
||||
</div>
|
||||
</os-head-bar>
|
||||
|
||||
<mat-card>
|
||||
<span translate>Requires comma or semicolon separated values with these column header names in the first row</span>:
|
||||
<br />
|
||||
<div class="code red-warning-text">
|
||||
<!-- TODO: class : indent, warning color -->
|
||||
<span translate>Identifier</span>, <span translate>Title</span>, <span translate>Text</span>,
|
||||
<span translate>Reason</span>, <span translate>Submitter</span>, <span translate>Category</span>,
|
||||
<span translate>Origin</span>, <span translate>Motion block</span>
|
||||
</div>
|
||||
<ul>
|
||||
<li translate>
|
||||
Identifier, reason, submitter, category, origin and motion block are optional and may be empty.
|
||||
</li>
|
||||
<li translate>Additional columns after the required ones may be present and won't affect the import.</li>
|
||||
<li translate>Only double quotes are accepted as text delimiter (no single quotes).</li>
|
||||
</ul>
|
||||
<button mat-button color="accent" (click)="downloadCsvExample()" translate>Download CSV example file</button>
|
||||
<div class="wrapper">
|
||||
<mat-form-field>
|
||||
<mat-label translate>Encoding of the file</mat-label>
|
||||
<mat-select
|
||||
class="selection"
|
||||
placeholder="translate.instant('Select encoding')"
|
||||
(selectionChange)="selectEncoding($event)"
|
||||
[value]="encodings[0].value"
|
||||
>
|
||||
<mat-option *ngFor="let option of encodings" [value]="option.value">
|
||||
{{ option.label | translate }}
|
||||
</mat-option>
|
||||
</mat-select>
|
||||
</mat-form-field>
|
||||
<mat-form-field>
|
||||
<mat-label translate> Column Separator</mat-label>
|
||||
<mat-select class="selection" (selectionChange)="selectColSep($event)" value="">
|
||||
<mat-option *ngFor="let option of columnSeparators" [value]="option.value">
|
||||
{{ option.label | translate }}
|
||||
</mat-option>
|
||||
</mat-select>
|
||||
</mat-form-field>
|
||||
<mat-form-field>
|
||||
<mat-label translate>Text separator</mat-label>
|
||||
<mat-select class="selection" (selectionChange)="selectTextSep($event)" value=""">
|
||||
<mat-option *ngFor="let option of textSeparators" [value]="option.value">
|
||||
{{ option.label | translate }}
|
||||
</mat-option>
|
||||
</mat-select>
|
||||
</mat-form-field>
|
||||
</div>
|
||||
<div>
|
||||
<div>
|
||||
<input
|
||||
id="motion-import-file-input"
|
||||
type="file"
|
||||
class="hidden-input"
|
||||
accept="text"
|
||||
#fileInput
|
||||
(change)="onSelectFile($event)"
|
||||
/>
|
||||
<button mat-button onclick="document.getElementById('motion-import-file-input').click()">
|
||||
<span translate> Select file</span>
|
||||
</button>
|
||||
</div>
|
||||
<span *ngIf="hasFile">{{ totalCount }} <span translate>entries found.</span></span>
|
||||
</div>
|
||||
</mat-card>
|
||||
|
||||
<!-- preview table -->
|
||||
<mat-card *ngIf="hasFile">
|
||||
<div class="summary">
|
||||
<!-- new entries -->
|
||||
<div *ngIf="newCount">
|
||||
|
||||
<mat-icon inline>playlist_add</mat-icon>
|
||||
<span> {{ newCount }} </span> <span translate>Motion(s) will be imported.</span>
|
||||
</div>
|
||||
<!-- errors/duplicates -->
|
||||
<div *ngIf="nonImportableCount" class="red-warning-text">
|
||||
|
||||
<mat-icon inline>warning</mat-icon>
|
||||
<span> {{ nonImportableCount }} </span> <span translate>entries will be ommitted.</span>
|
||||
</div>
|
||||
<!-- have been imported -->
|
||||
<div *ngIf="doneCount" class="green-text">
|
||||
|
||||
<mat-icon inline>done</mat-icon>
|
||||
<span> {{ doneCount }} </span> <span translate>Motions have been imported.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<mat-select [(value)]="shown" (selectionChange)="setFilter()">
|
||||
<!-- TODO: reduce item width to sane value -->
|
||||
<mat-option value="all" translate> Show all </mat-option>
|
||||
<mat-option *ngIf="nonImportableCount" value="error" translate> Show errors only </mat-option>
|
||||
<mat-option *ngIf="nonImportableCount" value="noerror" translate> Show correct entries </mat-option>
|
||||
</mat-select>
|
||||
<!-- TODO: Button to hide imported ones -->
|
||||
</div>
|
||||
<div class="table-container">
|
||||
<table mat-table class="on-transition-fade" [dataSource]="dataSource" matSort>
|
||||
<!-- Status column -->
|
||||
<ng-container matColumnDef="status" sticky>
|
||||
<mat-header-cell *matHeaderCellDef></mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry">
|
||||
<div *ngIf="entry.newMotion.status === 'error'">
|
||||
<mat-icon
|
||||
class="red-warning-text"
|
||||
matTooltip="{{ entry.newMotion.errors.length }} + {{ 'errors' | translate }}"
|
||||
>
|
||||
{{ getActionIcon(entry) }}
|
||||
</mat-icon>
|
||||
</div>
|
||||
<div *ngIf="entry.newMotion.status === 'new'">
|
||||
<mat-icon matTooltip="{{ 'Motion will be imported' | translate }}">
|
||||
{{ getActionIcon(entry) }}
|
||||
</mat-icon>
|
||||
</div>
|
||||
<div *ngIf="entry.newMotion.status === 'done'">
|
||||
<mat-icon matTooltip="{{ 'Motion has been imported' | translate }}">
|
||||
{{ getActionIcon(entry) }}
|
||||
</mat-icon>
|
||||
</div>
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- identifier column -->
|
||||
<ng-container matColumnDef="identifier">
|
||||
<mat-header-cell *matHeaderCellDef translate>Identifier</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry">
|
||||
<mat-icon
|
||||
color="warn"
|
||||
inline
|
||||
*ngIf="entry.newMotion.hasError('Duplicates')"
|
||||
matTooltip="{{ getVerboseError('Duplicates') | translate }}"
|
||||
>
|
||||
warning
|
||||
</mat-icon>
|
||||
{{ entry.newMotion.identifier }}
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- title column -->
|
||||
<ng-container matColumnDef="title">
|
||||
<mat-header-cell *matHeaderCellDef translate>Title</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry">
|
||||
<mat-icon
|
||||
color="warn"
|
||||
*ngIf="entry.newMotion.hasError('Title')"
|
||||
matTooltip="{{ getVerboseError('Title') | translate }}"
|
||||
>
|
||||
warning
|
||||
</mat-icon>
|
||||
{{ entry.newMotion.title }}
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- tect column TODO: Bigger-->
|
||||
<ng-container matColumnDef="text">
|
||||
<mat-header-cell *matHeaderCellDef translate>Motion text</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newMotion.text) }}">
|
||||
<mat-icon
|
||||
color="warn"
|
||||
*ngIf="entry.newMotion.hasError('Text')"
|
||||
matTooltip="{{ getVerboseError('Text') | translate }}"
|
||||
>
|
||||
warning
|
||||
</mat-icon>
|
||||
{{ getShortPreview(entry.newMotion.text) }}
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- reason column -->
|
||||
<ng-container matColumnDef="reason">
|
||||
<mat-header-cell *matHeaderCellDef translate>Reason</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newMotion.reason) }}">
|
||||
{{ getShortPreview(entry.newMotion.reason) }}
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- submitters column -->
|
||||
<ng-container matColumnDef="submitters">
|
||||
<mat-header-cell *matHeaderCellDef translate>Submitters</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry">
|
||||
<div *ngIf="entry.newMotion.csvSubmitters.length">
|
||||
<mat-icon
|
||||
color="warn"
|
||||
*ngIf="entry.newMotion.hasError('Submitters')"
|
||||
matTooltip="{{ getVerboseError('Submitters') | translate }}"
|
||||
>
|
||||
warning
|
||||
</mat-icon>
|
||||
<span *ngFor="let submitter of entry.newMotion.csvSubmitters">
|
||||
{{ submitter.name }}
|
||||
<mat-icon class="newBadge" color="accent" inline *ngIf="!submitter.id">add</mat-icon>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- category column -->
|
||||
<ng-container matColumnDef="category">
|
||||
<mat-header-cell *matHeaderCellDef translate>Category</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry">
|
||||
<div *ngIf="entry.newMotion.csvCategory">
|
||||
<mat-icon
|
||||
color="warn"
|
||||
*ngIf="entry.newMotion.hasError('Category')"
|
||||
matTooltip="{{ getVerboseError('Category') | translate }}"
|
||||
>
|
||||
warning
|
||||
</mat-icon>
|
||||
{{ entry.newMotion.csvCategory.name }}
|
||||
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newMotion.csvCategory.id"
|
||||
>add</mat-icon
|
||||
>
|
||||
</div>
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- origin column -->
|
||||
<ng-container matColumnDef="origin">
|
||||
<mat-header-cell *matHeaderCellDef translate>Origin</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry">{{ entry.newMotion.origin }}</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<!-- motion block column -->
|
||||
<ng-container matColumnDef="motion block">
|
||||
<mat-header-cell *matHeaderCellDef translate>Motion block</mat-header-cell>
|
||||
<mat-cell *matCellDef="let entry">
|
||||
<div *ngIf="entry.newMotion.csvMotionblock">
|
||||
<mat-icon
|
||||
color="warn"
|
||||
*ngIf="entry.newMotion.hasError('MotionBlock')"
|
||||
matTooltip="{{ getVerboseError('MotionBlock') | translate }}"
|
||||
>
|
||||
warning
|
||||
</mat-icon>
|
||||
{{ entry.newMotion.csvMotionblock.name }}
|
||||
<mat-icon class="newBadge" color="accent" inline *ngIf="!entry.newMotion.csvMotionblock.id">
|
||||
add
|
||||
</mat-icon>
|
||||
|
||||
</div>
|
||||
</mat-cell>
|
||||
</ng-container>
|
||||
|
||||
<mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
|
||||
<!-- TODO: class for invalid/undecided -->
|
||||
<mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
|
||||
</table>
|
||||
</div>
|
||||
</mat-card>
|
||||
@ -0,0 +1,53 @@
|
||||
.table-container {
|
||||
width: 100%;
|
||||
overflow-x: scroll;
|
||||
margin-top: 5px;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
overflow: scroll;
|
||||
}
|
||||
|
||||
.mat-header-cell {
|
||||
min-width: 100px;
|
||||
padding-right: 8px;
|
||||
}
|
||||
.mat-cell {
|
||||
min-width: 100px;
|
||||
padding-top: 2px;
|
||||
padding-right: 8px;
|
||||
}
|
||||
.selection {
|
||||
min-width: 80px;
|
||||
}
|
||||
|
||||
.import-done {
|
||||
background-color: #cfc;
|
||||
}
|
||||
.import-error {
|
||||
background-color: #fcc;
|
||||
}
|
||||
|
||||
.code {
|
||||
padding-left: 1em;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
div.wrapper {
|
||||
display: flex;
|
||||
vertical-align: bottom;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
div.summary {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.hidden-input {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.newBadge {
|
||||
margin-left: -3px;
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { MotionImportListComponent } from './motion-import-list.component';
|
||||
import { E2EImportsModule } from 'e2e-imports.module';
|
||||
|
||||
describe('MotionImportListComponent', () => {
|
||||
let component: MotionImportListComponent;
|
||||
let fixture: ComponentFixture<MotionImportListComponent>;
|
||||
|
||||
beforeEach(async(() => {
|
||||
TestBed.configureTestingModule({
|
||||
declarations: [MotionImportListComponent],
|
||||
imports: [E2EImportsModule]
|
||||
}).compileComponents();
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(MotionImportListComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,308 @@
|
||||
import { MatTableDataSource, MatTable, MatSnackBar, MatSelectChange } from '@angular/material';
|
||||
import { Title } from '@angular/platform-browser';
|
||||
import { TranslateService } from '@ngx-translate/core';
|
||||
import { ViewChild, Component, OnInit } from '@angular/core';
|
||||
|
||||
import { BaseViewComponent } from 'app/site/base/base-view';
|
||||
import { MotionCsvExportService } from '../../services/motion-csv-export.service';
|
||||
import { MotionImportService, NewMotionEntry, ValueLabelCombination } from '../../services/motion-import.service';
|
||||
|
||||
/**
|
||||
* Component for the motion import list view.
|
||||
*/
|
||||
@Component({
|
||||
selector: 'os-motion-import-list',
|
||||
templateUrl: './motion-import-list.component.html',
|
||||
styleUrls: ['./motion-import-list.component.scss']
|
||||
})
|
||||
export class MotionImportListComponent extends BaseViewComponent implements OnInit {
|
||||
/**
|
||||
* The data source for a table. Requires to be initialised with a BaseViewModel
|
||||
*/
|
||||
public dataSource: MatTableDataSource<NewMotionEntry>;
|
||||
|
||||
/**
|
||||
* Switch that turns true if a file has been selected in the input
|
||||
*/
|
||||
public hasFile = false;
|
||||
|
||||
/**
|
||||
* Currently selected encoding. Is set and changed by the config's available
|
||||
* encodings and user mat-select input
|
||||
*/
|
||||
public selectedEncoding = 'utf-8';
|
||||
|
||||
/**
|
||||
* indicator on which elements to display
|
||||
*/
|
||||
public shown: 'all' | 'error' | 'noerror' = 'all';
|
||||
|
||||
/**
|
||||
* The table itself
|
||||
*/
|
||||
@ViewChild(MatTable)
|
||||
protected table: MatTable<NewMotionEntry>;
|
||||
|
||||
/**
|
||||
* Returns the amount of total item successfully parsed
|
||||
*/
|
||||
public get totalCount(): number {
|
||||
return this.importer && this.hasFile ? this.importer.summary.total : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the encodings available and their labels
|
||||
*/
|
||||
public get encodings(): ValueLabelCombination[] {
|
||||
return this.importer.encodings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the available column separators and their labels
|
||||
*/
|
||||
public get columnSeparators(): ValueLabelCombination[] {
|
||||
return this.importer.columnSeparators;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the available text separators and their labels
|
||||
*/
|
||||
public get textSeparators(): ValueLabelCombination[] {
|
||||
return this.importer.textSeparators;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the amount of import items that will be imported
|
||||
*/
|
||||
public get newCount(): number {
|
||||
return this.importer && this.hasFile ? this.importer.summary.new : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of import items that cannot be imported
|
||||
*/
|
||||
public get nonImportableCount(): number {
|
||||
if (this.importer && this.hasFile) {
|
||||
return this.importer.summary.errors + this.importer.summary.duplicates;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of import items that have been successfully imported
|
||||
*/
|
||||
public get doneCount(): number {
|
||||
return this.importer && this.hasFile ? this.importer.summary.done : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor for list view bases
|
||||
*
|
||||
* @param titleService the title serivce
|
||||
* @param matSnackBar snackbar for displaying errors
|
||||
* @param translate the translate service
|
||||
* @param importer: The motion csv import service
|
||||
* @param motionCSVExport: service for exporting example data
|
||||
*/
|
||||
public constructor(
|
||||
titleService: Title,
|
||||
matSnackBar: MatSnackBar,
|
||||
public translate: TranslateService,
|
||||
private importer: MotionImportService,
|
||||
private motionCSVExport: MotionCsvExportService
|
||||
) {
|
||||
super(titleService, translate, matSnackBar);
|
||||
this.initTable();
|
||||
this.importer.errorEvent.subscribe(this.raiseError);
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts with a clean preview (removing any previously existing import previews)
|
||||
*/
|
||||
public ngOnInit(): void {
|
||||
this.importer.clearPreview();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the table
|
||||
*/
|
||||
public initTable(): void {
|
||||
this.dataSource = new MatTableDataSource();
|
||||
this.setFilter();
|
||||
this.importer.getNewEntries().subscribe(newEntries => {
|
||||
this.dataSource.data = newEntries;
|
||||
this.hasFile = newEntries.length > 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the table column definition. Fetches all headers from
|
||||
* {@link MotionImportService} and an additional status column
|
||||
*/
|
||||
public getColumnDefinition(): string[] {
|
||||
return ['status'].concat(this.importer.expectedHeader);
|
||||
}
|
||||
|
||||
/**
|
||||
* triggers the importer's onSelectFile after a file has been chosen
|
||||
*/
|
||||
public onSelectFile(event: any): void {
|
||||
this.importer.onSelectFile(event);
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers the importer's import
|
||||
*/
|
||||
public async doImport(): Promise<void> {
|
||||
await this.importer.doImport();
|
||||
this.setFilter();
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates and manually triggers the filter function.
|
||||
* See {@link hidden} for options
|
||||
* (changed from default mat-table filter)
|
||||
*/
|
||||
public setFilter(): void {
|
||||
this.dataSource.filter = '';
|
||||
if (this.shown === 'all') {
|
||||
this.dataSource.filterPredicate = (data, filter) => {
|
||||
return true;
|
||||
};
|
||||
} else if (this.shown === 'noerror') {
|
||||
this.dataSource.filterPredicate = (data, filter) => {
|
||||
if (data.newMotion.status === 'done') {
|
||||
return true;
|
||||
} else if (!(data.newMotion.status !== 'error') && !data.duplicates.length) {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
} else if (this.shown === 'error') {
|
||||
this.dataSource.filterPredicate = (data, filter) => {
|
||||
if (data.newMotion.errors.length || data.duplicates.length) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
||||
this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the appropiate css class for a row according to the import state
|
||||
*
|
||||
* @param row
|
||||
*/
|
||||
public getStateClass(row: NewMotionEntry): string {
|
||||
switch (row.newMotion.status) {
|
||||
case 'done':
|
||||
return 'import-done import-decided';
|
||||
case 'error':
|
||||
return 'import-error';
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first characters of a string, for preview purposes
|
||||
*
|
||||
* @param input
|
||||
*/
|
||||
public getShortPreview(input: string): string {
|
||||
if (input.length > 50) {
|
||||
return this.stripHtmlTags(input.substring(0, 47)) + '...';
|
||||
}
|
||||
return this.stripHtmlTags(input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first and last 150 characters of a string; used within
|
||||
* tooltips for the preview
|
||||
*
|
||||
* @param input
|
||||
*/
|
||||
public getLongPreview(input: string): string {
|
||||
if (input.length < 300) {
|
||||
return this.stripHtmlTags(input);
|
||||
}
|
||||
return (
|
||||
this.stripHtmlTags(input.substring(0, 147)) +
|
||||
' [...] ' +
|
||||
this.stripHtmlTags(input.substring(input.length - 150, input.length))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the icon for the action of the item
|
||||
* @param entry
|
||||
*/
|
||||
public getActionIcon(entry: NewMotionEntry): string {
|
||||
switch (entry.newMotion.status) {
|
||||
case 'error': // no import possible
|
||||
return 'block';
|
||||
case 'new': // new item, will be imported
|
||||
return 'playlist_add';
|
||||
case 'done': // item has been imported
|
||||
return 'done';
|
||||
default:
|
||||
// fallback: Error
|
||||
return 'block';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to remove html tags from a string.
|
||||
* CAUTION: It is just a basic "don't show distracting html tags in a
|
||||
* preview", not an actual tested sanitizer!
|
||||
* @param inputString
|
||||
*/
|
||||
private stripHtmlTags(inputString: string): string {
|
||||
const regexp = new RegExp(/<[^ ][^<>]*(>|$)/g);
|
||||
return inputString.replace(regexp, '').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers an example csv download
|
||||
*/
|
||||
public downloadCsvExample(): void {
|
||||
this.motionCSVExport.exportDummyMotion();
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger for the column separator selection
|
||||
*
|
||||
* @param event
|
||||
*/
|
||||
public selectColSep(event: MatSelectChange): void {
|
||||
this.importer.columnSeparator = event.value;
|
||||
this.importer.refreshFile();
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger for the column separator selection
|
||||
*
|
||||
* @param event
|
||||
*/
|
||||
public selectTextSep(event: MatSelectChange): void {
|
||||
this.importer.textSeparator = event.value;
|
||||
this.importer.refreshFile();
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger for the encoding selection
|
||||
*
|
||||
* @param event
|
||||
*/
|
||||
public selectEncoding(event: MatSelectChange): void {
|
||||
this.importer.encoding = event.value;
|
||||
this.importer.refreshFile();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a descriptive string for an import error
|
||||
*/
|
||||
public getVerboseError(error: string): string {
|
||||
return this.importer.verbose(error);
|
||||
}
|
||||
}
|
||||
@ -153,10 +153,15 @@
|
||||
<mat-icon>speaker_notes</mat-icon>
|
||||
<span translate>Comment fields</span>
|
||||
</button>
|
||||
|
||||
<button mat-menu-item (click)="csvExportMotionList()">
|
||||
<mat-icon>archive</mat-icon>
|
||||
<span translate>Export as CSV</span>
|
||||
</button>
|
||||
<button mat-menu-item *osPerms="'motions.can_manage'" routerLink="import">
|
||||
<mat-icon>save_alt</mat-icon>
|
||||
<span translate>Import</span><span> ...</span>
|
||||
</button>
|
||||
</div>
|
||||
<div *ngIf="isMultiSelect">
|
||||
<button mat-menu-item (click)="selectAll()">
|
||||
|
||||
@ -7,6 +7,10 @@ import { Motion } from 'app/shared/models/motions/motion';
|
||||
export class CreateMotion extends Motion {
|
||||
public submitters_id: number[];
|
||||
|
||||
public category_id: number;
|
||||
|
||||
public motion_block_id: number;
|
||||
|
||||
public constructor(input?: any) {
|
||||
super(input);
|
||||
}
|
||||
|
||||
180
client/src/app/site/motions/models/view-csv-create-motion.ts
Normal file
180
client/src/app/site/motions/models/view-csv-create-motion.ts
Normal file
@ -0,0 +1,180 @@
|
||||
import { ViewCreateMotion } from './view-create-motion';
|
||||
import { CreateMotion } from './create-motion';
|
||||
|
||||
/**
|
||||
* Interface for imported secondary data. A name can be matched to an existing
|
||||
* model instance by the solve... functions.
|
||||
* TODO MultiId will be filled if there is more than one match (to be used in case of 'I want to select one of these matches)
|
||||
*/
|
||||
export interface CsvMapping {
|
||||
name: string;
|
||||
id?: number;
|
||||
multiId?: number[];
|
||||
}
|
||||
|
||||
type CsvImportStatus = 'new' | 'error' | 'done';
|
||||
|
||||
/**
|
||||
* Create motion class for the View. Its different to ViewMotion in fact that the submitter handling is different
|
||||
* on motion creation.
|
||||
*
|
||||
* @ignore
|
||||
*/
|
||||
export class ViewCsvCreateMotion extends ViewCreateMotion {
|
||||
protected _motion: CreateMotion;
|
||||
|
||||
/**
|
||||
* Mapping for a new/existing category.
|
||||
*/
|
||||
public csvCategory: CsvMapping;
|
||||
|
||||
/**
|
||||
* Mapping for a new/existing motion block.
|
||||
*/
|
||||
public csvMotionblock: CsvMapping;
|
||||
|
||||
/**
|
||||
* Mapping for new/existing submitters.
|
||||
*/
|
||||
public csvSubmitters: CsvMapping[];
|
||||
|
||||
/**
|
||||
* The current import status of this motion.
|
||||
* Starts as 'new', if set to 'done', a proper {@link Motion} model will
|
||||
* probably exist in the dataStore. error status will be set if the import
|
||||
* cannot be done
|
||||
*/
|
||||
private _status: CsvImportStatus = 'new';
|
||||
|
||||
/**
|
||||
* list of import errors See {@link MotionImportService}
|
||||
*/
|
||||
public errors: string[] = [];
|
||||
|
||||
/**
|
||||
* Returns the current status.
|
||||
*/
|
||||
public get status(): CsvImportStatus {
|
||||
return this._status;
|
||||
}
|
||||
|
||||
public set status(status: CsvImportStatus) {
|
||||
this._status = status;
|
||||
}
|
||||
|
||||
public get motion(): CreateMotion {
|
||||
return this._motion;
|
||||
}
|
||||
|
||||
public constructor(motion?: CreateMotion) {
|
||||
super(motion);
|
||||
}
|
||||
|
||||
/**
|
||||
* Duplicate this motion into a copy of itself
|
||||
*/
|
||||
public copy(): ViewCreateMotion {
|
||||
return new ViewCreateMotion(
|
||||
this._motion,
|
||||
this._category,
|
||||
this._submitters,
|
||||
this._supporters,
|
||||
this._workflow,
|
||||
this._state
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a given error is present. TODO: Is more a ViewModel option
|
||||
*
|
||||
* @param error
|
||||
*/
|
||||
public hasError(error: string): boolean {
|
||||
return this.errors.includes(error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle to set a CreateMotion to a 'successfully parsed' status
|
||||
*/
|
||||
public done(): void {
|
||||
this._status = 'done';
|
||||
}
|
||||
|
||||
/**
|
||||
* takes a list of motion block mappings to update the current csvMotionblock.
|
||||
* Returns the amount of entries that remain unmatched
|
||||
*
|
||||
* @param motionBlocks
|
||||
*/
|
||||
public solveMotionBlocks(motionBlocks: CsvMapping[]): number {
|
||||
if (!this.csvMotionblock) {
|
||||
return 0;
|
||||
} else if (this.csvMotionblock.id) {
|
||||
this.motion.motion_block_id = this.csvMotionblock.id;
|
||||
return 0;
|
||||
} else {
|
||||
const newBlock = motionBlocks.find(newMotionBlock => newMotionBlock.name === this.csvMotionblock.name);
|
||||
if (newBlock) {
|
||||
this.csvMotionblock = newBlock;
|
||||
this.motion.motion_block_id = newBlock.id;
|
||||
return 0;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* takes a list of category maps to update the current csv_category.
|
||||
* Returns the amount of entries that remain unmatched
|
||||
*
|
||||
* @param categories
|
||||
*/
|
||||
public solveCategory(categories: CsvMapping[]): number {
|
||||
if (!this.csvCategory) {
|
||||
return 0;
|
||||
} else if (this.csvCategory.id) {
|
||||
this.motion.category_id = this.csvCategory.id;
|
||||
return 0;
|
||||
} else {
|
||||
const newCat = categories.find(newCategory => newCategory.name === this.csvCategory.name);
|
||||
if (newCat) {
|
||||
this.csvCategory = newCat;
|
||||
this.motion.category_id = newCat.id;
|
||||
return 0;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* takes a list of solved submitter maps to update. Returns the amount of
|
||||
* entries that remain unmatched
|
||||
*
|
||||
* @param submitters
|
||||
*/
|
||||
public solveSubmitters(submitters: CsvMapping[]): number {
|
||||
let open = 0;
|
||||
const ids: number[] = [];
|
||||
this.csvSubmitters.forEach(csvSubmitter => {
|
||||
if (csvSubmitter.id) {
|
||||
ids.push(csvSubmitter.id);
|
||||
return;
|
||||
}
|
||||
if (!submitters.length) {
|
||||
open += 1;
|
||||
return;
|
||||
}
|
||||
const mapped = submitters.find(newSubmitter => newSubmitter.name === csvSubmitter.name);
|
||||
if (mapped) {
|
||||
csvSubmitter.id = mapped.id;
|
||||
ids.push(mapped.id);
|
||||
} else {
|
||||
open += 1;
|
||||
}
|
||||
});
|
||||
this.motion.submitters_id = ids;
|
||||
return open;
|
||||
}
|
||||
}
|
||||
@ -1,15 +1,17 @@
|
||||
import { NgModule } from '@angular/core';
|
||||
import { Routes, RouterModule } from '@angular/router';
|
||||
import { MotionListComponent } from './components/motion-list/motion-list.component';
|
||||
import { MotionDetailComponent } from './components/motion-detail/motion-detail.component';
|
||||
import { CategoryListComponent } from './components/category-list/category-list.component';
|
||||
import { MotionCommentSectionListComponent } from './components/motion-comment-section-list/motion-comment-section-list.component';
|
||||
import { StatuteParagraphListComponent } from './components/statute-paragraph-list/statute-paragraph-list.component';
|
||||
import { SpeakerListComponent } from '../agenda/components/speaker-list/speaker-list.component';
|
||||
import { CallListComponent } from './components/call-list/call-list.component';
|
||||
|
||||
import { AmendmentCreateWizardComponent } from './components/amendment-create-wizard/amendment-create-wizard.component';
|
||||
import { CallListComponent } from './components/call-list/call-list.component';
|
||||
import { CategoryListComponent } from './components/category-list/category-list.component';
|
||||
import { MotionBlockListComponent } from './components/motion-block-list/motion-block-list.component';
|
||||
import { MotionBlockDetailComponent } from './components/motion-block-detail/motion-block-detail.component';
|
||||
import { MotionCommentSectionListComponent } from './components/motion-comment-section-list/motion-comment-section-list.component';
|
||||
import { MotionDetailComponent } from './components/motion-detail/motion-detail.component';
|
||||
import { MotionImportListComponent } from './components/motion-import-list/motion-import-list.component';
|
||||
import { MotionListComponent } from './components/motion-list/motion-list.component';
|
||||
import { SpeakerListComponent } from '../agenda/components/speaker-list/speaker-list.component';
|
||||
import { StatuteParagraphListComponent } from './components/statute-paragraph-list/statute-paragraph-list.component';
|
||||
|
||||
const routes: Routes = [
|
||||
{ path: '', component: MotionListComponent },
|
||||
@ -20,6 +22,7 @@ const routes: Routes = [
|
||||
{ path: 'blocks', component: MotionBlockListComponent },
|
||||
{ path: 'blocks/:id', component: MotionBlockDetailComponent },
|
||||
{ path: 'new', component: MotionDetailComponent },
|
||||
{ path: 'import', component: MotionImportListComponent },
|
||||
{ path: ':id', component: MotionDetailComponent },
|
||||
{ path: ':id/speakers', component: SpeakerListComponent },
|
||||
{ path: ':id/create-amendment', component: AmendmentCreateWizardComponent }
|
||||
|
||||
@ -18,6 +18,7 @@ import { CallListComponent } from './components/call-list/call-list.component';
|
||||
import { AmendmentCreateWizardComponent } from './components/amendment-create-wizard/amendment-create-wizard.component';
|
||||
import { MotionBlockListComponent } from './components/motion-block-list/motion-block-list.component';
|
||||
import { MotionBlockDetailComponent } from './components/motion-block-detail/motion-block-detail.component';
|
||||
import { MotionImportListComponent } from './components/motion-import-list/motion-import-list.component';
|
||||
import { ManageSubmittersComponent } from './components/manage-submitters/manage-submitters.component';
|
||||
|
||||
@NgModule({
|
||||
@ -38,6 +39,7 @@ import { ManageSubmittersComponent } from './components/manage-submitters/manage
|
||||
AmendmentCreateWizardComponent,
|
||||
MotionBlockListComponent,
|
||||
MotionBlockDetailComponent,
|
||||
MotionImportListComponent,
|
||||
ManageSubmittersComponent
|
||||
],
|
||||
entryComponents: [
|
||||
|
||||
@ -112,4 +112,13 @@ export class MotionBlockRepositoryService extends BaseRepository<ViewMotionBlock
|
||||
.getViewModelListObservable()
|
||||
.pipe(map(viewMotions => viewMotions.filter(viewMotion => viewMotion.motion_block_id === block.id)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves motion block(s) by name
|
||||
* TODO: check if a title is unique for a motionBlock
|
||||
* @param title Strign to check for
|
||||
*/
|
||||
public getMotionBlockByTitle(title: string): MotionBlock {
|
||||
return this.DS.find(MotionBlock, block => block.title === title);
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import { TranslateService } from '@ngx-translate/core';
|
||||
|
||||
import { CsvExportService } from 'app/core/services/csv-export.service';
|
||||
import { ViewMotion } from '../models/view-motion';
|
||||
import { FileExportService } from 'app/core/services/file-export.service';
|
||||
|
||||
/**
|
||||
* Exports CSVs for motions. Collect all CSV types here to have them in one place.
|
||||
@ -18,7 +19,11 @@ export class MotionCsvExportService {
|
||||
* @param csvExport CsvExportService
|
||||
* @param translate TranslateService
|
||||
*/
|
||||
public constructor(private csvExport: CsvExportService, private translate: TranslateService) {}
|
||||
public constructor(
|
||||
private csvExport: CsvExportService,
|
||||
private translate: TranslateService,
|
||||
private fileExport: FileExportService
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Export all motions as CSV
|
||||
@ -64,4 +69,17 @@ export class MotionCsvExportService {
|
||||
this.translate.instant('Call list') + '.csv'
|
||||
);
|
||||
}
|
||||
|
||||
public exportDummyMotion(): void {
|
||||
const headerRow = ['Identifier', 'Title', 'Text', 'Reason', 'Submitters', 'Category', 'Origin', 'Motion block']
|
||||
.map(item => this.translate.instant(item))
|
||||
.join(',');
|
||||
const rows = [
|
||||
headerRow,
|
||||
'A1,Title 1,Text 1,Reason 1,Submitter A,Category A,"Last Year Conference A", Block A',
|
||||
'B1,Title 2,Text 2,Reason 2,Submitter B, Category B,, Block A',
|
||||
',Title 3, Text 3,,,,,'
|
||||
];
|
||||
this.fileExport.saveFile(rows.join('\n'), this.translate.instant('Motion example') + '.csv');
|
||||
}
|
||||
}
|
||||
|
||||
599
client/src/app/site/motions/services/motion-import.service.ts
Normal file
599
client/src/app/site/motions/services/motion-import.service.ts
Normal file
@ -0,0 +1,599 @@
|
||||
import { BehaviorSubject, Observable } from 'rxjs';
|
||||
import { Injectable, EventEmitter } from '@angular/core';
|
||||
import { MatSnackBar } from '@angular/material';
|
||||
import { Papa, PapaParseConfig } from 'ngx-papaparse';
|
||||
import { TranslateService } from '@ngx-translate/core';
|
||||
|
||||
import { Category } from 'app/shared/models/motions/category';
|
||||
import { CategoryRepositoryService } from './category-repository.service';
|
||||
import { CreateMotion } from '../models/create-motion';
|
||||
import { MotionBlock } from 'app/shared/models/motions/motion-block';
|
||||
import { MotionBlockRepositoryService } from './motion-block-repository.service';
|
||||
import { MotionRepositoryService } from './motion-repository.service';
|
||||
import { UserRepositoryService } from '../../users/services/user-repository.service';
|
||||
import { ViewMotion } from '../models/view-motion';
|
||||
import { ViewCsvCreateMotion, CsvMapping } from '../models/view-csv-create-motion';
|
||||
|
||||
/**
|
||||
* Interface for value- Label combinations.
|
||||
* Map objects didn't work, TODO: Use map objects (needs iterating through all objects of a map)
|
||||
*/
|
||||
export interface ValueLabelCombination {
|
||||
value: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface for a new Motion and their (if any) duplicates
|
||||
*/
|
||||
export interface NewMotionEntry {
|
||||
newMotion: ViewCsvCreateMotion;
|
||||
duplicates: ViewMotion[];
|
||||
}
|
||||
|
||||
/**
|
||||
* interface for a preview summary
|
||||
*/
|
||||
interface ImportMotionCSVPreview {
|
||||
total: number;
|
||||
duplicates: number;
|
||||
errors: number;
|
||||
new: number;
|
||||
done: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* List of possible import errors specific for motion imports.
|
||||
*/
|
||||
const errorList = {
|
||||
MotionBlock: 'Could not resolve the motion block',
|
||||
Category: 'Could not resolve the category',
|
||||
Submitters: 'Could not resolve the submitters',
|
||||
Title: 'A title is required',
|
||||
Text: "A content in the 'text' column is required",
|
||||
Duplicates: 'A motion with this identifier already exists.',
|
||||
generic: 'Server upload failed' // TODO
|
||||
};
|
||||
|
||||
/**
|
||||
* Service for motion imports
|
||||
*/
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
})
|
||||
export class MotionImportService {
|
||||
/** The header (order and items) that is expected from the imported file
|
||||
*
|
||||
*/
|
||||
public expectedHeader = [
|
||||
'identifier',
|
||||
'title',
|
||||
'text',
|
||||
'reason',
|
||||
'submitters',
|
||||
'category',
|
||||
'origin',
|
||||
'motion block'
|
||||
];
|
||||
|
||||
/**
|
||||
* The last parsed file object (may be reparsed with new encoding, thus kept in memory)
|
||||
*/
|
||||
private _rawFile: File;
|
||||
|
||||
/**
|
||||
* The used column Separator. If left on an empty string (default),
|
||||
* the papaparse parser will automatically decide on separators.
|
||||
*/
|
||||
public columnSeparator = '';
|
||||
|
||||
public textSeparator = '"';
|
||||
|
||||
public encoding = 'utf-8';
|
||||
|
||||
/**
|
||||
* List of possible encodings and their label
|
||||
*/
|
||||
public encodings: ValueLabelCombination[] = [
|
||||
{ value: 'utf-8', label: 'UTF 8 - Unicode' },
|
||||
{ value: 'iso-8859-1', label: 'ISO 8859-1 - West European' },
|
||||
{ value: 'iso-8859-15', label: 'ISO 8859-15 - West European (with €)' }
|
||||
];
|
||||
|
||||
/**
|
||||
* List of possible column separators
|
||||
*/
|
||||
public columnSeparators: ValueLabelCombination[] = [
|
||||
{ label: 'Comma', value: ',' },
|
||||
{ label: 'Semicolon', value: ';' },
|
||||
// {label: 'Tabulator', value: '\t'},
|
||||
{ label: 'Automatic', value: '' }
|
||||
];
|
||||
|
||||
public textSeparators: ValueLabelCombination[] = [
|
||||
{ label: 'Double quotes (")', value: '"' },
|
||||
{ label: "Single quotes (')", value: "'" }
|
||||
];
|
||||
|
||||
/**
|
||||
* submitters that need to be created prior to importing
|
||||
*/
|
||||
public newSubmitters: CsvMapping[] = [];
|
||||
|
||||
/**
|
||||
* Categories that need to be created prior to importing
|
||||
*/
|
||||
public newCategories: CsvMapping[] = [];
|
||||
|
||||
/**
|
||||
* MotionBlocks that need to be created prior to importing
|
||||
*/
|
||||
public newMotionBlocks: CsvMapping[] = [];
|
||||
|
||||
/**
|
||||
* FileReader object for file import
|
||||
*/
|
||||
private reader = new FileReader();
|
||||
|
||||
/**
|
||||
* the list of parsed models that have been extracted from the opened file
|
||||
*/
|
||||
private _entries: NewMotionEntry[] = [];
|
||||
|
||||
/**
|
||||
* BehaviorSubject for displaying a preview for the currently selected entries
|
||||
*/
|
||||
public newEntries = new BehaviorSubject<NewMotionEntry[]>([]);
|
||||
|
||||
/**
|
||||
* Emits an error string to display if a file import cannot be done
|
||||
*/
|
||||
public errorEvent = new EventEmitter<string>();
|
||||
|
||||
/**
|
||||
* storing the summary preview for the import, to avoid recalculating it
|
||||
* at each display change.
|
||||
*/
|
||||
private _preview: ImportMotionCSVPreview;
|
||||
|
||||
/**
|
||||
* Returns a summary on actions that will be taken/not taken.
|
||||
*/
|
||||
public get summary(): ImportMotionCSVPreview {
|
||||
if (!this._preview) {
|
||||
this.updatePreview();
|
||||
}
|
||||
return this._preview;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor. Creates a fileReader to subscribe to it for incoming parsed
|
||||
* strings
|
||||
* @param categoryRepo Repository to fetch pre-existing categories
|
||||
* @param motionBlockRepo Repository to fetch pre-existing motionBlocks
|
||||
* @param userRepo Repository to query/ create users
|
||||
* @param translate Translation service
|
||||
* @param papa External csv parser (ngx-papaparser)
|
||||
* @param matSnackBar snackBar to display import errors
|
||||
*/
|
||||
public constructor(
|
||||
private repo: MotionRepositoryService,
|
||||
private categoryRepo: CategoryRepositoryService,
|
||||
private motionBlockRepo: MotionBlockRepositoryService,
|
||||
private userRepo: UserRepositoryService,
|
||||
private translate: TranslateService,
|
||||
private papa: Papa,
|
||||
private matSnackbar: MatSnackBar
|
||||
) {
|
||||
this.reader.onload = (event: any) => {
|
||||
// TODO type: event is a progressEvent,
|
||||
// but has a property target.result, which typescript doesn't recognize
|
||||
this.parseInput(event.target.result);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the data input. Expects a string as returned by via a
|
||||
* File.readAsText() operation
|
||||
*
|
||||
* @param file
|
||||
*/
|
||||
public parseInput(file: string): void {
|
||||
this._entries = [];
|
||||
this.newSubmitters = [];
|
||||
this.newCategories = [];
|
||||
this.newMotionBlocks = [];
|
||||
const papaConfig: PapaParseConfig = {
|
||||
header: false,
|
||||
skipEmptyLines: true,
|
||||
quoteChar: this.textSeparator
|
||||
};
|
||||
if (this.columnSeparator) {
|
||||
papaConfig.delimiter = this.columnSeparator;
|
||||
}
|
||||
const entryLines = this.papa.parse(file, papaConfig).data;
|
||||
const valid = this.checkHeader(entryLines.shift());
|
||||
if (!valid) {
|
||||
return;
|
||||
}
|
||||
entryLines.forEach(line => {
|
||||
const newMotion = new ViewCsvCreateMotion(new CreateMotion());
|
||||
const headerLength = Math.min(this.expectedHeader.length, line.length);
|
||||
for (let idx = 0; idx < headerLength; idx++) {
|
||||
// iterate over items, find existing ones (thier id) and collect new entries
|
||||
switch (this.expectedHeader[idx]) {
|
||||
case 'submitters':
|
||||
newMotion.csvSubmitters = this.getSubmitters(line[idx]);
|
||||
break;
|
||||
case 'category':
|
||||
newMotion.csvCategory = this.getCategory(line[idx]);
|
||||
break;
|
||||
case 'motion block':
|
||||
newMotion.csvMotionblock = this.getMotionBlock(line[idx]);
|
||||
break;
|
||||
default:
|
||||
newMotion.motion[this.expectedHeader[idx]] = line[idx];
|
||||
}
|
||||
}
|
||||
const updateModels = this.getDuplicates(newMotion.motion);
|
||||
if (updateModels.length) {
|
||||
this.setError(newMotion, 'Duplicates');
|
||||
}
|
||||
this._entries.push({ newMotion: newMotion, duplicates: updateModels });
|
||||
});
|
||||
this.newEntries.next(this._entries);
|
||||
this.updatePreview();
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers the import.
|
||||
*/
|
||||
public async doImport(): Promise<void> {
|
||||
this.newMotionBlocks = await this.createNewMotionBlocks();
|
||||
this.newCategories = await this.createNewCategories();
|
||||
this.newSubmitters = await this.createNewUsers();
|
||||
|
||||
for (const entry of this._entries) {
|
||||
if (entry.newMotion.status !== 'new') {
|
||||
continue;
|
||||
}
|
||||
const openBlocks = entry.newMotion.solveMotionBlocks(this.newMotionBlocks);
|
||||
if (openBlocks) {
|
||||
this.setError(entry.newMotion, 'MotionBlock');
|
||||
// TODO error handling if not all submitters could be matched
|
||||
this.updatePreview();
|
||||
continue;
|
||||
}
|
||||
const openCategories = entry.newMotion.solveCategory(this.newCategories);
|
||||
if (openCategories) {
|
||||
this.setError(entry.newMotion, 'Category');
|
||||
this.updatePreview();
|
||||
continue;
|
||||
}
|
||||
const openUsers = entry.newMotion.solveSubmitters(this.newSubmitters);
|
||||
if (openUsers) {
|
||||
this.setError(entry.newMotion, 'Submitters');
|
||||
this.updatePreview();
|
||||
continue;
|
||||
}
|
||||
await this.repo.create(entry.newMotion.motion);
|
||||
entry.newMotion.done();
|
||||
}
|
||||
this.updatePreview();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the dataStore for duplicates
|
||||
* @returns an array of duplicates with the same identifier.
|
||||
* @param motion
|
||||
*/
|
||||
public getDuplicates(motion: CreateMotion): ViewMotion[] {
|
||||
return this.repo.getMotionDuplicates(motion);
|
||||
}
|
||||
|
||||
/**
|
||||
* counts the amount of duplicates that have no decision on the action to
|
||||
* be taken
|
||||
*/
|
||||
public updatePreview(): void {
|
||||
const summary = {
|
||||
total: 0,
|
||||
new: 0,
|
||||
duplicates: 0,
|
||||
errors: 0,
|
||||
done: 0
|
||||
};
|
||||
this._entries.forEach(entry => {
|
||||
summary.total += 1;
|
||||
if (entry.newMotion.status === 'done') {
|
||||
summary.done += 1;
|
||||
return;
|
||||
} else if (entry.newMotion.status === 'error' && !entry.duplicates.length) {
|
||||
// errors that are not due to duplicates
|
||||
summary.errors += 1;
|
||||
return;
|
||||
} else if (entry.duplicates.length) {
|
||||
summary.duplicates += 1;
|
||||
return;
|
||||
} else if (entry.newMotion.status === 'new') {
|
||||
summary.new += 1;
|
||||
}
|
||||
});
|
||||
this._preview = summary;
|
||||
}
|
||||
|
||||
/**
|
||||
* returns a subscribable representation of the new Users to be imported
|
||||
*/
|
||||
public getNewEntries(): Observable<NewMotionEntry[]> {
|
||||
return this.newEntries.asObservable();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the provided submitter(s) and returns an object with mapping of
|
||||
* existing users and of users that need to be created
|
||||
* @param submitterlist
|
||||
*/
|
||||
public getSubmitters(submitterlist: string): CsvMapping[] {
|
||||
const result: CsvMapping[] = [];
|
||||
if (!submitterlist) {
|
||||
return result;
|
||||
}
|
||||
const submitterArray = submitterlist.split(','); // TODO fails with 'full name'
|
||||
for (const submitter of submitterArray) {
|
||||
const existingSubmitters = this.userRepo.getUsersByName(submitter);
|
||||
if (!existingSubmitters.length) {
|
||||
if (!this.newSubmitters.find(listedSubmitter => listedSubmitter.name === submitter)) {
|
||||
this.newSubmitters.push({ name: submitter });
|
||||
}
|
||||
result.push({ name: submitter });
|
||||
}
|
||||
if (existingSubmitters.length === 1) {
|
||||
result.push({
|
||||
name: existingSubmitters[0].short_name,
|
||||
id: existingSubmitters[0].id
|
||||
});
|
||||
}
|
||||
if (existingSubmitters.length > 1) {
|
||||
result.push({
|
||||
name: submitter,
|
||||
multiId: existingSubmitters.map(ex => ex.id)
|
||||
});
|
||||
this.matSnackbar.open('TODO: multiple possible users found for this string', 'ok');
|
||||
// TODO How to handle several submitters ? Is this possible?
|
||||
// should have some kind of choice dialog there
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the provided category/ies and returns a mapping, expands
|
||||
* newCategories if needed.
|
||||
*
|
||||
* The assumption is that there may or not be a prefix wit up to 5
|
||||
* characters at the beginning, separated by ' - ' from the name.
|
||||
* It will also accept a registered translation between the current user's
|
||||
* language and english
|
||||
* @param categoryString
|
||||
*/
|
||||
public getCategory(categoryString: string): CsvMapping {
|
||||
if (!categoryString) {
|
||||
return null;
|
||||
}
|
||||
const category = this.splitCategoryString(categoryString);
|
||||
const existingCategory = this.categoryRepo.getViewModelList().find(cat => {
|
||||
if (category.prefix && cat.prefix !== category.prefix) {
|
||||
return false;
|
||||
}
|
||||
if (cat.name === category.name) {
|
||||
return true;
|
||||
}
|
||||
if (this.translate.instant(cat.name) === category.name) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
if (existingCategory) {
|
||||
return {
|
||||
name: existingCategory.prefixedName,
|
||||
id: existingCategory.id
|
||||
};
|
||||
} else {
|
||||
if (!this.newCategories.find(newCat => newCat.name === categoryString)) {
|
||||
this.newCategories.push({ name: categoryString });
|
||||
}
|
||||
return { name: categoryString };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the motionBlock provided in the string for existance, expands newMotionBlocks
|
||||
* if needed. Note that it will also check for translation between the current
|
||||
* user's language and english
|
||||
* @param blockString
|
||||
*/
|
||||
public getMotionBlock(blockString: string): CsvMapping {
|
||||
if (!blockString) {
|
||||
return null;
|
||||
}
|
||||
blockString = blockString.trim();
|
||||
let existingBlock = this.motionBlockRepo.getMotionBlockByTitle(blockString);
|
||||
if (!existingBlock) {
|
||||
existingBlock = this.motionBlockRepo.getMotionBlockByTitle(this.translate.instant(blockString));
|
||||
}
|
||||
if (existingBlock) {
|
||||
return { id: existingBlock.id, name: existingBlock.title };
|
||||
} else {
|
||||
if (!this.newMotionBlocks.find(newBlock => newBlock.name === blockString)) {
|
||||
this.newMotionBlocks.push({ name: blockString });
|
||||
}
|
||||
return { name: blockString };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates all new Users needed for the import.
|
||||
*/
|
||||
private async createNewUsers(): Promise<CsvMapping[]> {
|
||||
const promises: Promise<CsvMapping>[] = [];
|
||||
for (const user of this.newSubmitters) {
|
||||
promises.push(this.userRepo.createFromString(user.name));
|
||||
}
|
||||
return await Promise.all(promises);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates all new Motion Blocks needed for the import.
|
||||
*/
|
||||
private async createNewMotionBlocks(): Promise<CsvMapping[]> {
|
||||
const promises: Promise<CsvMapping>[] = [];
|
||||
for (const block of this.newMotionBlocks) {
|
||||
promises.push(
|
||||
this.motionBlockRepo.create(new MotionBlock({ title: block.name })).then(identifiable => {
|
||||
return { name: block.name, id: identifiable.id };
|
||||
})
|
||||
);
|
||||
}
|
||||
return await Promise.all(promises);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates all new Categories needed for the import.
|
||||
*/
|
||||
private async createNewCategories(): Promise<CsvMapping[]> {
|
||||
const promises: Promise<CsvMapping>[] = [];
|
||||
for (const category of this.newCategories) {
|
||||
const cat = this.splitCategoryString(category.name);
|
||||
promises.push(
|
||||
this.categoryRepo
|
||||
.create(
|
||||
new Category({
|
||||
name: cat.name,
|
||||
prefix: cat.prefix ? cat.prefix : null
|
||||
})
|
||||
)
|
||||
.then(identifiable => {
|
||||
return { name: category.name, id: identifiable.id };
|
||||
})
|
||||
);
|
||||
}
|
||||
return await Promise.all(promises);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handler after a file was selected. Basic checking for type, then hand
|
||||
* over to parsing
|
||||
*
|
||||
* @param event type is Event, but has target.files, which typescript doesn't seem to recognize
|
||||
*/
|
||||
public onSelectFile(event: any): void {
|
||||
// TODO type
|
||||
if (event.target.files && event.target.files.length === 1) {
|
||||
if (event.target.files[0].type === 'text/csv') {
|
||||
this._rawFile = event.target.files[0];
|
||||
this.readFile(event.target.files[0]);
|
||||
} else {
|
||||
this.matSnackbar.open(this.translate.instant('Wrong file type detected. Import failed.'), '', {
|
||||
duration: 3000
|
||||
});
|
||||
this.clearPreview();
|
||||
this._rawFile = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rereads the (previously selected) file, if present. Thought to be triggered
|
||||
* by parameter changes on encoding, column, text separators
|
||||
*/
|
||||
public refreshFile(): void {
|
||||
if (this._rawFile) {
|
||||
this.readFile(this._rawFile);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* (re)-reads a given file with the current parameter
|
||||
*/
|
||||
private readFile(file: File): void {
|
||||
this.reader.readAsText(file, this.encoding);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the first line of the csv (the header) for consistency (length)
|
||||
* @param row expected to be an array parsed from the first line of a csv file
|
||||
*/
|
||||
private checkHeader(row: string[]): boolean {
|
||||
const snackbarDuration = 3000;
|
||||
if (row.length < 4) {
|
||||
this.matSnackbar.open(this.translate.instant('The file has too few columns to be parsed properly.'), '', {
|
||||
duration: snackbarDuration
|
||||
});
|
||||
|
||||
this.clearPreview();
|
||||
return false;
|
||||
} else if (row.length < this.expectedHeader.length) {
|
||||
this.matSnackbar.open(
|
||||
this.translate.instant('The file seems to have some ommitted columns. They will be considered empty.'),
|
||||
'',
|
||||
{ duration: snackbarDuration }
|
||||
);
|
||||
} else if (row.length > this.expectedHeader.length) {
|
||||
this.matSnackbar.open(
|
||||
this.translate.instant('The file seems to have additional columns. They will be ignored.'),
|
||||
'',
|
||||
{ duration: snackbarDuration }
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the data and preview (triggered upon selecting an invalid file)
|
||||
*/
|
||||
public clearPreview(): void {
|
||||
this._entries = [];
|
||||
this.newEntries.next([]);
|
||||
this._preview = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* set a list of short names for error, indicating which column failed
|
||||
*/
|
||||
public setError(motion: ViewCsvCreateMotion, error: string): void {
|
||||
if (errorList.hasOwnProperty(error) && !motion.errors.includes(error)) {
|
||||
motion.errors.push(error);
|
||||
motion.status = 'error';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an extended error description.
|
||||
* @param error
|
||||
*/
|
||||
public verbose(error: string): string {
|
||||
return errorList[error];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to separate a category string from its' prefix. Assumes that a prefix is no longer
|
||||
* than 5 chars and separated by a ' - '
|
||||
* @param categoryString the string to parse
|
||||
*/
|
||||
private splitCategoryString(categoryString: string): { prefix: string; name: string } {
|
||||
let prefixSeparator = ' - ';
|
||||
if (categoryString.startsWith(prefixSeparator)) {
|
||||
prefixSeparator = prefixSeparator.substring(1);
|
||||
}
|
||||
categoryString = categoryString.trim();
|
||||
let prefix = '';
|
||||
const separatorIndex = categoryString.indexOf(prefixSeparator);
|
||||
|
||||
if (separatorIndex >= 0 && separatorIndex < 6) {
|
||||
prefix = categoryString.substring(0, separatorIndex);
|
||||
categoryString = categoryString.substring(separatorIndex + prefixSeparator.length);
|
||||
}
|
||||
return { prefix: prefix, name: categoryString };
|
||||
}
|
||||
}
|
||||
@ -149,6 +149,7 @@ export class MotionRepositoryService extends BaseRepository<ViewMotion, Motion>
|
||||
* @param viewMotion The View Motion. If not present, a new motion will be created
|
||||
*/
|
||||
public async create(motion: CreateMotion): Promise<Identifiable> {
|
||||
// TODO how to handle category id and motion_block id in CreateMotion?
|
||||
return await this.dataSend.createModel(motion);
|
||||
}
|
||||
|
||||
@ -618,4 +619,17 @@ export class MotionRepositoryService extends BaseRepository<ViewMotion, Motion>
|
||||
)
|
||||
.filter((para: ViewMotionAmendedParagraph) => para !== null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all Motion duplicates (sharing specific values given in input)
|
||||
* @param viewMotion the ViewMotion to compare against the list of Motions
|
||||
* in the data
|
||||
* @param sharedValues properties that must be equal to consider it a duplicate
|
||||
*/
|
||||
public getMotionDuplicates(motion: Motion): ViewMotion[] {
|
||||
const duplicates = this.DS.filter(Motion, item => motion.identifier === item.identifier);
|
||||
const viewMotions: ViewMotion[] = [];
|
||||
duplicates.forEach(item => viewMotions.push(this.createViewModel(item)));
|
||||
return viewMotions;
|
||||
}
|
||||
}
|
||||
|
||||
@ -86,11 +86,6 @@
|
||||
<span translate>Groups</span>
|
||||
</button>
|
||||
|
||||
<button mat-menu-item>
|
||||
<mat-icon>save_alt</mat-icon>
|
||||
<span translate>Import ...</span>
|
||||
</button>
|
||||
|
||||
<button mat-menu-item (click)="csvExportUserList()">
|
||||
<mat-icon>archive</mat-icon>
|
||||
<span translate>Export as CSV</span>
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
|
||||
import { BaseRepository } from '../../base/base-repository';
|
||||
import { ViewUser } from '../models/view-user';
|
||||
import { User } from '../../../shared/models/users/user';
|
||||
@ -197,4 +198,48 @@ export class UserRepositoryService extends BaseRepository<ViewUser, User> {
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches and returns Users by full name
|
||||
* @param name
|
||||
*/
|
||||
public getUsersByName(name: string): ViewUser[] {
|
||||
const results: ViewUser[] = [];
|
||||
const users = this.DS.getAll(User).filter(user => {
|
||||
if (user.full_name === name || user.short_name === name) {
|
||||
return true;
|
||||
}
|
||||
if (user.number === name) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
users.forEach(user => {
|
||||
results.push(this.createViewModel(user));
|
||||
});
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new User from a string
|
||||
* @param user: String to create the user from
|
||||
* TODO: return 'user' + new id
|
||||
*/
|
||||
public async createFromString(user: string): Promise<{ id: number; name: string }> {
|
||||
const splitUser = user.split(' ');
|
||||
const newUser: Partial<User> = {};
|
||||
switch (splitUser.length) {
|
||||
case 1:
|
||||
newUser.first_name = splitUser[0];
|
||||
break;
|
||||
case 2:
|
||||
newUser.first_name = splitUser[0];
|
||||
newUser.last_name = splitUser[1];
|
||||
break;
|
||||
default:
|
||||
newUser.first_name = user;
|
||||
}
|
||||
const createdUser = await this.create(newUser);
|
||||
return { id: createdUser.id, name: user };
|
||||
}
|
||||
}
|
||||
|
||||
@ -94,6 +94,14 @@ body {
|
||||
}
|
||||
}
|
||||
|
||||
.accent-text {
|
||||
color: mat-color($openslides-accent);
|
||||
}
|
||||
|
||||
.green-text { // TODO better name/theming
|
||||
color: #5A5;
|
||||
}
|
||||
|
||||
.icon-text-distance {
|
||||
margin-left: 5px;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user