Merge pull request #4224 from MaximilianKrambach/statuteImportExport

import/export for statutes

client/src/app/site/motions/components/statute-paragraph-list/statute-import-list/statute-import-list.component.html

@@ -0,0 +1,159 @@
+<os-head-bar [nav]="false">
+    <!-- Title -->
+    <div class="title-slot"><h2 translate>Import Statutes</h2></div>
+
+    <div class="menu-slot">
+        <button *ngIf="hasFile && newCount" mat-button (click)="doImport()">
+            <span class="upper" translate> Import</span>
+        </button>
+    </div>
+</os-head-bar>
+
+<mat-card class="os-form-card import-table">
+    <span translate>Required comma or semicolon separated values with these column header names in the first row:</span>
+    <br />
+    <div class="code red-warning-text"><span translate>Title</span>, <span translate>Text</span></div>
+    <ul>
+        <li translate>Additional columns after the required ones may be present and won't affect the import.</li>
+    </ul>
+    <button mat-button color="accent" (click)="downloadCsvExample()" translate>Download CSV example file</button>
+    <div class="wrapper">
+        <mat-form-field>
+            <mat-label translate>Encoding of the file</mat-label>
+            <mat-select
+                class="selection"
+                placeholder="translate.instant('Select encoding')"
+                (selectionChange)="selectEncoding($event)"
+                [value]="encodings[0].value"
+            >
+                <mat-option *ngFor="let option of encodings" [value]="option.value">
+                    {{ option.label | translate }}
+                </mat-option>
+            </mat-select>
+        </mat-form-field>
+        <mat-form-field>
+            <mat-label translate>Column separator</mat-label>
+            <mat-select class="selection" (selectionChange)="selectColSep($event)" value="">
+                <mat-option *ngFor="let option of columnSeparators" [value]="option.value">
+                    {{ option.label | translate }}
+                </mat-option>
+            </mat-select>
+        </mat-form-field>
+        <mat-form-field>
+            <mat-label translate>Text separator</mat-label>
+            <mat-select class="selection" (selectionChange)="selectTextSep($event)" value='"'>
+                <mat-option *ngFor="let option of textSeparators" [value]="option.value">
+                    {{ option.label | translate }}
+                </mat-option>
+            </mat-select>
+        </mat-form-field>
+    </div>
+    <div>
+        <input
+            id="statute-import-file-input"
+            type="file"
+            class="hidden-input"
+            accept="text"
+            #fileInput
+            (change)="onSelectFile($event)"
+        />
+        <button mat-button osAutofocus onclick="document.getElementById('statute-import-file-input').click()">
+            <span translate> Select file</span>
+        </button>
+    </div>
+</mat-card>
+
+<!-- preview table -->
+<mat-card *ngIf="hasFile" class="os-form-card import-table">
+    <h3 translate>Preview</h3>
+    <div class="summary">
+        <!-- new entries -->
+        <div *ngIf="newCount">
+            &nbsp;
+            <mat-icon inline>playlist_add</mat-icon>
+            <span>&nbsp;{{ newCount }}&nbsp;</span> <span translate>Statute paragraphs(s) will be imported.</span>
+        </div>
+        <!-- errors/duplicates -->
+        <div *ngIf="nonImportableCount" class="red-warning-text">
+            &nbsp;
+            <mat-icon inline>warning</mat-icon>
+            <span>&nbsp;{{ nonImportableCount }}&nbsp;</span> <span translate>entries will be ommitted.</span>
+        </div>
+        <!-- have been imported -->
+        <div *ngIf="doneCount" class="green-text">
+            &nbsp;
+            <mat-icon inline>done</mat-icon>
+            <span>&nbsp;{{ doneCount }}&nbsp;</span> <span translate>Statute paragraphs have been imported.</span>
+        </div>
+    </div>
+    <div *ngIf="newCount">
+        <span translate>After verifiy the preview click on 'import' please (see top right).</span>
+    </div>
+    <div>
+        <mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()">
+            <mat-option value="all" translate> Show all </mat-option>
+            <mat-option value="error" translate> Show errors only </mat-option>
+            <mat-option value="noerror" translate> Show correct entries </mat-option>
+        </mat-select>
+    </div>
+    <div class="table-container">
+        <table mat-table class="on-transition-fade" [dataSource]="dataSource" matSort>
+            <!-- Status column -->
+            <ng-container matColumnDef="status" sticky>
+                <mat-header-cell *matHeaderCellDef class="first-column"></mat-header-cell>
+                <mat-cell *matCellDef="let entry" class="first-column">
+                    <div *ngIf="entry.status === 'error'">
+                        <mat-icon
+                            class="red-warning-text"
+                            matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
+                        >
+                            {{ getActionIcon(entry) }}
+                        </mat-icon>
+                    </div>
+                    <div *ngIf="entry.status === 'new'">
+                        <mat-icon matTooltip="{{ 'Statute paragraph will be imported' | translate }}">
+                            {{ getActionIcon(entry) }}
+                        </mat-icon>
+                    </div>
+                    <div *ngIf="entry.status === 'done'">
+                        <mat-icon matTooltip="{{ 'Statute paragraph has been imported' | translate }}">
+                            {{ getActionIcon(entry) }}
+                        </mat-icon>
+                    </div>
+                </mat-cell>
+            </ng-container>
+
+            <!-- title column -->
+            <ng-container matColumnDef="title">
+                <mat-header-cell *matHeaderCellDef translate>Title</mat-header-cell>
+                <mat-cell *matCellDef="let entry">
+                    <mat-icon
+                        color="warn"
+                        *ngIf="hasError(entry, 'Title')"
+                        matTooltip="{{ getVerboseError('Title') | translate }}"
+                    >
+                        warning
+                    </mat-icon>
+                    {{ entry.newEntry.title }}
+                </mat-cell>
+            </ng-container>
+
+            <!-- text column -->
+            <ng-container matColumnDef="text">
+                <mat-header-cell *matHeaderCellDef translate>Text</mat-header-cell>
+                <mat-cell *matCellDef="let entry" matTooltip="{{ getLongPreview(entry.newEntry.text) }}">
+                    <mat-icon
+                        color="warn"
+                        *ngIf="hasError(entry, 'Text')"
+                        matTooltip="{{ getVerboseError('Text') | translate }}"
+                    >
+                        warning
+                    </mat-icon>
+                    {{ getShortPreview(entry.newEntry.text) }}
+                </mat-cell>
+            </ng-container>
+            <mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
+            <mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
+        </table>
+    </div>
+</mat-card>

client/src/app/site/motions/components/statute-paragraph-list/statute-import-list/statute-import-list.component.spec.ts

@@ -0,0 +1,26 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { StatuteImportListComponent } from './statute-import-list.component';
+import { E2EImportsModule } from 'e2e-imports.module';
+
+describe('StatuteImportListComponent', () => {
+    let component: StatuteImportListComponent;
+    let fixture: ComponentFixture<StatuteImportListComponent>;
+
+    beforeEach(async(() => {
+        TestBed.configureTestingModule({
+            declarations: [StatuteImportListComponent],
+            imports: [E2EImportsModule]
+        }).compileComponents();
+    }));
+
+    beforeEach(() => {
+        fixture = TestBed.createComponent(StatuteImportListComponent);
+        component = fixture.componentInstance;
+        fixture.detectChanges();
+    });
+
+    it('should create', () => {
+        expect(component).toBeTruthy();
+    });
+});

client/src/app/site/motions/components/statute-paragraph-list/statute-import-list/statute-import-list.component.ts

@@ -0,0 +1,84 @@
+import { Component } from '@angular/core';
+import { MatSnackBar } from '@angular/material';
+import { Title } from '@angular/platform-browser';
+import { TranslateService } from '@ngx-translate/core';
+
+import { BaseImportListComponent } from 'app/site/base/base-import-list';
+import { ViewStatuteParagraph } from 'app/site/motions/models/view-statute-paragraph';
+import { StatuteImportService } from 'app/site/motions/services/statute-import.service';
+import { StatuteCsvExportService } from 'app/site/motions/services/statute-csv-export.service';
+
+/**
+ * Component for the statute paragraphs import list view.
+ */
+@Component({
+    selector: 'os-statute-import-list',
+    templateUrl: './statute-import-list.component.html'
+})
+export class StatuteImportListComponent extends BaseImportListComponent<ViewStatuteParagraph> {
+    /**
+     * Constructor for list view bases
+     *
+     * @param titleService the title serivce
+     * @param matSnackBar snackbar for displaying errors
+     * @param translate the translate service
+     * @param importer: The statute csv import service
+     * @param statuteCSVExport: service for exporting example data
+     */
+    public constructor(
+        titleService: Title,
+        matSnackBar: MatSnackBar,
+        translate: TranslateService,
+        importer: StatuteImportService,
+        private statuteCSVExport: StatuteCsvExportService
+    ) {
+        super(importer, titleService, translate, matSnackBar);
+    }
+
+    /**
+     * Returns the first characters of a string, for preview purposes
+     *
+     * @param input
+     */
+    public getShortPreview(input: string): string {
+        if (input.length > 50) {
+            return this.stripHtmlTags(input.substring(0, 47)) + '...';
+        }
+        return this.stripHtmlTags(input);
+    }
+
+    /**
+     * Returns the first and last 150 characters of a string; used within
+     * tooltips for the preview
+     *
+     * @param input
+     */
+    public getLongPreview(input: string): string {
+        if (input.length < 300) {
+            return this.stripHtmlTags(input);
+        }
+        return (
+            this.stripHtmlTags(input.substring(0, 147)) +
+            ' [...] ' +
+            this.stripHtmlTags(input.substring(input.length - 150, input.length))
+        );
+    }
+
+    /**
+     * Helper to remove html tags from a string.
+     * CAUTION: It is just a basic "don't show distracting html tags in a
+     * preview", not an actual tested sanitizer!
+     * @param inputString
+     */
+    private stripHtmlTags(inputString: string): string {
+        const regexp = new RegExp(/<[^ ][^<>]*(>|$)/g);
+        return inputString.replace(regexp, '').trim();
+    }
+
+    /**
+     * Triggers an example csv download
+     */
+    public downloadCsvExample(): void {
+        this.statuteCSVExport.exportDummyCSV();
+    }
+}

client/src/app/site/motions/components/statute-paragraph-list/statute-paragraph-list.component.html

@@ -40,7 +40,7 @@
         <button mat-button (click)="create()">
             <span translate>Save</span>
         </button>
-        <button mat-button (click)="onCancel()">
+        <button mat-button (click)="onCancelCreate()">
             <span translate>Cancel</span>
         </button>
     </mat-card-actions>
@@ -110,8 +110,12 @@
 </mat-card>
 
 <mat-menu #commentMenu="matMenu">
-    <button mat-menu-item (click)="sortStatuteParagraphs()">
+    <button mat-menu-item (click)="onCsvExport()">
-        <mat-icon>sort</mat-icon>
+        <mat-icon>archive</mat-icon>
-        <span translate>Sort ...</span>
+        <span translate>Export as CSV</span>
+    </button>
+    <button mat-menu-item *osPerms="'motions.can_manage'" routerLink="import">
+        <mat-icon>save_alt</mat-icon>
+        <span translate>Import</span><span>&nbsp;...</span>
     </button>
 </mat-menu>

client/src/app/site/motions/components/statute-paragraph-list/statute-paragraph-list.component.ts

@@ -10,6 +10,7 @@ import { ViewStatuteParagraph } from '../../models/view-statute-paragraph';
 import { StatuteParagraphRepositoryService } from '../../services/statute-paragraph-repository.service';
 import { BaseViewComponent } from '../../../base/base-view';
 import { MatSnackBar } from '@angular/material';
+import { StatuteCsvExportService } from '../../services/statute-csv-export.service';
 
 /**
  * List view for the statute paragraphs.
@@ -38,13 +39,15 @@ export class StatuteParagraphListComponent extends BaseViewComponent implements
     public editId: number | null;
 
     /**
-     * The usual component constructor
+     * The usual component constructor. Initializes the forms
+     *
      * @param titleService
      * @param translate
      * @param matSnackBar
      * @param repo
      * @param formBuilder
      * @param promptService
+     * @param csvExportService
      */
     public constructor(
         titleService: Title,
@@ -52,7 +55,8 @@ export class StatuteParagraphListComponent extends BaseViewComponent implements
         matSnackBar: MatSnackBar,
         private repo: StatuteParagraphRepositoryService,
         private formBuilder: FormBuilder,
-        private promptService: PromptService
+        private promptService: PromptService,
+        private csvExportService: StatuteCsvExportService
     ) {
         super(titleService, translate, matSnackBar);
 
@@ -200,4 +204,11 @@ export class StatuteParagraphListComponent extends BaseViewComponent implements
     public onCancelUpdate(): void {
         this.editId = null;
     }
+
+    /**
+     * Triggers a csv export of the statute paragraphs
+     */
+    public onCsvExport(): void {
+        this.csvExportService.exportStatutes(this.statuteParagraphs);
+    }
 }

client/src/app/site/motions/motions-routing.module.ts

@@ -11,6 +11,7 @@ import { MotionDetailComponent } from './components/motion-detail/motion-detail.
 import { MotionImportListComponent } from './components/motion-import-list/motion-import-list.component';
 import { MotionListComponent } from './components/motion-list/motion-list.component';
 import { SpeakerListComponent } from '../agenda/components/speaker-list/speaker-list.component';
+import { StatuteImportListComponent } from './components/statute-paragraph-list/statute-import-list/statute-import-list.component';
 import { StatuteParagraphListComponent } from './components/statute-paragraph-list/statute-paragraph-list.component';
 
 const routes: Routes = [
@@ -18,6 +19,7 @@ const routes: Routes = [
     { path: 'category', component: CategoryListComponent },
     { path: 'comment-section', component: MotionCommentSectionListComponent },
     { path: 'statute-paragraphs', component: StatuteParagraphListComponent },
+    { path: 'statute-paragraphs/import', component: StatuteImportListComponent },
     { path: 'call-list', component: CallListComponent },
     { path: 'blocks', component: MotionBlockListComponent },
     { path: 'blocks/:id', component: MotionBlockDetailComponent },

client/src/app/site/motions/motions.module.ts

@@ -22,6 +22,7 @@ import { ManageSubmittersComponent } from './components/manage-submitters/manage
 import { MotionPollComponent } from './components/motion-poll/motion-poll.component';
 import { MotionPollDialogComponent } from './components/motion-poll/motion-poll-dialog.component';
 import { MotionExportDialogComponent } from './components/motion-export-dialog/motion-export-dialog.component';
+import { StatuteImportListComponent } from './components/statute-paragraph-list/statute-import-list/statute-import-list.component';
 
 @NgModule({
     imports: [CommonModule, MotionsRoutingModule, SharedModule],
@@ -44,7 +45,8 @@ import { MotionExportDialogComponent } from './components/motion-export-dialog/m
         ManageSubmittersComponent,
         MotionPollComponent,
         MotionPollDialogComponent,
-        MotionExportDialogComponent
+        MotionExportDialogComponent,
+        StatuteImportListComponent
     ],
     entryComponents: [
         MotionChangeRecommendationComponent,

client/src/app/site/motions/services/statute-csv-export.service.spec.ts

@@ -0,0 +1,17 @@
+import { TestBed, inject } from '@angular/core/testing';
+
+import { E2EImportsModule } from 'e2e-imports.module';
+import { StatuteCsvExportService } from './statute-csv-export.service';
+
+describe('StatuteCsvExportService', () => {
+    beforeEach(() => {
+        TestBed.configureTestingModule({
+            imports: [E2EImportsModule],
+            providers: [StatuteCsvExportService]
+        });
+    });
+
+    it('should be created', inject([StatuteCsvExportService], (service: StatuteCsvExportService) => {
+        expect(service).toBeTruthy();
+    }));
+});

client/src/app/site/motions/services/statute-csv-export.service.ts

@@ -0,0 +1,58 @@
+import { Injectable } from '@angular/core';
+
+import { TranslateService } from '@ngx-translate/core';
+
+import { CsvExportService, CsvColumnDefinitionProperty } from 'app/core/services/csv-export.service';
+import { ViewStatuteParagraph } from '../models/view-statute-paragraph';
+import { FileExportService } from 'app/core/services/file-export.service';
+
+/**
+ * Exports CSVs for statute paragraphs.
+ */
+@Injectable({
+    providedIn: 'root'
+})
+export class StatuteCsvExportService {
+    /**
+     * Does nothing.
+     *
+     * @param csvExport CsvExportService
+     * @param translate TranslateService
+     * @param fileExport FileExportService
+     */
+    public constructor(
+        private csvExport: CsvExportService,
+        private translate: TranslateService,
+        private fileExport: FileExportService
+    ) {}
+
+    /**
+     * Export all statute paragraphs as CSV
+     *
+     * @param statute statute PParagraphs to export
+     */
+    public exportStatutes(statutes: ViewStatuteParagraph[]): void {
+        const exportProperties: CsvColumnDefinitionProperty<ViewStatuteParagraph>[] = [
+            { property: 'title' },
+            { property: 'text' }
+        ];
+        this.csvExport.export(statutes, exportProperties, this.translate.instant('Statutes') + '.csv');
+    }
+
+    /**
+     * Exports a short example file
+     */
+    public exportDummyCSV(): void {
+        const headerRow = ['Title', 'Text'].map(item => this.translate.instant(item)).join(',');
+        const rows = [
+            headerRow,
+            '§1,"This is the first section"',
+            '"§1, A 3", "This is another important aspect"',
+            '§2,Yet another'
+        ];
+        this.fileExport.saveFile(
+            rows.join('\n'),
+            `${this.translate.instant('Statutes')} - ${this.translate.instant('example')}.csv`
+        );
+    }
+}

client/src/app/site/motions/services/statute-import.service.ts

@@ -0,0 +1,98 @@
+import { Injectable } from '@angular/core';
+import { MatSnackBar } from '@angular/material';
+import { Papa } from 'ngx-papaparse';
+import { TranslateService } from '@ngx-translate/core';
+
+import { BaseImportService, NewEntry } from 'app/core/services/base-import.service';
+import { StatuteParagraph } from 'app/shared/models/motions/statute-paragraph';
+import { StatuteParagraphRepositoryService } from './statute-paragraph-repository.service';
+import { ViewStatuteParagraph } from '../models/view-statute-paragraph';
+
+/**
+ * Service for motion imports
+ */
+@Injectable({
+    providedIn: 'root'
+})
+export class StatuteImportService extends BaseImportService<ViewStatuteParagraph> {
+    /**
+     * List of possible errors and their verbose explanation
+     */
+    public errorList = {
+        Duplicates: 'A statute with this title already exists.'
+    };
+
+    /**
+     * The minimimal number of header entries needed to successfully create an entry
+     */
+    public requiredHeaderLength = 2;
+
+    /**
+     * Constructor. Defines the headers expected and calls the abstract class
+     * @param repo: The repository for statuteparagraphs.
+     * @param translate Translation service
+     * @param papa External csv parser (ngx-papaparser)
+     * @param matSnackBar snackBar to display import errors
+     */
+    public constructor(
+        private repo: StatuteParagraphRepositoryService,
+        translate: TranslateService,
+        papa: Papa,
+        matSnackbar: MatSnackBar
+    ) {
+        super(translate, papa, matSnackbar);
+
+        this.expectedHeader = ['title', 'text'];
+    }
+
+    /**
+     * Clears all temporary data specific to this importer.
+     */
+    public clearData(): void {
+        // does nothing
+    }
+
+    /**
+     * Parses a string representing an entry, extracting secondary data, appending
+     * the array of secondary imports as needed
+     *
+     * @param line
+     * @returns a new Entry representing a Motion
+     */
+    public mapData(line: string): NewEntry<ViewStatuteParagraph> {
+        const newEntry = new ViewStatuteParagraph(new StatuteParagraph());
+        const headerLength = Math.min(this.expectedHeader.length, line.length);
+        for (let idx = 0; idx < headerLength; idx++) {
+            switch (this.expectedHeader[idx]) {
+                case 'title':
+                    newEntry.statuteParagraph.title = line[idx];
+                    break;
+                case 'text':
+                    newEntry.statuteParagraph.text = line[idx];
+                    break;
+            }
+        }
+        const updateModels = this.repo.getViewModelList().filter(paragraph => paragraph.title === newEntry.title);
+        return {
+            newEntry: newEntry,
+            duplicates: updateModels,
+            status: updateModels.length ? 'error' : 'new',
+            errors: updateModels.length ? ['Duplicates'] : []
+        };
+    }
+
+    /**
+     * Executes the import. Creates all entries without errors by submitting
+     * them to the server. The entries will receive the status 'done' on success.
+     */
+    public async doImport(): Promise<void> {
+        for (const entry of this.entries) {
+            if (entry.status !== 'new') {
+                continue;
+            }
+            await this.repo.create(entry.newEntry.statuteParagraph);
+            entry.status = 'done';
+        }
+        this.updatePreview();
+    }
+}