From f3fe98436e670be27ea71b93c652f1dfead7383f Mon Sep 17 00:00:00 2001
From: Sean <sean.f.t.engelhardt@gmail.com>
Date: Thu, 3 Sep 2020 17:10:43 +0200
Subject: [PATCH] Vscroll for user import

Allows to import giant sets of users as CSV.
Tested 500k. The client is fine. The python server and the SQL data base really do not like that.
---
 .../list-view-table.component.scss            |  10 +
 client/src/app/site/base/base-import-list.ts  |  46 ++--
 .../user-import-list.component.html           | 204 ++++--------------
 .../user-import-list.component.scss           |   8 +
 .../user-import/user-import-list.component.ts |  59 ++++-
 client/src/styles.scss                        |  10 -
 6 files changed, 149 insertions(+), 188 deletions(-)
 create mode 100644 client/src/app/site/users/components/user-import/user-import-list.component.scss

diff --git a/client/src/app/shared/components/list-view-table/list-view-table.component.scss b/client/src/app/shared/components/list-view-table/list-view-table.component.scss
index 0c3e27635..395e1e535 100644
--- a/client/src/app/shared/components/list-view-table/list-view-table.component.scss
+++ b/client/src/app/shared/components/list-view-table/list-view-table.component.scss
@@ -17,6 +17,16 @@ $pbl-height: var(--pbl-height);
         display: block;
     }
 
+    /**
+    * Compilcated ngrid hack: The meta row won't disappear (just like that)
+    * Select the first ever container pbl-ngrid-container div and hide
+    */
+    .pbl-ngrid-container {
+        > div {
+            height: 0;
+        }
+    }
+
     .vscroll-list-view {
         flex: 1 1 auto;
         height: 100%;
diff --git a/client/src/app/site/base/base-import-list.ts b/client/src/app/site/base/base-import-list.ts
index e783ca690..7096dba17 100644
--- a/client/src/app/site/base/base-import-list.ts
+++ b/client/src/app/site/base/base-import-list.ts
@@ -5,7 +5,8 @@ import { MatTable, MatTableDataSource } from '@angular/material/table';
 import { Title } from '@angular/platform-browser';
 
 import { TranslateService } from '@ngx-translate/core';
-import { auditTime } from 'rxjs/operators';
+import { createDS, PblDataSource } from '@pebula/ngrid';
+import { auditTime, distinctUntilChanged } from 'rxjs/operators';
 
 import { BaseImportService, NewEntry, ValueLabelCombination } from 'app/core/ui-services/base-import.service';
 import { BaseModel } from 'app/shared/models/base/base-model';
@@ -20,6 +21,11 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
      */
     public dataSource: MatTableDataSource<NewEntry<M>>;
 
+    /**
+     * Data source for ngrid
+     */
+    public vScrollDataSource: PblDataSource<NewEntry<M>>;
+
     /**
      * Helper function for previews
      */
@@ -136,14 +142,23 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
      */
     public initTable(): void {
         this.dataSource = new MatTableDataSource();
-        this.setFilter();
-        this.importer
-            .getNewEntries()
-            .pipe(auditTime(100))
-            .subscribe(newEntries => {
-                this.dataSource.data = newEntries;
+
+        const entryObservable = this.importer.getNewEntries();
+        this.subscriptions.push(
+            entryObservable.pipe(distinctUntilChanged(), auditTime(100)).subscribe(newEntries => {
+                if (newEntries?.length) {
+                    this.dataSource.data = newEntries;
+                }
                 this.hasFile = newEntries.length > 0;
-            });
+            })
+        );
+
+        this.vScrollDataSource = createDS<NewEntry<M>>()
+            .keepAlive()
+            .onTrigger(() => entryObservable)
+            .create();
+
+        this.setFilter();
     }
 
     /**
@@ -180,21 +195,26 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
     public setFilter(): void {
         this.dataSource.filter = '';
         if (this.shown === 'all') {
-            this.dataSource.filterPredicate = (data, filter) => {
-                return true;
-            };
+            this.dataSource.filterPredicate = () => true;
+            this.vScrollDataSource.setFilter();
         } else if (this.shown === 'noerror') {
-            this.dataSource.filterPredicate = (data, filter) => {
+            const noErrorFilter = data => {
                 if (data.status === 'done') {
                     return true;
                 } else if (data.status !== 'error') {
                     return true;
                 }
             };
+
+            this.dataSource.filterPredicate = noErrorFilter;
+            this.vScrollDataSource.setFilter(noErrorFilter);
         } else if (this.shown === 'error') {
-            this.dataSource.filterPredicate = (data, filter) => {
+            const hasErrorFilter = data => {
                 return !!data.errors.length || data.hasDuplicates;
             };
+
+            this.dataSource.filterPredicate = hasErrorFilter;
+            this.vScrollDataSource.setFilter(hasErrorFilter);
         }
         this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter
     }
diff --git a/client/src/app/site/users/components/user-import/user-import-list.component.html b/client/src/app/site/users/components/user-import/user-import-list.component.html
index c42c2a2b8..9fa75eeda 100644
--- a/client/src/app/site/users/components/user-import/user-import-list.component.html
+++ b/client/src/app/site/users/components/user-import/user-import-list.component.html
@@ -51,7 +51,7 @@
             </span>
             <br />
             <div class="code red-warning-text">
-                <span *ngFor="let entry of headerRow; let last = last">
+                <span *ngFor="let entry of headerRowDefinition; let last = last">
                     {{ entry | translate }}<span *ngIf="!last">, </span>
                 </span>
             </div>
@@ -126,9 +126,9 @@
 </mat-card>
 
 <!-- preview table -->
-<mat-card *ngIf="hasFile" class="os-form-card import-table spacer-bottom-60">
+<mat-card *ngIf="hasFile" class="os-form-card spacer-bottom-60">
     <h3>{{ 'Preview' | translate }}</h3>
-    <div class="summary">
+    <div>
         <!-- new entries -->
         <div *ngIf="newCount">
             &nbsp;
@@ -151,168 +151,54 @@
     <div *ngIf="newCount">
         <span>{{ 'After verifiy the preview click on "import" please (see top right).' | translate }}</span>
     </div>
+
     <mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()">
         <mat-option value="all">{{ 'Show all' | translate }}</mat-option>
         <mat-option value="error">{{ 'Show errors only' | translate }}</mat-option>
         <mat-option value="noerror">{{ 'Show correct entries only' | translate }}</mat-option>
     </mat-select>
-    <div class="table-container">
-        <table mat-table [dataSource]="dataSource" matSort>
-            <!-- Status column -->
-            <ng-container matColumnDef="status" sticky>
-                <mat-header-cell *matHeaderCellDef class="first-column"></mat-header-cell>
-                <mat-cell *matCellDef="let entry" class="first-column">
-                    <div *ngIf="entry.status === 'error'">
-                        <mat-icon
-                            class="red-warning-text"
-                            matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
-                        >
-                            {{ getActionIcon(entry) }}
-                        </mat-icon>
-                        <mat-icon
-                            color="warn"
-                            *ngIf="hasError(entry, 'ParsingErrors')"
-                            matTooltip="{{ getVerboseError('ParsingErrors') | translate }}"
-                        >
-                            warning
-                        </mat-icon>
-                    </div>
-                    <div *ngIf="entry.status === 'new'">
-                        <mat-icon matTooltip="{{ 'Participant will be imported' | translate }}">
-                            {{ getActionIcon(entry) }}
-                        </mat-icon>
-                    </div>
-                    <div *ngIf="entry.status === 'done'">
-                        <mat-icon matTooltip="{{ 'Participant has been imported' | translate }}">
-                            {{ getActionIcon(entry) }}
-                        </mat-icon>
-                    </div>
-                </mat-cell>
-            </ng-container>
 
-            <!-- Title column -->
-            <ng-container matColumnDef="title">
-                <mat-header-cell *matHeaderCellDef>{{ 'Title' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry">
-                    <span *ngIf="nameErrors(entry)">
-                        <mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
-                            warning
-                        </mat-icon>
-                        &nbsp;
-                    </span>
-                    {{ entry.newEntry.title }}
-                </mat-cell>
-            </ng-container>
+    <div>
+        <pbl-ngrid
+            class="import-preview-table"
+            vScrollFixed="50"
+            [showHeader]="true"
+            [dataSource]="vScrollDataSource"
+            [columns]="columnSet"
+        >
+            <!-- ngrid template for boolean values -->
+            <div *pblNgridCellTypeDef="'boolean'; value as value">
+                <mat-checkbox disabled [checked]="value"></mat-checkbox>
+            </div>
 
-            <!-- title column -->
-            <ng-container matColumnDef="first_name">
-                <mat-header-cell *matHeaderCellDef>{{ 'Given name' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry">
-                    <span *ngIf="nameErrors(entry)">
-                        <mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
-                            warning
-                        </mat-icon>
-                        &nbsp;
-                    </span>
-                    {{ entry.newEntry.first_name }}
-                </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="last_name">
-                <mat-header-cell *matHeaderCellDef>{{ 'Surname' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry">
-                    <span *ngIf="nameErrors(entry)">
-                        <mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
-                            warning
-                        </mat-icon>
-                        &nbsp;
-                    </span>
-                    {{ entry.newEntry.last_name }}
-                </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="structure_level">
-                <mat-header-cell *matHeaderCellDef>{{ 'Structure level' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.structure_level }} </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="number">
-                <mat-header-cell *matHeaderCellDef>{{ 'Participant number' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.number }} </mat-cell>
-            </ng-container>
-
-            <!-- groups column -->
-            <ng-container matColumnDef="groups_id">
-                <mat-header-cell *matHeaderCellDef>{{ 'Groups' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry">
-                    <div *ngIf="entry.newEntry.csvGroups.length">
-                        <span *ngIf="hasError(entry, 'Groups')">
-                            <mat-icon color="warn" matTooltip="{{ getVerboseError('Groups') | translate }}">
-                                warning
-                            </mat-icon>
-                        </span>
-                        <span *ngFor="let group of entry.newEntry.csvGroups">
-                            {{ group.name }}
-                            <mat-icon class="newBadge" color="accent" inline *ngIf="!group.id">add</mat-icon>
-                            &nbsp;
-                        </span>
-                    </div>
-                </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="comment">
-                <mat-header-cell *matHeaderCellDef>{{ 'Comment' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.comment }} </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="is_active">
-                <mat-header-cell *matHeaderCellDef>{{ 'Is active' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry">
-                    <mat-checkbox disabled [checked]="entry.newEntry.is_active"> </mat-checkbox>
-                </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="is_present">
-                <mat-header-cell *matHeaderCellDef>{{ 'Is present' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry">
-                    <mat-checkbox disabled [checked]="entry.newEntry.is_present"> </mat-checkbox>
-                </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="is_committee">
-                <mat-header-cell *matHeaderCellDef>{{ 'Is committee' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry">
-                    <mat-checkbox disabled [checked]="entry.newEntry.is_committee"> </mat-checkbox>
-                </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="default_password">
-                <mat-header-cell *matHeaderCellDef>{{ 'Initial password' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.default_password }} </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="email">
-                <mat-header-cell *matHeaderCellDef>{{ 'Email' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.email }} </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="username">
-                <mat-header-cell *matHeaderCellDef>{{ 'Username' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.username }} </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="gender">
-                <mat-header-cell *matHeaderCellDef>{{ 'Gender' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.gender }} </mat-cell>
-            </ng-container>
-
-            <ng-container matColumnDef="vote_weight">
-                <mat-header-cell *matHeaderCellDef>{{ 'Vote weight' | translate }}</mat-header-cell>
-                <mat-cell *matCellDef="let entry"> {{ entry.newEntry.vote_weight }} </mat-cell>
-            </ng-container>
-
-            <mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
-            <mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
-        </table>
+            <!-- special row handling for the status column -->
+            <div *pblNgridCellDef="'status'; row as entry">
+                <div *ngIf="entry.status === 'error'">
+                    <mat-icon
+                        class="red-warning-text"
+                        matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
+                    >
+                        {{ getActionIcon(entry) }}
+                    </mat-icon>
+                    <mat-icon
+                        color="warn"
+                        *ngIf="hasError(entry, 'ParsingErrors')"
+                        matTooltip="{{ getVerboseError('ParsingErrors') | translate }}"
+                    >
+                        warning
+                    </mat-icon>
+                </div>
+                <div *ngIf="entry.status === 'new'">
+                    <mat-icon matTooltip="{{ 'Participant will be imported' | translate }}">
+                        {{ getActionIcon(entry) }}
+                    </mat-icon>
+                </div>
+                <div *ngIf="entry.status === 'done'">
+                    <mat-icon matTooltip="{{ 'Participant has been imported' | translate }}">
+                        {{ getActionIcon(entry) }}
+                    </mat-icon>
+                </div>
+            </div>
+        </pbl-ngrid>
     </div>
 </mat-card>
diff --git a/client/src/app/site/users/components/user-import/user-import-list.component.scss b/client/src/app/site/users/components/user-import/user-import-list.component.scss
new file mode 100644
index 000000000..a993beed9
--- /dev/null
+++ b/client/src/app/site/users/components/user-import/user-import-list.component.scss
@@ -0,0 +1,8 @@
+.import-preview-table {
+    display: block;
+    height: calc(100vh - 200px);
+}
+
+.pbl-ngrid-row {
+    height: 50px;
+}
diff --git a/client/src/app/site/users/components/user-import/user-import-list.component.ts b/client/src/app/site/users/components/user-import/user-import-list.component.ts
index be4987692..7e9dde1da 100644
--- a/client/src/app/site/users/components/user-import/user-import-list.component.ts
+++ b/client/src/app/site/users/components/user-import/user-import-list.component.ts
@@ -1,9 +1,10 @@
-import { Component } from '@angular/core';
+import { Component, ViewEncapsulation } from '@angular/core';
 import { FormBuilder, FormGroup } from '@angular/forms';
 import { MatSnackBar } from '@angular/material/snack-bar';
 import { Title } from '@angular/platform-browser';
 
 import { TranslateService } from '@ngx-translate/core';
+import { columnFactory, PblColumnDefinition } from '@pebula/ngrid';
 
 import { NewEntry } from 'app/core/ui-services/base-import.service';
 import { CsvExportService } from 'app/core/ui-services/csv-export.service';
@@ -16,12 +17,14 @@ import { UserImportService } from '../../services/user-import.service';
  */
 @Component({
     selector: 'os-user-import-list',
-    templateUrl: './user-import-list.component.html'
+    templateUrl: './user-import-list.component.html',
+    styleUrls: ['./user-import-list.component.scss'],
+    encapsulation: ViewEncapsulation.None
 })
 export class UserImportListComponent extends BaseImportListComponentDirective<User> {
     public textAreaForm: FormGroup;
 
-    public headerRow = [
+    public headerRowDefinition = [
         'Title',
         'Given name',
         'Surname',
@@ -39,6 +42,29 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
         'Vote weight'
     ];
 
+    private statusImportColumn: PblColumnDefinition = {
+        label: this.translate.instant('Status'),
+        prop: `status`
+    };
+
+    private get generateImportColumns(): PblColumnDefinition[] {
+        return this.importer.headerMap.map((property, index: number) => {
+            const singleColumnDef: PblColumnDefinition = {
+                label: this.translate.instant(this.headerRowDefinition[index]),
+                prop: `newEntry.${property}`,
+                type: this.guessType(property as keyof User)
+            };
+            console.log('singleColumnDef ', singleColumnDef);
+
+            return singleColumnDef;
+        });
+    }
+
+    public columnSet = columnFactory()
+        .default({ minWidth: 150 })
+        .table(this.statusImportColumn, ...this.generateImportColumns)
+        .build();
+
     /**
      * Constructor for list view bases
      *
@@ -55,7 +81,7 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
         formBuilder: FormBuilder,
         public translate: TranslateService,
         private exporter: CsvExportService,
-        importer: UserImportService
+        protected importer: UserImportService
     ) {
         super(importer, titleService, translate, matSnackBar);
         this.textAreaForm = formBuilder.group({ inputtext: [''] });
@@ -103,7 +129,28 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
             [null, 'Julia', 'Bloggs', 'London', null, null, null, null, null, null, null, null, 'jbloggs', 'f', 1.5],
             [null, null, 'Executive Board', null, null, null, null, null, null, 1, null, null, 'executive', null, 2.5]
         ];
-        this.exporter.dummyCSVExport(this.headerRow, rows, `${this.translate.instant('participants-example')}.csv`);
+        this.exporter.dummyCSVExport(
+            this.headerRowDefinition,
+            rows,
+            `${this.translate.instant('participants-example')}.csv`
+        );
+    }
+
+    /**
+     * Guess the type of the property, since
+     * `const type = typeof User[property];`
+     * always returns undefined
+     */
+    private guessType(userProperty: keyof User): 'string' | 'number' | 'boolean' {
+        const numberProperties: (keyof User)[] = ['id', 'vote_weight'];
+        const booleanProperties: (keyof User)[] = ['is_present', 'is_committee', 'is_active'];
+        if (numberProperties.includes(userProperty)) {
+            return 'number';
+        } else if (booleanProperties.includes(userProperty)) {
+            return 'boolean';
+        } else {
+            return 'string';
+        }
     }
 
     /**
@@ -125,7 +172,7 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
      * Sends the data in the text field input area to the importer
      */
     public parseTextArea(): void {
-        (this.importer as UserImportService).parseTextArea(this.textAreaForm.get('inputtext').value);
+        this.importer.parseTextArea(this.textAreaForm.get('inputtext').value);
     }
 
     /**
diff --git a/client/src/styles.scss b/client/src/styles.scss
index 476638e45..88b83ffa4 100644
--- a/client/src/styles.scss
+++ b/client/src/styles.scss
@@ -505,16 +505,6 @@ button.mat-menu-item.selected {
     width: -webkit-fill-available;
 }
 
-/**
- * Compilcated ngrid hack: The meta row won't disappear (just like that)
- * Select the first ever container pbl-ngrid-container div and hide
- */
-.pbl-ngrid-container {
-    > div {
-        height: 0;
-    }
-}
-
 .cdk-column-menu {
     padding: 0 16px 0 0 !important;
 }