ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Vscroll for user import

Browse Source 
Allows to import giant sets of users as CSV.
Tested 500k. The client is fine. The python server and the SQL data base really do not like that.
This commit is contained in:
Sean 2020-09-03 17:10:43 +02:00
parent 5b63809b12
commit f3fe98436e
6 changed files with 149 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
client/src/app/shared/components/list-view-table/list-view-table.component.scss
View File

@ -17,6 +17,16 @@ $pbl-height: var(--pbl-height);
display: block; display: block;
} }
/**
* Compilcated ngrid hack: The meta row won't disappear (just like that)
* Select the first ever container pbl-ngrid-container div and hide
*/
.pbl-ngrid-container {
> div {
height: 0;
}
}
.vscroll-list-view { .vscroll-list-view {
flex: 1 1 auto; flex: 1 1 auto;
height: 100%; height: 100%;

46
client/src/app/site/base/base-import-list.ts
View File

@ -5,7 +5,8 @@ import { MatTable, MatTableDataSource } from '@angular/material/table';
import { Title } from '@angular/platform-browser'; import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core'; import { TranslateService } from '@ngx-translate/core';
import { auditTime } from 'rxjs/operators'; import { createDS, PblDataSource } from '@pebula/ngrid';
import { auditTime, distinctUntilChanged } from 'rxjs/operators';
import { BaseImportService, NewEntry, ValueLabelCombination } from 'app/core/ui-services/base-import.service'; import { BaseImportService, NewEntry, ValueLabelCombination } from 'app/core/ui-services/base-import.service';
import { BaseModel } from 'app/shared/models/base/base-model'; import { BaseModel } from 'app/shared/models/base/base-model';
@ -20,6 +21,11 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
*/ */
public dataSource: MatTableDataSource<NewEntry<M>>; public dataSource: MatTableDataSource<NewEntry<M>>;
/**
* Data source for ngrid
*/
public vScrollDataSource: PblDataSource<NewEntry<M>>;
/** /**
* Helper function for previews * Helper function for previews
*/ */
@ -136,14 +142,23 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
*/ */
public initTable(): void { public initTable(): void {
this.dataSource = new MatTableDataSource(); this.dataSource = new MatTableDataSource();
this.setFilter();
this.importer const entryObservable = this.importer.getNewEntries();
.getNewEntries() this.subscriptions.push(
.pipe(auditTime(100)) entryObservable.pipe(distinctUntilChanged(), auditTime(100)).subscribe(newEntries => {
.subscribe(newEntries => { if (newEntries?.length) {
this.dataSource.data = newEntries; this.dataSource.data = newEntries;
}
this.hasFile = newEntries.length > 0; this.hasFile = newEntries.length > 0;
}); })
);
this.vScrollDataSource = createDS<NewEntry<M>>()
.keepAlive()
.onTrigger(() => entryObservable)
.create();
this.setFilter();
} }
/** /**
@ -180,21 +195,26 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
public setFilter(): void { public setFilter(): void {
this.dataSource.filter = ''; this.dataSource.filter = '';
if (this.shown === 'all') { if (this.shown === 'all') {
this.dataSource.filterPredicate = (data, filter) => { this.dataSource.filterPredicate = () => true;
return true; this.vScrollDataSource.setFilter();
};
} else if (this.shown === 'noerror') { } else if (this.shown === 'noerror') {
this.dataSource.filterPredicate = (data, filter) => { const noErrorFilter = data => {
if (data.status === 'done') { if (data.status === 'done') {
return true; return true;
} else if (data.status !== 'error') { } else if (data.status !== 'error') {
return true; return true;
} }
}; };
this.dataSource.filterPredicate = noErrorFilter;
this.vScrollDataSource.setFilter(noErrorFilter);
} else if (this.shown === 'error') { } else if (this.shown === 'error') {
this.dataSource.filterPredicate = (data, filter) => { const hasErrorFilter = data => {
return !!data.errors.length || data.hasDuplicates; return !!data.errors.length || data.hasDuplicates;
}; };
this.dataSource.filterPredicate = hasErrorFilter;
this.vScrollDataSource.setFilter(hasErrorFilter);
} }
this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter
} }

204
client/src/app/site/users/components/user-import/user-import-list.component.html
View File

@ -51,7 +51,7 @@
</span> </span>
<br /> <br />
<div class="code red-warning-text"> <div class="code red-warning-text">
<span *ngFor="let entry of headerRow; let last = last"> <span *ngFor="let entry of headerRowDefinition; let last = last">
{{ entry | translate }}<span *ngIf="!last">, </span> {{ entry | translate }}<span *ngIf="!last">, </span>
</span> </span>
</div> </div>
@ -126,9 +126,9 @@
</mat-card> </mat-card>
<!-- preview table --> <!-- preview table -->
<mat-card *ngIf="hasFile" class="os-form-card import-table spacer-bottom-60"> <mat-card *ngIf="hasFile" class="os-form-card spacer-bottom-60">
<h3>{{ 'Preview' | translate }}</h3> <h3>{{ 'Preview' | translate }}</h3>
<div class="summary"> <div>
<!-- new entries --> <!-- new entries -->
<div *ngIf="newCount"> <div *ngIf="newCount">
&nbsp; &nbsp;
@ -151,168 +151,54 @@
<div *ngIf="newCount"> <div *ngIf="newCount">
<span>{{ 'After verifiy the preview click on "import" please (see top right).' | translate }}</span> <span>{{ 'After verifiy the preview click on "import" please (see top right).' | translate }}</span>
</div> </div>
<mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()"> <mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()">
<mat-option value="all">{{ 'Show all' | translate }}</mat-option> <mat-option value="all">{{ 'Show all' | translate }}</mat-option>
<mat-option value="error">{{ 'Show errors only' | translate }}</mat-option> <mat-option value="error">{{ 'Show errors only' | translate }}</mat-option>
<mat-option value="noerror">{{ 'Show correct entries only' | translate }}</mat-option> <mat-option value="noerror">{{ 'Show correct entries only' | translate }}</mat-option>
</mat-select> </mat-select>
<div class="table-container">
<table mat-table [dataSource]="dataSource" matSort>
<!-- Status column -->
<ng-container matColumnDef="status" sticky>
<mat-header-cell *matHeaderCellDef class="first-column"></mat-header-cell>
<mat-cell *matCellDef="let entry" class="first-column">
<div *ngIf="entry.status === 'error'">
<mat-icon
class="red-warning-text"
matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
>
{{ getActionIcon(entry) }}
</mat-icon>
<mat-icon
color="warn"
*ngIf="hasError(entry, 'ParsingErrors')"
matTooltip="{{ getVerboseError('ParsingErrors') | translate }}"
>
warning
</mat-icon>
</div>
<div *ngIf="entry.status === 'new'">
<mat-icon matTooltip="{{ 'Participant will be imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.status === 'done'">
<mat-icon matTooltip="{{ 'Participant has been imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
</mat-cell>
</ng-container>
<!-- Title column --> <div>
<ng-container matColumnDef="title"> <pbl-ngrid
<mat-header-cell *matHeaderCellDef>{{ 'Title' | translate }}</mat-header-cell> class="import-preview-table"
<mat-cell *matCellDef="let entry"> vScrollFixed="50"
<span *ngIf="nameErrors(entry)"> [showHeader]="true"
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}"> [dataSource]="vScrollDataSource"
warning [columns]="columnSet"
</mat-icon> >
&nbsp; <!-- ngrid template for boolean values -->
</span> <div *pblNgridCellTypeDef="'boolean'; value as value">
{{ entry.newEntry.title }} <mat-checkbox disabled [checked]="value"></mat-checkbox>
</mat-cell> </div>
</ng-container>
<!-- title column --> <!-- special row handling for the status column -->
<ng-container matColumnDef="first_name"> <div *pblNgridCellDef="'status'; row as entry">
<mat-header-cell *matHeaderCellDef>{{ 'Given name' | translate }}</mat-header-cell> <div *ngIf="entry.status === 'error'">
<mat-cell *matCellDef="let entry"> <mat-icon
<span *ngIf="nameErrors(entry)"> class="red-warning-text"
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}"> matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
warning >
</mat-icon> {{ getActionIcon(entry) }}
&nbsp; </mat-icon>
</span> <mat-icon
{{ entry.newEntry.first_name }} color="warn"
</mat-cell> *ngIf="hasError(entry, 'ParsingErrors')"
</ng-container> matTooltip="{{ getVerboseError('ParsingErrors') | translate }}"
>
<ng-container matColumnDef="last_name"> warning
<mat-header-cell *matHeaderCellDef>{{ 'Surname' | translate }}</mat-header-cell> </mat-icon>
<mat-cell *matCellDef="let entry"> </div>
<span *ngIf="nameErrors(entry)"> <div *ngIf="entry.status === 'new'">
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}"> <mat-icon matTooltip="{{ 'Participant will be imported' | translate }}">
warning {{ getActionIcon(entry) }}
</mat-icon> </mat-icon>
&nbsp; </div>
</span> <div *ngIf="entry.status === 'done'">
{{ entry.newEntry.last_name }} <mat-icon matTooltip="{{ 'Participant has been imported' | translate }}">
</mat-cell> {{ getActionIcon(entry) }}
</ng-container> </mat-icon>
</div>
<ng-container matColumnDef="structure_level"> </div>
<mat-header-cell *matHeaderCellDef>{{ 'Structure level' | translate }}</mat-header-cell> </pbl-ngrid>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.structure_level }} </mat-cell>
</ng-container>
<ng-container matColumnDef="number">
<mat-header-cell *matHeaderCellDef>{{ 'Participant number' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.number }} </mat-cell>
</ng-container>
<!-- groups column -->
<ng-container matColumnDef="groups_id">
<mat-header-cell *matHeaderCellDef>{{ 'Groups' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newEntry.csvGroups.length">
<span *ngIf="hasError(entry, 'Groups')">
<mat-icon color="warn" matTooltip="{{ getVerboseError('Groups') | translate }}">
warning
</mat-icon>
</span>
<span *ngFor="let group of entry.newEntry.csvGroups">
{{ group.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!group.id">add</mat-icon>
&nbsp;
</span>
</div>
</mat-cell>
</ng-container>
<ng-container matColumnDef="comment">
<mat-header-cell *matHeaderCellDef>{{ 'Comment' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.comment }} </mat-cell>
</ng-container>
<ng-container matColumnDef="is_active">
<mat-header-cell *matHeaderCellDef>{{ 'Is active' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_active"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="is_present">
<mat-header-cell *matHeaderCellDef>{{ 'Is present' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_present"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="is_committee">
<mat-header-cell *matHeaderCellDef>{{ 'Is committee' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_committee"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="default_password">
<mat-header-cell *matHeaderCellDef>{{ 'Initial password' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.default_password }} </mat-cell>
</ng-container>
<ng-container matColumnDef="email">
<mat-header-cell *matHeaderCellDef>{{ 'Email' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.email }} </mat-cell>
</ng-container>
<ng-container matColumnDef="username">
<mat-header-cell *matHeaderCellDef>{{ 'Username' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.username }} </mat-cell>
</ng-container>
<ng-container matColumnDef="gender">
<mat-header-cell *matHeaderCellDef>{{ 'Gender' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.gender }} </mat-cell>
</ng-container>
<ng-container matColumnDef="vote_weight">
<mat-header-cell *matHeaderCellDef>{{ 'Vote weight' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.vote_weight }} </mat-cell>
</ng-container>
<mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
<mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
</table>
</div> </div>
</mat-card> </mat-card>

8
client/src/app/site/users/components/user-import/user-import-list.component.scss Normal file
View File

@ -0,0 +1,8 @@
.import-preview-table {
display: block;
height: calc(100vh - 200px);
}
.pbl-ngrid-row {
height: 50px;
}

59
client/src/app/site/users/components/user-import/user-import-list.component.ts
View File

@ -1,9 +1,10 @@
import { Component } from '@angular/core'; import { Component, ViewEncapsulation } from '@angular/core';
import { FormBuilder, FormGroup } from '@angular/forms'; import { FormBuilder, FormGroup } from '@angular/forms';
import { MatSnackBar } from '@angular/material/snack-bar'; import { MatSnackBar } from '@angular/material/snack-bar';
import { Title } from '@angular/platform-browser'; import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core'; import { TranslateService } from '@ngx-translate/core';
import { columnFactory, PblColumnDefinition } from '@pebula/ngrid';
import { NewEntry } from 'app/core/ui-services/base-import.service'; import { NewEntry } from 'app/core/ui-services/base-import.service';
import { CsvExportService } from 'app/core/ui-services/csv-export.service'; import { CsvExportService } from 'app/core/ui-services/csv-export.service';
@ -16,12 +17,14 @@ import { UserImportService } from '../../services/user-import.service';
*/ */
@Component({ @Component({
selector: 'os-user-import-list', selector: 'os-user-import-list',
templateUrl: './user-import-list.component.html' templateUrl: './user-import-list.component.html',
styleUrls: ['./user-import-list.component.scss'],
encapsulation: ViewEncapsulation.None
}) })
export class UserImportListComponent extends BaseImportListComponentDirective<User> { export class UserImportListComponent extends BaseImportListComponentDirective<User> {
public textAreaForm: FormGroup; public textAreaForm: FormGroup;
public headerRow = [ public headerRowDefinition = [
'Title', 'Title',
'Given name', 'Given name',
'Surname', 'Surname',
@ -39,6 +42,29 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
'Vote weight' 'Vote weight'
]; ];
private statusImportColumn: PblColumnDefinition = {
label: this.translate.instant('Status'),
prop: `status`
};
private get generateImportColumns(): PblColumnDefinition[] {
return this.importer.headerMap.map((property, index: number) => {
const singleColumnDef: PblColumnDefinition = {
label: this.translate.instant(this.headerRowDefinition[index]),
prop: `newEntry.${property}`,
type: this.guessType(property as keyof User)
};
console.log('singleColumnDef ', singleColumnDef);
return singleColumnDef;
});
}
public columnSet = columnFactory()
.default({ minWidth: 150 })
.table(this.statusImportColumn, ...this.generateImportColumns)
.build();
/** /**
* Constructor for list view bases * Constructor for list view bases
* *
@ -55,7 +81,7 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
formBuilder: FormBuilder, formBuilder: FormBuilder,
public translate: TranslateService, public translate: TranslateService,
private exporter: CsvExportService, private exporter: CsvExportService,
importer: UserImportService protected importer: UserImportService
) { ) {
super(importer, titleService, translate, matSnackBar); super(importer, titleService, translate, matSnackBar);
this.textAreaForm = formBuilder.group({ inputtext: [''] }); this.textAreaForm = formBuilder.group({ inputtext: [''] });
@ -103,7 +129,28 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
[null, 'Julia', 'Bloggs', 'London', null, null, null, null, null, null, null, null, 'jbloggs', 'f', 1.5], [null, 'Julia', 'Bloggs', 'London', null, null, null, null, null, null, null, null, 'jbloggs', 'f', 1.5],
[null, null, 'Executive Board', null, null, null, null, null, null, 1, null, null, 'executive', null, 2.5] [null, null, 'Executive Board', null, null, null, null, null, null, 1, null, null, 'executive', null, 2.5]
]; ];
this.exporter.dummyCSVExport(this.headerRow, rows, `${this.translate.instant('participants-example')}.csv`); this.exporter.dummyCSVExport(
this.headerRowDefinition,
rows,
`${this.translate.instant('participants-example')}.csv`
);
}
/**
* Guess the type of the property, since
* `const type = typeof User[property];`
* always returns undefined
*/
private guessType(userProperty: keyof User): 'string' | 'number' | 'boolean' {
const numberProperties: (keyof User)[] = ['id', 'vote_weight'];
const booleanProperties: (keyof User)[] = ['is_present', 'is_committee', 'is_active'];
if (numberProperties.includes(userProperty)) {
return 'number';
} else if (booleanProperties.includes(userProperty)) {
return 'boolean';
} else {
return 'string';
}
} }
/** /**
@ -125,7 +172,7 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
* Sends the data in the text field input area to the importer * Sends the data in the text field input area to the importer
*/ */
public parseTextArea(): void { public parseTextArea(): void {
(this.importer as UserImportService).parseTextArea(this.textAreaForm.get('inputtext').value); this.importer.parseTextArea(this.textAreaForm.get('inputtext').value);
} }
/** /**

10
client/src/styles.scss
View File

@ -505,16 +505,6 @@ button.mat-menu-item.selected {
width: -webkit-fill-available; width: -webkit-fill-available;
} }
/**
* Compilcated ngrid hack: The meta row won't disappear (just like that)
* Select the first ever container pbl-ngrid-container div and hide
*/
.pbl-ngrid-container {
> div {
height: 0;
}
}
.cdk-column-menu { .cdk-column-menu {
padding: 0 16px 0 0 !important; padding: 0 16px 0 0 !important;
} }