ag-admin
/
OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Vscroll for user import 

Allows to import giant sets of users as CSV.
Tested 500k. The client is fine. The python server and the SQL data base really do not like that.
This commit is contained in:
Sean 2020-09-03 17:10:43 +02:00
parent 5b63809b12
commit f3fe98436e
6 changed files with 149 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
client/src/app/shared/components/list-view-table/list-view-table.component.scss
View File

@ -17,6 +17,16 @@ $pbl-height: var(--pbl-height);
display: block;
}
/**
* Compilcated ngrid hack: The meta row won't disappear (just like that)
* Select the first ever container pbl-ngrid-container div and hide
*/
.pbl-ngrid-container {
> div {
height: 0;
}
}
.vscroll-list-view {
flex: 1 1 auto;
height: 100%;

46
client/src/app/site/base/base-import-list.ts
View File

@ -5,7 +5,8 @@ import { MatTable, MatTableDataSource } from '@angular/material/table';
import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core';
import { auditTime } from 'rxjs/operators';
import { createDS, PblDataSource } from '@pebula/ngrid';
import { auditTime, distinctUntilChanged } from 'rxjs/operators';
import { BaseImportService, NewEntry, ValueLabelCombination } from 'app/core/ui-services/base-import.service';
import { BaseModel } from 'app/shared/models/base/base-model';
@ -20,6 +21,11 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
*/
public dataSource: MatTableDataSource<NewEntry<M>>;
/**
* Data source for ngrid
*/
public vScrollDataSource: PblDataSource<NewEntry<M>>;
/**
* Helper function for previews
*/
@ -136,14 +142,23 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
*/
public initTable(): void {
this.dataSource = new MatTableDataSource();
this.setFilter();
this.importer
.getNewEntries()
.pipe(auditTime(100))
.subscribe(newEntries => {
this.dataSource.data = newEntries;
const entryObservable = this.importer.getNewEntries();
this.subscriptions.push(
entryObservable.pipe(distinctUntilChanged(), auditTime(100)).subscribe(newEntries => {
if (newEntries?.length) {
this.dataSource.data = newEntries;
}
this.hasFile = newEntries.length > 0;
});
})
);
this.vScrollDataSource = createDS<NewEntry<M>>()
.keepAlive()
.onTrigger(() => entryObservable)
.create();
this.setFilter();
}
/**
@ -180,21 +195,26 @@ export abstract class BaseImportListComponentDirective<M extends BaseModel> exte
public setFilter(): void {
this.dataSource.filter = '';
if (this.shown === 'all') {
this.dataSource.filterPredicate = (data, filter) => {
return true;
};
this.dataSource.filterPredicate = () => true;
this.vScrollDataSource.setFilter();
} else if (this.shown === 'noerror') {
this.dataSource.filterPredicate = (data, filter) => {
const noErrorFilter = data => {
if (data.status === 'done') {
return true;
} else if (data.status !== 'error') {
return true;
}
};
this.dataSource.filterPredicate = noErrorFilter;
this.vScrollDataSource.setFilter(noErrorFilter);
} else if (this.shown === 'error') {
this.dataSource.filterPredicate = (data, filter) => {
const hasErrorFilter = data => {
return !!data.errors.length || data.hasDuplicates;
};
this.dataSource.filterPredicate = hasErrorFilter;
this.vScrollDataSource.setFilter(hasErrorFilter);
}
this.dataSource.filter = 'X'; // TODO: This is just a bogus non-null string to trigger the filter
}

204
client/src/app/site/users/components/user-import/user-import-list.component.html
View File

@ -51,7 +51,7 @@
</span>
<br />
<div class="code red-warning-text">
<span *ngFor="let entry of headerRow; let last = last">
<span *ngFor="let entry of headerRowDefinition; let last = last">
{{ entry | translate }}<span *ngIf="!last">, </span>
</span>
</div>
@ -126,9 +126,9 @@
</mat-card>
<!-- preview table -->
<mat-card *ngIf="hasFile" class="os-form-card import-table spacer-bottom-60">
<mat-card *ngIf="hasFile" class="os-form-card spacer-bottom-60">
<h3>{{ 'Preview' | translate }}</h3>
<div class="summary">
<div>
<!-- new entries -->
<div *ngIf="newCount">
&nbsp;
@ -151,168 +151,54 @@
<div *ngIf="newCount">
<span>{{ 'After verifiy the preview click on "import" please (see top right).' | translate }}</span>
</div>
<mat-select *ngIf="nonImportableCount" class="filter-imports" [(value)]="shown" (selectionChange)="setFilter()">
<mat-option value="all">{{ 'Show all' | translate }}</mat-option>
<mat-option value="error">{{ 'Show errors only' | translate }}</mat-option>
<mat-option value="noerror">{{ 'Show correct entries only' | translate }}</mat-option>
</mat-select>
<div class="table-container">
<table mat-table [dataSource]="dataSource" matSort>
<!-- Status column -->
<ng-container matColumnDef="status" sticky>
<mat-header-cell *matHeaderCellDef class="first-column"></mat-header-cell>
<mat-cell *matCellDef="let entry" class="first-column">
<div *ngIf="entry.status === 'error'">
<mat-icon
class="red-warning-text"
matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
>
{{ getActionIcon(entry) }}
</mat-icon>
<mat-icon
color="warn"
*ngIf="hasError(entry, 'ParsingErrors')"
matTooltip="{{ getVerboseError('ParsingErrors') | translate }}"
>
warning
</mat-icon>
</div>
<div *ngIf="entry.status === 'new'">
<mat-icon matTooltip="{{ 'Participant will be imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.status === 'done'">
<mat-icon matTooltip="{{ 'Participant has been imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
</mat-cell>
</ng-container>
<!-- Title column -->
<ng-container matColumnDef="title">
<mat-header-cell *matHeaderCellDef>{{ 'Title' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<span *ngIf="nameErrors(entry)">
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
warning
</mat-icon>
&nbsp;
</span>
{{ entry.newEntry.title }}
</mat-cell>
</ng-container>
<div>
<pbl-ngrid
class="import-preview-table"
vScrollFixed="50"
[showHeader]="true"
[dataSource]="vScrollDataSource"
[columns]="columnSet"
>
<!-- ngrid template for boolean values -->
<div *pblNgridCellTypeDef="'boolean'; value as value">
<mat-checkbox disabled [checked]="value"></mat-checkbox>
</div>
<!-- title column -->
<ng-container matColumnDef="first_name">
<mat-header-cell *matHeaderCellDef>{{ 'Given name' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<span *ngIf="nameErrors(entry)">
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
warning
</mat-icon>
&nbsp;
</span>
{{ entry.newEntry.first_name }}
</mat-cell>
</ng-container>
<ng-container matColumnDef="last_name">
<mat-header-cell *matHeaderCellDef>{{ 'Surname' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<span *ngIf="nameErrors(entry)">
<mat-icon color="warn" inline matTooltip="{{ nameErrors(entry) | translate }}">
warning
</mat-icon>
&nbsp;
</span>
{{ entry.newEntry.last_name }}
</mat-cell>
</ng-container>
<ng-container matColumnDef="structure_level">
<mat-header-cell *matHeaderCellDef>{{ 'Structure level' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.structure_level }} </mat-cell>
</ng-container>
<ng-container matColumnDef="number">
<mat-header-cell *matHeaderCellDef>{{ 'Participant number' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.number }} </mat-cell>
</ng-container>
<!-- groups column -->
<ng-container matColumnDef="groups_id">
<mat-header-cell *matHeaderCellDef>{{ 'Groups' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<div *ngIf="entry.newEntry.csvGroups.length">
<span *ngIf="hasError(entry, 'Groups')">
<mat-icon color="warn" matTooltip="{{ getVerboseError('Groups') | translate }}">
warning
</mat-icon>
</span>
<span *ngFor="let group of entry.newEntry.csvGroups">
{{ group.name }}
<mat-icon class="newBadge" color="accent" inline *ngIf="!group.id">add</mat-icon>
&nbsp;
</span>
</div>
</mat-cell>
</ng-container>
<ng-container matColumnDef="comment">
<mat-header-cell *matHeaderCellDef>{{ 'Comment' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.comment }} </mat-cell>
</ng-container>
<ng-container matColumnDef="is_active">
<mat-header-cell *matHeaderCellDef>{{ 'Is active' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_active"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="is_present">
<mat-header-cell *matHeaderCellDef>{{ 'Is present' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_present"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="is_committee">
<mat-header-cell *matHeaderCellDef>{{ 'Is committee' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry">
<mat-checkbox disabled [checked]="entry.newEntry.is_committee"> </mat-checkbox>
</mat-cell>
</ng-container>
<ng-container matColumnDef="default_password">
<mat-header-cell *matHeaderCellDef>{{ 'Initial password' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.default_password }} </mat-cell>
</ng-container>
<ng-container matColumnDef="email">
<mat-header-cell *matHeaderCellDef>{{ 'Email' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.email }} </mat-cell>
</ng-container>
<ng-container matColumnDef="username">
<mat-header-cell *matHeaderCellDef>{{ 'Username' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.username }} </mat-cell>
</ng-container>
<ng-container matColumnDef="gender">
<mat-header-cell *matHeaderCellDef>{{ 'Gender' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.gender }} </mat-cell>
</ng-container>
<ng-container matColumnDef="vote_weight">
<mat-header-cell *matHeaderCellDef>{{ 'Vote weight' | translate }}</mat-header-cell>
<mat-cell *matCellDef="let entry"> {{ entry.newEntry.vote_weight }} </mat-cell>
</ng-container>
<mat-header-row *matHeaderRowDef="getColumnDefinition()"></mat-header-row>
<mat-row [ngClass]="getStateClass(row)" *matRowDef="let row; columns: getColumnDefinition()"> </mat-row>
</table>
<!-- special row handling for the status column -->
<div *pblNgridCellDef="'status'; row as entry">
<div *ngIf="entry.status === 'error'">
<mat-icon
class="red-warning-text"
matTooltip="{{ entry.errors.length }} {{ 'errors' | translate }}"
>
{{ getActionIcon(entry) }}
</mat-icon>
<mat-icon
color="warn"
*ngIf="hasError(entry, 'ParsingErrors')"
matTooltip="{{ getVerboseError('ParsingErrors') | translate }}"
>
warning
</mat-icon>
</div>
<div *ngIf="entry.status === 'new'">
<mat-icon matTooltip="{{ 'Participant will be imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
<div *ngIf="entry.status === 'done'">
<mat-icon matTooltip="{{ 'Participant has been imported' | translate }}">
{{ getActionIcon(entry) }}
</mat-icon>
</div>
</div>
</pbl-ngrid>
</div>
</mat-card>

8
client/src/app/site/users/components/user-import/user-import-list.component.scss Normal file
View File

@ -0,0 +1,8 @@
.import-preview-table {
display: block;
height: calc(100vh - 200px);
}
.pbl-ngrid-row {
height: 50px;
}

59
client/src/app/site/users/components/user-import/user-import-list.component.ts
View File

@ -1,9 +1,10 @@
import { Component } from '@angular/core';
import { Component, ViewEncapsulation } from '@angular/core';
import { FormBuilder, FormGroup } from '@angular/forms';
import { MatSnackBar } from '@angular/material/snack-bar';
import { Title } from '@angular/platform-browser';
import { TranslateService } from '@ngx-translate/core';
import { columnFactory, PblColumnDefinition } from '@pebula/ngrid';
import { NewEntry } from 'app/core/ui-services/base-import.service';
import { CsvExportService } from 'app/core/ui-services/csv-export.service';
@ -16,12 +17,14 @@ import { UserImportService } from '../../services/user-import.service';
*/
@Component({
selector: 'os-user-import-list',
templateUrl: './user-import-list.component.html'
templateUrl: './user-import-list.component.html',
styleUrls: ['./user-import-list.component.scss'],
encapsulation: ViewEncapsulation.None
})
export class UserImportListComponent extends BaseImportListComponentDirective<User> {
public textAreaForm: FormGroup;
public headerRow = [
public headerRowDefinition = [
'Title',
'Given name',
'Surname',
@ -39,6 +42,29 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
'Vote weight'
];
private statusImportColumn: PblColumnDefinition = {
label: this.translate.instant('Status'),
prop: `status`
};
private get generateImportColumns(): PblColumnDefinition[] {
return this.importer.headerMap.map((property, index: number) => {
const singleColumnDef: PblColumnDefinition = {
label: this.translate.instant(this.headerRowDefinition[index]),
prop: `newEntry.${property}`,
type: this.guessType(property as keyof User)
};
console.log('singleColumnDef ', singleColumnDef);
return singleColumnDef;
});
}
public columnSet = columnFactory()
.default({ minWidth: 150 })
.table(this.statusImportColumn, ...this.generateImportColumns)
.build();
/**
* Constructor for list view bases
*
@ -55,7 +81,7 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
formBuilder: FormBuilder,
public translate: TranslateService,
private exporter: CsvExportService,
importer: UserImportService
protected importer: UserImportService
) {
super(importer, titleService, translate, matSnackBar);
this.textAreaForm = formBuilder.group({ inputtext: [''] });
@ -103,7 +129,28 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
[null, 'Julia', 'Bloggs', 'London', null, null, null, null, null, null, null, null, 'jbloggs', 'f', 1.5],
[null, null, 'Executive Board', null, null, null, null, null, null, 1, null, null, 'executive', null, 2.5]
];
this.exporter.dummyCSVExport(this.headerRow, rows, `${this.translate.instant('participants-example')}.csv`);
this.exporter.dummyCSVExport(
this.headerRowDefinition,
rows,
`${this.translate.instant('participants-example')}.csv`
);
}
/**
* Guess the type of the property, since
* `const type = typeof User[property];`
* always returns undefined
*/
private guessType(userProperty: keyof User): 'string' | 'number' | 'boolean' {
const numberProperties: (keyof User)[] = ['id', 'vote_weight'];
const booleanProperties: (keyof User)[] = ['is_present', 'is_committee', 'is_active'];
if (numberProperties.includes(userProperty)) {
return 'number';
} else if (booleanProperties.includes(userProperty)) {
return 'boolean';
} else {
return 'string';
}
}
/**
@ -125,7 +172,7 @@ export class UserImportListComponent extends BaseImportListComponentDirective<Us
* Sends the data in the text field input area to the importer
*/
public parseTextArea(): void {
(this.importer as UserImportService).parseTextArea(this.textAreaForm.get('inputtext').value);
this.importer.parseTextArea(this.textAreaForm.get('inputtext').value);
}
/**

10
client/src/styles.scss
View File

@ -505,16 +505,6 @@ button.mat-menu-item.selected {
width: -webkit-fill-available;
}
/**
* Compilcated ngrid hack: The meta row won't disappear (just like that)
* Select the first ever container pbl-ngrid-container div and hide
*/
.pbl-ngrid-container {
> div {
height: 0;
}
}
.cdk-column-menu {
padding: 0 16px 0 0 !important;
}