ag-admin/OpenSlides
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

New csv import using PapaParse

Browse Source
This commit is contained in:
FinnStutzenstein 2016-12-12 16:50:37 +01:00
parent 85c70b27f5
commit f5fa3575df
8 changed files with 181 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG
View File

@ -51,6 +51,7 @@ Motions:
- New PDF layout.
- Added DOCX export with docxtemplater.
- Changed label of former state "commited a bill" to "refered to committee".
- New csv import layout and using Papa Parse for parsing the csv.
Elections:
- Added options to calculate percentages on different bases.

1
README.rst
View File

@ -222,6 +222,7 @@ OpenSlides uses the following projects or parts of them:
* `ngStorage <https://github.com/gsklee/ngStorage>`_, License: MIT
* `ngbootbox <https://github.com/eriktufvesson/ngBootbox>`_, License: MIT
* `open-sans-fontface <https://github.com/FontFaceKit/open-sans>`_, License: Apache License version 2.0
* `Papa Parse <http://papaparse.com/>`_, License: MIT
* `pdfjs-dist <http://mozilla.github.io/pdf.js/>`_, License: Apache-2.0
* `roboto-condensed <https://github.com/davidcunningham/roboto-condensed>`_, License: Apache 2.0

1
bower.json
View File

@ -35,6 +35,7 @@
"ng-file-upload": "~11.2.3",
"ngstorage": "~0.3.11",
"ngBootbox": "~0.1.3",
"papaparse": "~4.1.2",
"pdfmake": "~0.1.23",
"roboto-fontface": "~0.6.0"
},

27
openslides/core/static/css/app.css
View File

@ -1453,22 +1453,31 @@ img {
margin-right: .2em;
}
/* for angular-csv-import form */
/* for csv import form */
.import {
margin-left: 15px;
width: 35%;
}
.import .label {
color: #333 !important;
font-size: 100%;
.import .file-select input {
display: none;
}
.import .file-select label {
font-weight: normal;
width: 100px;
text-align: left;
display: inline-block;
cursor: pointer;
}
.import .label::after {
content: ': ';
.import .clear-file {
color: #555;
}
.import .help-block {
padding-bottom: 0;
}
.import .help-block-big {
font-size: 100%;
}
/* voting results */

78
openslides/core/static/js/core/site.js
View File

@ -661,6 +661,84 @@ angular.module('OpenSlidesApp.core.site', [
}
])
/* This directive provides a csv import template.
* Papa Parse is used to parse the csv file. Accepted attributes:
* * change:
* Callback if file changes. The one parameter is csv passing the parsed file
* * config (optional):
* - accept: String with extensions: default '.csv .txt'
* - encodingOptions: List with encodings. Default ['UTF-8', 'ISO-8859-1']
* - parseConfig: a dict passed to PapaParse
*/
.directive('csvImport', [
function () {
return {
restrict: 'E',
templateUrl: 'static/templates/csv-import.html',
scope: {
change: '&',
config: '=?',
},
controller: function ($scope, $element, $attrs, $location) {
// set config if it is not given
if (!$scope.config) {
$scope.config = {};
}
if (!$scope.config.parseConfig) {
$scope.config.parseConfig = {};
}
$scope.inputElement = angular.element($element[0].querySelector('#csvFileSelector'));
// set accept and encoding
$scope.accept = $scope.config.accept || '.csv';
$scope.encodingOptions = $scope.config.encodingOptions || ['UTF-8'];
$scope.encoding = $scope.encodingOptions[0];
$scope.parse = function () {
var inputElement = $scope.inputElement[0];
if (!inputElement.files.length) {
$scope.change({csv: {data: {}}});
} else {
var parseConfig = _.defaults(_.clone($scope.config.parseConfig), {
delimiter: $scope.delimiter,
encoding: $scope.encoding,
header: false, // we do not want to have dicts in result
complete: function (csv) {
csv.data = csv.data.splice(1); // do not interpret the header as data
$scope.$apply(function () {
if (csv.meta.delimiter) {
$scope.autodelimiter = csv.meta.delimiter;
}
$scope.change({csv: csv});
});
},
error: function () {
$scope.$apply(function () {
$scope.change({csv: {data: {}}});
});
},
});
Papa.parse(inputElement.files[0], parseConfig);
}
};
$scope.clearFile = function () {
$scope.inputElement[0].value = '';
$scope.selectedFile = undefined;
$scope.parse();
};
$scope.inputElement.on('change', function () {
$scope.selectedFile = _.last($scope.inputElement[0].value.split('\\'));
$scope.parse();
});
},
};
}
])
.controller('MainMenuCtrl', [
'$scope',
'mainMenu',

30
openslides/core/static/templates/csv-import.html Normal file
View File

@ -0,0 +1,30 @@
<form class="import">
<label for="fileInput" translate>CSV file</label>
<div id="fileInput" class="file-select">
<div class="form-control">
<label for="csvFileSelector">
<i class="fa fa-upload"></i>
{{ selectedFile || ('Select a file' | translate) }}
</label>
<a href class="pull-right clear-file" ng-if="selectedFile" ng-click="clearFile()">
<i class="fa fa-times" title="{{ 'Deselect file' | translate }}"></i>
</a>
</div>
<input id="csvFileSelector" type="file" value="" accept="{{ accept }}">
<p class="help-block">
<translate translate-context="special filetypes in a file open dialog">Accept</translate>: {{ accept }}
</p>
</div>
<div class="form-group">
<label for="selectEncoding" translate>Encoding</label>
<select class="form-control" ng-model="$parent.encoding" ng-if="encodingOptions.length > 1" ng-change="parse()"
id="selectEncoding" ng-options="enc for enc in encodingOptions"></select>
</div>
<div class="form-group">
<label for="inputDelimiter" translate>Separator</label>
<input type="text" class="form-control" ng-model="delimiter" ng-change="parse()" id="inputDelimiter">
<p class="help-block help-block-big" ng-if="autodelimiter"><translate>Autodetect</translate>:&nbsp;&nbsp;{{ autodelimiter }}</p>
<p class="help-block" ng-if="!autodelimiter" translate>Leave empty for autodetection of the separator.</p>
</div>
</form>

99
openslides/motions/static/js/motions/site.js
View File

@ -1607,73 +1607,56 @@ angular.module('OpenSlidesApp.motions.site', [
'Category',
'Motion',
'User',
function($scope, $q, gettext, Category, Motion, User) {
'gettextCatalog',
function($scope, $q, gettext, Category, Motion, User, gettextCatalog) {
// set initial data for csv import
$scope.motions = [];
$scope.separator = ',';
$scope.encoding = 'UTF-8';
$scope.encodingOptions = ['UTF-8', 'ISO-8859-1'];
$scope.accept = '.csv, .txt';
$scope.csv = {
content: null,
header: true,
headerVisible: false,
separator: $scope.separator,
separatorVisible: false,
encoding: $scope.encoding,
encodingVisible: false,
accept: $scope.accept,
result: null
// set csv
$scope.csvConfig = {
accept: '.csv, .txt',
encodingOptions: ['UTF-8', 'ISO-8859-1'],
parseConfig: {
skipEmptyLines: true,
},
};
// set csv file encoding
$scope.setEncoding = function () {
$scope.csv.encoding = $scope.encoding;
};
// set csv file encoding
$scope.setSeparator = function () {
$scope.csv.separator = $scope.separator;
};
// detect if csv file is loaded
$scope.$watch('csv.result', function () {
var FIELDS = ['identifier', 'title', 'text', 'reason', 'submitter', 'category', 'origin'];
$scope.motions = [];
$scope.onCsvChange = function (csv) {
$scope.motions = [];
var quotionRe = /^"(.*)"$/;
angular.forEach($scope.csv.result, function (motion) {
if (motion.identifier) {
motion.identifier = motion.identifier.replace(quotionRe, '$1');
if (motion.identifier !== '') {
// All motion objects are already loaded via the resolve statement from ui-router.
var motions = Motion.getAll();
if (_.find(motions, function (item) {
return item.identifier == motion.identifier;
})) {
motion.importerror = true;
motion.identifier_error = gettext('Error: Identifier already exists.');
}
var motions = [];
_.forEach(csv.data, function (row) {
if (row.length >= 3) {
var filledRow = _.zipObject(FIELDS, row);
motions.push(filledRow);
}
});
_.forEach(motions, function (motion) {
// identifier
if (motion.identifier !== '') {
// All motion objects are already loaded via the resolve statement from ui-router.
var motions = Motion.getAll();
if (_.find(motions, function (item) {
return item.identifier === motion.identifier;
})) {
motion.importerror = true;
motion.identifier_error = gettext('Error: Identifier already exists.');
}
}
// title
if (motion.title) {
motion.title = motion.title.replace(quotionRe, '$1');
}
if (!motion.title) {
motion.importerror = true;
motion.title_error = gettext('Error: Title is required.');
}
// text
if (motion.text) {
motion.text = motion.text.replace(quotionRe, '$1');
}
if (!motion.text) {
motion.importerror = true;
motion.text_error = gettext('Error: Text is required.');
}
// reason
if (motion.reason) {
motion.reason = motion.reason.replace(quotionRe, '$1');
}
// submitter
if (motion.submitter) {
motion.submitter = motion.submitter.replace(quotionRe, '$1');
if (motion.submitter !== '') {
// All user objects are already loaded via the resolve statement from ui-router.
var users = User.getAll();
@ -1690,7 +1673,6 @@ angular.module('OpenSlidesApp.motions.site', [
}
// category
if (motion.category) {
motion.category = motion.category.replace(quotionRe, '$1');
if (motion.category !== '') {
// All categore objects are already loaded via the resolve statement from ui-router.
var categories = Category.getAll();
@ -1706,13 +1688,10 @@ angular.module('OpenSlidesApp.motions.site', [
if (motion.category && motion.category !== '' && !motion.category_id) {
motion.category_create = gettext('New category will be created.');
}
// origin
if (motion.origin) {
motion.origin = motion.origin.replace(quotionRe, '$1');
}
$scope.motions.push(motion);
});
});
};
// Counter for creations
$scope.usersCreated = 0;
@ -1824,14 +1803,16 @@ angular.module('OpenSlidesApp.motions.site', [
$scope.csvimported = true;
};
$scope.clear = function () {
$scope.csv.result = null;
$scope.motions = [];
};
// download CSV example file
$scope.downloadCSVExample = function () {
var headerline = ['Identifier', 'Title', 'Text', 'Reason', 'Submitter', 'Category', 'Origin'];
headerline = _.map(headerline, function (entry) {
return gettextCatalog.getString(entry);
});
var element = document.getElementById('downloadLink');
var csvRows = [
// column header line
['identifier', 'title', 'text', 'reason', 'submitter', 'category', 'origin'],
var csvRows = [headerline,
// example entries
['A1', 'Title 1', 'Text 1', 'Reason 1', 'Submitter A', 'Category A', 'Last Year Conference A'],
['B1', 'Title 2', 'Text 2', 'Reason 2', 'Submitter B', 'Category B', '' ],

37
openslides/motions/static/templates/motions/motion-import.html
View File

@ -12,40 +12,27 @@
<div class="details">
<div class="block row">
<div class="title">
<h3 translate>Select a CSV file
</div>
<div class="block right import">
<label class="label" for="inputSeparator" translate>Separator</label>
<input type="text" ng-model="separator" ng-change="setSeparator()" ng-init="separator=separator" id="inputSeparator">
<br>
<label class="label" for="selectEncoding" translate>Encoding</label>
<select ng-model="encoding" ng-options="enc as enc for enc in encodingOptions"
ng-selected="setEncoding()" ng-init="encoding=encoding" id="selectEncoding"></select>
<ng-csv-import
content="csv.content"
header="csv.header"
header-visible="csv.headerVisible"
separator="csv.separator"
separator-visible="csv.separatorVisible"
result="csv.result"
encoding="csv.encoding"
accept="csv.accept"
encoding-visible="csv.encodingVisible"></ng-csv-import>
</div>
</div>
<h3 translate>Select a CSV file</h3>
<csv-import change="onCsvChange(csv)" config="csvConfig"></csv-import>
<h4 translate>Please note:</h4>
<ul class="indentation">
<li><translate>Required comma or semicolon separated values with these column header names in the first row</translate>:<br>
<code>identifier, title, text, reason, submitter, category, origin</code>
<code>
<translate>Identifier</translate>,
<translate>Title</translate>,
<translate>Text</translate>,
<translate>Reason</translate>,
<translate>Submitter</translate>,
<translate>Category</translate>,
<translate>Origin</translate>
</code>
<li translate>Identifier, reason, submitter, category and origin are optional and may be empty.
<li translate>Only double quotes are accepted as text delimiter (no single quotes).
<li><a id="downloadLink" href="" ng-click="downloadCSVExample()" translate>Download CSV example file</a>
</ul>
<div ng-if="csv.result">
<div ng-if="motions.length">
<h3 translate>Preview</h3>
<table class="table table-striped table-bordered table-condensed">
<thead>