From f5fa3575dfc341f7c8767de6270a850e5e2d5cf5 Mon Sep 17 00:00:00 2001 From: FinnStutzenstein Date: Mon, 12 Dec 2016 16:50:37 +0100 Subject: [PATCH] New csv import using PapaParse --- CHANGELOG | 1 + README.rst | 1 + bower.json | 1 + openslides/core/static/css/app.css | 27 +++-- openslides/core/static/js/core/site.js | 78 +++++++++++++++ .../core/static/templates/csv-import.html | 30 ++++++ openslides/motions/static/js/motions/site.js | 99 ++++++++----------- .../templates/motions/motion-import.html | 37 +++---- 8 files changed, 181 insertions(+), 93 deletions(-) create mode 100644 openslides/core/static/templates/csv-import.html diff --git a/CHANGELOG b/CHANGELOG index fcdf808cd..87e855ac6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -51,6 +51,7 @@ Motions: - New PDF layout. - Added DOCX export with docxtemplater. - Changed label of former state "commited a bill" to "refered to committee". +- New csv import layout and using Papa Parse for parsing the csv. Elections: - Added options to calculate percentages on different bases. diff --git a/README.rst b/README.rst index a2faf9640..77450342a 100644 --- a/README.rst +++ b/README.rst @@ -222,6 +222,7 @@ OpenSlides uses the following projects or parts of them: * `ngStorage `_, License: MIT * `ngbootbox `_, License: MIT * `open-sans-fontface `_, License: Apache License version 2.0 + * `Papa Parse `_, License: MIT * `pdfjs-dist `_, License: Apache-2.0 * `roboto-condensed `_, License: Apache 2.0 diff --git a/bower.json b/bower.json index b6672bc8c..afc333f8b 100644 --- a/bower.json +++ b/bower.json @@ -35,6 +35,7 @@ "ng-file-upload": "~11.2.3", "ngstorage": "~0.3.11", "ngBootbox": "~0.1.3", + "papaparse": "~4.1.2", "pdfmake": "~0.1.23", "roboto-fontface": "~0.6.0" }, diff --git a/openslides/core/static/css/app.css b/openslides/core/static/css/app.css index 1ed6004c3..524058188 100644 --- a/openslides/core/static/css/app.css +++ b/openslides/core/static/css/app.css @@ -1453,22 +1453,31 @@ img { margin-right: .2em; } -/* for angular-csv-import form */ +/* for csv import form */ .import { margin-left: 15px; + width: 35%; } -.import .label { - color: #333 !important; - font-size: 100%; +.import .file-select input { + display: none; +} + +.import .file-select label { font-weight: normal; - width: 100px; - text-align: left; - display: inline-block; + cursor: pointer; } -.import .label::after { - content: ': '; +.import .clear-file { + color: #555; +} + +.import .help-block { + padding-bottom: 0; +} + +.import .help-block-big { + font-size: 100%; } /* voting results */ diff --git a/openslides/core/static/js/core/site.js b/openslides/core/static/js/core/site.js index 174ec4e75..645b289c7 100644 --- a/openslides/core/static/js/core/site.js +++ b/openslides/core/static/js/core/site.js @@ -661,6 +661,84 @@ angular.module('OpenSlidesApp.core.site', [ } ]) +/* This directive provides a csv import template. + * Papa Parse is used to parse the csv file. Accepted attributes: + * * change: + * Callback if file changes. The one parameter is csv passing the parsed file + * * config (optional): + * - accept: String with extensions: default '.csv .txt' + * - encodingOptions: List with encodings. Default ['UTF-8', 'ISO-8859-1'] + * - parseConfig: a dict passed to PapaParse + */ +.directive('csvImport', [ + function () { + return { + restrict: 'E', + templateUrl: 'static/templates/csv-import.html', + scope: { + change: '&', + config: '=?', + }, + controller: function ($scope, $element, $attrs, $location) { + // set config if it is not given + if (!$scope.config) { + $scope.config = {}; + } + if (!$scope.config.parseConfig) { + $scope.config.parseConfig = {}; + } + + $scope.inputElement = angular.element($element[0].querySelector('#csvFileSelector')); + + // set accept and encoding + $scope.accept = $scope.config.accept || '.csv'; + $scope.encodingOptions = $scope.config.encodingOptions || ['UTF-8']; + $scope.encoding = $scope.encodingOptions[0]; + + $scope.parse = function () { + var inputElement = $scope.inputElement[0]; + if (!inputElement.files.length) { + $scope.change({csv: {data: {}}}); + } else { + var parseConfig = _.defaults(_.clone($scope.config.parseConfig), { + delimiter: $scope.delimiter, + encoding: $scope.encoding, + header: false, // we do not want to have dicts in result + complete: function (csv) { + csv.data = csv.data.splice(1); // do not interpret the header as data + $scope.$apply(function () { + if (csv.meta.delimiter) { + $scope.autodelimiter = csv.meta.delimiter; + } + $scope.change({csv: csv}); + }); + }, + error: function () { + $scope.$apply(function () { + $scope.change({csv: {data: {}}}); + }); + }, + }); + + Papa.parse(inputElement.files[0], parseConfig); + } + }; + + $scope.clearFile = function () { + $scope.inputElement[0].value = ''; + $scope.selectedFile = undefined; + $scope.parse(); + }; + + $scope.inputElement.on('change', function () { + $scope.selectedFile = _.last($scope.inputElement[0].value.split('\\')); + $scope.parse(); + }); + }, + }; + } +]) + .controller('MainMenuCtrl', [ '$scope', 'mainMenu', diff --git a/openslides/core/static/templates/csv-import.html b/openslides/core/static/templates/csv-import.html new file mode 100644 index 000000000..2bb0f1747 --- /dev/null +++ b/openslides/core/static/templates/csv-import.html @@ -0,0 +1,30 @@ +
+ +
+
+ + + + +
+ +

+ Accept: {{ accept }} +

+
+ +
+ + +
+
+ + +

Autodetect:  {{ autodelimiter }}

+

Leave empty for autodetection of the separator.

+
+
diff --git a/openslides/motions/static/js/motions/site.js b/openslides/motions/static/js/motions/site.js index 18908441e..ef9342a00 100644 --- a/openslides/motions/static/js/motions/site.js +++ b/openslides/motions/static/js/motions/site.js @@ -1607,73 +1607,56 @@ angular.module('OpenSlidesApp.motions.site', [ 'Category', 'Motion', 'User', - function($scope, $q, gettext, Category, Motion, User) { + 'gettextCatalog', + function($scope, $q, gettext, Category, Motion, User, gettextCatalog) { // set initial data for csv import $scope.motions = []; - $scope.separator = ','; - $scope.encoding = 'UTF-8'; - $scope.encodingOptions = ['UTF-8', 'ISO-8859-1']; - $scope.accept = '.csv, .txt'; - $scope.csv = { - content: null, - header: true, - headerVisible: false, - separator: $scope.separator, - separatorVisible: false, - encoding: $scope.encoding, - encodingVisible: false, - accept: $scope.accept, - result: null + + // set csv + $scope.csvConfig = { + accept: '.csv, .txt', + encodingOptions: ['UTF-8', 'ISO-8859-1'], + parseConfig: { + skipEmptyLines: true, + }, }; - // set csv file encoding - $scope.setEncoding = function () { - $scope.csv.encoding = $scope.encoding; - }; - // set csv file encoding - $scope.setSeparator = function () { - $scope.csv.separator = $scope.separator; - }; - // detect if csv file is loaded - $scope.$watch('csv.result', function () { + + var FIELDS = ['identifier', 'title', 'text', 'reason', 'submitter', 'category', 'origin']; + $scope.motions = []; + $scope.onCsvChange = function (csv) { $scope.motions = []; - var quotionRe = /^"(.*)"$/; - angular.forEach($scope.csv.result, function (motion) { - if (motion.identifier) { - motion.identifier = motion.identifier.replace(quotionRe, '$1'); - if (motion.identifier !== '') { - // All motion objects are already loaded via the resolve statement from ui-router. - var motions = Motion.getAll(); - if (_.find(motions, function (item) { - return item.identifier == motion.identifier; - })) { - motion.importerror = true; - motion.identifier_error = gettext('Error: Identifier already exists.'); - } + var motions = []; + _.forEach(csv.data, function (row) { + if (row.length >= 3) { + var filledRow = _.zipObject(FIELDS, row); + motions.push(filledRow); + } + }); + + _.forEach(motions, function (motion) { + // identifier + if (motion.identifier !== '') { + // All motion objects are already loaded via the resolve statement from ui-router. + var motions = Motion.getAll(); + if (_.find(motions, function (item) { + return item.identifier === motion.identifier; + })) { + motion.importerror = true; + motion.identifier_error = gettext('Error: Identifier already exists.'); } } // title - if (motion.title) { - motion.title = motion.title.replace(quotionRe, '$1'); - } if (!motion.title) { motion.importerror = true; motion.title_error = gettext('Error: Title is required.'); } // text - if (motion.text) { - motion.text = motion.text.replace(quotionRe, '$1'); - } if (!motion.text) { motion.importerror = true; motion.text_error = gettext('Error: Text is required.'); } - // reason - if (motion.reason) { - motion.reason = motion.reason.replace(quotionRe, '$1'); - } // submitter if (motion.submitter) { - motion.submitter = motion.submitter.replace(quotionRe, '$1'); if (motion.submitter !== '') { // All user objects are already loaded via the resolve statement from ui-router. var users = User.getAll(); @@ -1690,7 +1673,6 @@ angular.module('OpenSlidesApp.motions.site', [ } // category if (motion.category) { - motion.category = motion.category.replace(quotionRe, '$1'); if (motion.category !== '') { // All categore objects are already loaded via the resolve statement from ui-router. var categories = Category.getAll(); @@ -1706,13 +1688,10 @@ angular.module('OpenSlidesApp.motions.site', [ if (motion.category && motion.category !== '' && !motion.category_id) { motion.category_create = gettext('New category will be created.'); } - // origin - if (motion.origin) { - motion.origin = motion.origin.replace(quotionRe, '$1'); - } + $scope.motions.push(motion); }); - }); + }; // Counter for creations $scope.usersCreated = 0; @@ -1824,14 +1803,16 @@ angular.module('OpenSlidesApp.motions.site', [ $scope.csvimported = true; }; $scope.clear = function () { - $scope.csv.result = null; + $scope.motions = []; }; // download CSV example file $scope.downloadCSVExample = function () { + var headerline = ['Identifier', 'Title', 'Text', 'Reason', 'Submitter', 'Category', 'Origin']; + headerline = _.map(headerline, function (entry) { + return gettextCatalog.getString(entry); + }); var element = document.getElementById('downloadLink'); - var csvRows = [ - // column header line - ['identifier', 'title', 'text', 'reason', 'submitter', 'category', 'origin'], + var csvRows = [headerline, // example entries ['A1', 'Title 1', 'Text 1', 'Reason 1', 'Submitter A', 'Category A', 'Last Year Conference A'], ['B1', 'Title 2', 'Text 2', 'Reason 2', 'Submitter B', 'Category B', '' ], diff --git a/openslides/motions/static/templates/motions/motion-import.html b/openslides/motions/static/templates/motions/motion-import.html index b12263873..a6a646acd 100644 --- a/openslides/motions/static/templates/motions/motion-import.html +++ b/openslides/motions/static/templates/motions/motion-import.html @@ -12,40 +12,27 @@
-
-
-

Select a CSV file -

-
- - -
- - - -
-
+

Select a CSV file

+

Please note:

  • Required comma or semicolon separated values with these column header names in the first row:
    - identifier, title, text, reason, submitter, category, origin + + Identifier, + Title, + Text, + Reason, + Submitter, + Category, + Origin +
  • Identifier, reason, submitter, category and origin are optional and may be empty.
  • Only double quotes are accepted as text delimiter (no single quotes).
  • Download CSV example file
-
+

Preview