Merge pull request #2783 from FinnStutzenstein/PapaParse

New csv import using PapaParse
2017-01-06 11:33:06 +01:00 · 2017-01-06 11:33:06 +01:00 · cc0c0bf0d4
commit cc0c0bf0d4
parent c12d985ab6 f5fa3575df
8 changed files with 181 additions and 93 deletions
--- a/1
+++ b/1
@ -51,6 +51,7 @@ Motions:
 - New PDF layout.
 - Added DOCX export with docxtemplater.
 - Changed label of former state "commited a bill" to "refered to committee".
+- New csv import layout and using Papa Parse for parsing the csv.

 Elections:
 - Added options to calculate percentages on different bases.
--- a/README.rst
+++ b/README.rst
@ -222,6 +222,7 @@ OpenSlides uses the following projects or parts of them:
  * `ngStorage <https://github.com/gsklee/ngStorage>`_, License: MIT
  * `ngbootbox <https://github.com/eriktufvesson/ngBootbox>`_, License: MIT
  * `open-sans-fontface <https://github.com/FontFaceKit/open-sans>`_, License: Apache License version 2.0
+  * `Papa Parse <http://papaparse.com/>`_, License: MIT
  * `pdfjs-dist <http://mozilla.github.io/pdf.js/>`_, License: Apache-2.0
  * `roboto-condensed <https://github.com/davidcunningham/roboto-condensed>`_, License: Apache 2.0

--- a/bower.json
+++ b/bower.json
@ -35,6 +35,7 @@
    "ng-file-upload": "~11.2.3",
    "ngstorage": "~0.3.11",
    "ngBootbox": "~0.1.3",
+    "papaparse": "~4.1.2",
    "pdfmake": "~0.1.23",
    "roboto-fontface": "~0.6.0"
  },
--- a/openslides/core/static/css/app.css
+++ b/openslides/core/static/css/app.css
@ -1453,22 +1453,31 @@ img {
    margin-right: .2em;
 }

-/* for angular-csv-import form */
+/* for csv import form */
 .import {
    margin-left: 15px;
+    width: 35%;
 }

-.import .label {
-    color: #333 !important;
-    font-size: 100%;
+.import .file-select input {
+    display: none;
+}
+
+.import .file-select label {
    font-weight: normal;
-    width: 100px;
-    text-align: left;
-    display: inline-block;
+    cursor: pointer;
 }

-.import .label::after {
-    content: ': ';
+.import .clear-file {
+    color: #555;
+}
+
+.import .help-block {
+    padding-bottom: 0;
+}
+
+.import .help-block-big {
+    font-size: 100%;
 }

 /* voting results */
--- a/openslides/core/static/js/core/site.js
+++ b/openslides/core/static/js/core/site.js
@ -661,6 +661,84 @@ angular.module('OpenSlidesApp.core.site', [
    }
 ])

+/* This directive provides a csv import template.
+ * Papa Parse is used to parse the csv file. Accepted attributes:
+ * * change:
+ *   Callback if file changes. The one parameter is csv passing the parsed file
+ * * config (optional):
+ *   - accept: String with extensions: default '.csv .txt'
+ *   - encodingOptions: List with encodings. Default ['UTF-8', 'ISO-8859-1']
+ *   - parseConfig: a dict passed to PapaParse
+ */
+.directive('csvImport', [
+    function () {
+        return {
+            restrict: 'E',
+            templateUrl: 'static/templates/csv-import.html',
+            scope: {
+                change: '&',
+                config: '=?',
+            },
+            controller: function ($scope, $element, $attrs, $location) {
+                // set config if it is not given
+                if (!$scope.config) {
+                    $scope.config = {};
+                }
+                if (!$scope.config.parseConfig) {
+                    $scope.config.parseConfig = {};
+                }
+
+                $scope.inputElement = angular.element($element[0].querySelector('#csvFileSelector'));
+
+                // set accept and encoding
+                $scope.accept = $scope.config.accept || '.csv';
+                $scope.encodingOptions = $scope.config.encodingOptions || ['UTF-8'];
+                $scope.encoding = $scope.encodingOptions[0];
+
+                $scope.parse = function () {
+                    var inputElement = $scope.inputElement[0];
+                    if (!inputElement.files.length) {
+                        $scope.change({csv: {data: {}}});
+                    } else {
+                        var parseConfig = _.defaults(_.clone($scope.config.parseConfig), {
+                            delimiter: $scope.delimiter,
+                            encoding: $scope.encoding,
+                            header: false, // we do not want to have dicts in result
+                            complete: function (csv) {
+                                csv.data = csv.data.splice(1); // do not interpret the header as data
+                                $scope.$apply(function () {
+                                    if (csv.meta.delimiter) {
+                                        $scope.autodelimiter = csv.meta.delimiter;
+                                    }
+                                    $scope.change({csv: csv});
+                                });
+                            },
+                            error: function () {
+                                $scope.$apply(function () {
+                                    $scope.change({csv: {data: {}}});
+                                });
+                            },
+                        });
+
+                        Papa.parse(inputElement.files[0], parseConfig);
+                    }
+                };
+
+                $scope.clearFile = function () {
+                    $scope.inputElement[0].value = '';
+                    $scope.selectedFile = undefined;
+                    $scope.parse();
+                };
+
+                $scope.inputElement.on('change', function () {
+                    $scope.selectedFile = _.last($scope.inputElement[0].value.split('\\'));
+                    $scope.parse();
+                });
+            },
+        };
+    }
+])
+
 .controller('MainMenuCtrl', [
    '$scope',
    'mainMenu',
--- a/openslides/core/static/templates/csv-import.html
+++ b/openslides/core/static/templates/csv-import.html
@ -0,0 +1,30 @@
+<form class="import">
+  <label for="fileInput" translate>CSV file</label>
+  <div id="fileInput" class="file-select">
+    <div class="form-control">
+      <label for="csvFileSelector">
+        <i class="fa fa-upload"></i>
+        {{ selectedFile || ('Select a file' | translate) }}
+      </label>
+      <a href class="pull-right clear-file" ng-if="selectedFile" ng-click="clearFile()">
+        <i class="fa fa-times" title="{{ 'Deselect file' | translate }}"></i>
+      </a>
+    </div>
+    <input id="csvFileSelector" type="file" value="" accept="{{ accept }}">
+    <p class="help-block">
+      <translate translate-context="special filetypes in a file open dialog">Accept</translate>: {{ accept }}
+    </p>
+  </div>
+
+  <div class="form-group">
+    <label for="selectEncoding" translate>Encoding</label>
+    <select class="form-control" ng-model="$parent.encoding" ng-if="encodingOptions.length > 1" ng-change="parse()"
+      id="selectEncoding" ng-options="enc for enc in encodingOptions"></select>
+  </div>
+  <div class="form-group">
+    <label for="inputDelimiter" translate>Separator</label>
+    <input type="text" class="form-control" ng-model="delimiter" ng-change="parse()" id="inputDelimiter">
+    <p class="help-block help-block-big" ng-if="autodelimiter"><translate>Autodetect</translate>:&nbsp;&nbsp;{{ autodelimiter }}</p>
+    <p class="help-block" ng-if="!autodelimiter" translate>Leave empty for autodetection of the separator.</p>
+  </div>
+</form>
--- a/openslides/motions/static/js/motions/site.js
+++ b/openslides/motions/static/js/motions/site.js
@ -1607,73 +1607,56 @@ angular.module('OpenSlidesApp.motions.site', [
    'Category',
    'Motion',
    'User',
-    function($scope, $q, gettext, Category, Motion, User) {
+    'gettextCatalog',
+    function($scope, $q, gettext, Category, Motion, User, gettextCatalog) {
        // set initial data for csv import
        $scope.motions = [];
-        $scope.separator = ',';
-        $scope.encoding = 'UTF-8';
-        $scope.encodingOptions = ['UTF-8', 'ISO-8859-1'];
-        $scope.accept = '.csv, .txt';
-        $scope.csv = {
-            content: null,
-            header: true,
-            headerVisible: false,
-            separator: $scope.separator,
-            separatorVisible: false,
-            encoding: $scope.encoding,
-            encodingVisible: false,
-            accept: $scope.accept,
-            result: null
+
+        // set csv 
+        $scope.csvConfig = {
+            accept: '.csv, .txt',
+            encodingOptions: ['UTF-8', 'ISO-8859-1'],
+            parseConfig: {
+                skipEmptyLines: true,
+            },
        };
-        // set csv file encoding
-        $scope.setEncoding = function () {
-            $scope.csv.encoding = $scope.encoding;
-        };
-        // set csv file encoding
-        $scope.setSeparator = function () {
-            $scope.csv.separator = $scope.separator;
-        };
-        // detect if csv file is loaded
-        $scope.$watch('csv.result', function () {
+
+        var FIELDS = ['identifier', 'title', 'text', 'reason', 'submitter', 'category', 'origin'];
+        $scope.motions = [];
+        $scope.onCsvChange = function (csv) {
            $scope.motions = [];
-            var quotionRe = /^"(.*)"$/;
-            angular.forEach($scope.csv.result, function (motion) {
-                if (motion.identifier) {
-                    motion.identifier = motion.identifier.replace(quotionRe, '$1');
-                    if (motion.identifier !== '') {
-                        // All motion objects are already loaded via the resolve statement from ui-router.
-                        var motions = Motion.getAll();
-                        if (_.find(motions, function (item) {
-                            return item.identifier == motion.identifier;
-                        })) {
-                            motion.importerror = true;
-                            motion.identifier_error = gettext('Error: Identifier already exists.');
-                        }
+            var motions = [];
+            _.forEach(csv.data, function (row) {
+                if (row.length >= 3) {
+                    var filledRow = _.zipObject(FIELDS, row);
+                    motions.push(filledRow);
+                }
+            });
+
+            _.forEach(motions, function (motion) {
+                // identifier
+                if (motion.identifier !== '') {
+                    // All motion objects are already loaded via the resolve statement from ui-router.
+                    var motions = Motion.getAll();
+                    if (_.find(motions, function (item) {
+                        return item.identifier === motion.identifier;
+                    })) {
+                        motion.importerror = true;
+                        motion.identifier_error = gettext('Error: Identifier already exists.');
                    }
                }
                // title
-                if (motion.title) {
-                    motion.title = motion.title.replace(quotionRe, '$1');
-                }
                if (!motion.title) {
                    motion.importerror = true;
                    motion.title_error = gettext('Error: Title is required.');
                }
                // text
-                if (motion.text) {
-                    motion.text = motion.text.replace(quotionRe, '$1');
-                }
                if (!motion.text) {
                    motion.importerror = true;
                    motion.text_error = gettext('Error: Text is required.');
                }
-                // reason
-                if (motion.reason) {
-                    motion.reason = motion.reason.replace(quotionRe, '$1');
-                }
                // submitter
                if (motion.submitter) {
-                    motion.submitter = motion.submitter.replace(quotionRe, '$1');
                    if (motion.submitter !== '') {
                        // All user objects are already loaded via the resolve statement from ui-router.
                        var users = User.getAll();
@ -1690,7 +1673,6 @@ angular.module('OpenSlidesApp.motions.site', [
                }
                // category
                if (motion.category) {
-                    motion.category = motion.category.replace(quotionRe, '$1');
                    if (motion.category !== '') {
                        // All categore objects are already loaded via the resolve statement from ui-router.
                        var categories = Category.getAll();
@ -1706,13 +1688,10 @@ angular.module('OpenSlidesApp.motions.site', [
                if (motion.category && motion.category !== '' && !motion.category_id) {
                    motion.category_create = gettext('New category will be created.');
                }
-                // origin
-                if (motion.origin) {
-                    motion.origin = motion.origin.replace(quotionRe, '$1');
-                }
+
                $scope.motions.push(motion);
            });
-        });
+        };

        // Counter for creations
        $scope.usersCreated = 0;
@ -1824,14 +1803,16 @@ angular.module('OpenSlidesApp.motions.site', [
            $scope.csvimported = true;
        };
        $scope.clear = function () {
-            $scope.csv.result = null;
+            $scope.motions = [];
        };
        // download CSV example file
        $scope.downloadCSVExample = function () {
+            var headerline = ['Identifier', 'Title', 'Text', 'Reason', 'Submitter', 'Category', 'Origin'];
+            headerline = _.map(headerline, function (entry) {
+                return gettextCatalog.getString(entry);
+            });
            var element = document.getElementById('downloadLink');
-            var csvRows = [
-                // column header line
-                ['identifier', 'title', 'text', 'reason', 'submitter', 'category', 'origin'],
+            var csvRows = [headerline,
                // example entries
                ['A1', 'Title 1', 'Text 1', 'Reason 1', 'Submitter A', 'Category A', 'Last Year Conference A'],
                ['B1', 'Title 2', 'Text 2', 'Reason 2', 'Submitter B', 'Category B', ''                      ],
--- a/openslides/motions/static/templates/motions/motion-import.html
+++ b/openslides/motions/static/templates/motions/motion-import.html
@ -12,40 +12,27 @@

 <div class="details">

-  <div class="block row">
-    <div class="title">
-      <h3 translate>Select a CSV file
-    </div>
-    <div class="block right import">
-      <label class="label" for="inputSeparator" translate>Separator</label>
-      <input type="text" ng-model="separator" ng-change="setSeparator()" ng-init="separator=separator" id="inputSeparator">
-      <br>
-      <label class="label" for="selectEncoding" translate>Encoding</label>
-      <select ng-model="encoding" ng-options="enc as enc for enc in encodingOptions"
-          ng-selected="setEncoding()" ng-init="encoding=encoding" id="selectEncoding"></select>
-      <ng-csv-import
-          content="csv.content"
-          header="csv.header"
-          header-visible="csv.headerVisible"
-          separator="csv.separator"
-          separator-visible="csv.separatorVisible"
-          result="csv.result"
-          encoding="csv.encoding"
-          accept="csv.accept"
-          encoding-visible="csv.encodingVisible"></ng-csv-import>
-    </div>
-  </div>
+  <h3 translate>Select a CSV file</h3>
+  <csv-import change="onCsvChange(csv)" config="csvConfig"></csv-import>

  <h4 translate>Please note:</h4>
  <ul class="indentation">
      <li><translate>Required comma or semicolon separated values with these column header names in the first row</translate>:<br>
-          <code>identifier, title, text, reason, submitter, category, origin</code>
+        <code>
+          <translate>Identifier</translate>,
+          <translate>Title</translate>,
+          <translate>Text</translate>,
+          <translate>Reason</translate>,
+          <translate>Submitter</translate>,
+          <translate>Category</translate>,
+          <translate>Origin</translate>
+        </code>
      <li translate>Identifier, reason, submitter, category and origin are optional and may be empty.
      <li translate>Only double quotes are accepted as text delimiter (no single quotes).
      <li><a id="downloadLink" href="" ng-click="downloadCSVExample()" translate>Download CSV example file</a>
  </ul>

-  <div ng-if="csv.result">
+  <div ng-if="motions.length">
    <h3 translate>Preview</h3>
    <table class="table table-striped table-bordered table-condensed">
      <thead>