Merge pull request #2917 from FinnStutzenstein/Worker

Use workers for pdf generation

CHANGELOG

@@ -18,6 +18,7 @@ Core:
 - Added control for the resolution of the projectors.
 - Set the projector language in the settings.
 - Used pdfMake for clientside generation of PDFs.
+  Run pdf creation in background (in a web worker thread).
 - Introduced new table design for list views with serveral filters and
   CSV export.
 - Used session cookies and store filter settings in session storage.

bower.json

@@ -39,10 +39,7 @@
   },
   "overrides": {
     "pdfmake": {
-      "main": [
+      "main": []
-          "build/pdfmake.js",
-          "build/vfs_fonts.js"
-      ]
     },
     "pdfjs-dist": {
       "main": "build/pdf.combined.js"

gulpfile.js

@@ -41,10 +41,13 @@ var output_directory = path.join('openslides', 'static');
  * Default tasks to be run before start.
  */
 
-// Catches all JavaScript files from all core apps and concats them to one
+// Catches all JavaScript files (excluded worker files) from all core apps and concats them to one
 // file js/openslides.js. In production mode the file is uglified.
 gulp.task('js', function () {
-    return gulp.src(path.join('openslides', '*', 'static', 'js', '**', '*.js'))
+    return gulp.src([
+            path.join('openslides', '*', 'static', 'js', '**', '*.js'),
+            '!' + path.join('openslides', 'core', 'static', 'js', 'core', 'pdf-worker.js'),
+        ])
         .pipe(sourcemaps.init())
         .pipe(concat('openslides.js'))
         .pipe(sourcemaps.write())
@@ -68,6 +71,24 @@ gulp.task('js-libs', function () {
         .pipe(gulp.dest(path.join(output_directory, 'js')));
 });
 
+// Catches all pdfmake files for pdf worker.
+gulp.task('pdf-worker', function () {
+    return gulp.src([
+            path.join('openslides', 'core', 'static', 'js', 'core', 'pdf-worker.js'),
+        ])
+        .pipe(gulpif(argv.production, uglify()))
+        .pipe(gulp.dest(path.join(output_directory, 'js', 'workers')));
+});
+// pdfmake files
+gulp.task('pdf-worker-libs', function () {
+    return gulp.src([
+            path.join('bower_components', 'pdfmake', 'build', 'pdfmake.min.js'),
+            path.join('bower_components', 'pdfmake', 'build', 'vfs_fonts.js'),
+        ])
+        .pipe(concat('pdf-worker-libs.js'))
+        .pipe(gulp.dest(path.join(output_directory, 'js', 'workers')));
+});
+
 // Catches all template files from all core apps and concats them to one
 // file js/openslides-templates.js. In production mode the file is uglified.
 gulp.task('templates', function () {
@@ -191,6 +212,8 @@ gulp.task('translations', function () {
 gulp.task('default', [
         'js',
         'js-libs',
+        'pdf-worker',
+        'pdf-worker-libs',
         'templates',
         'css-libs',
         'fonts-libs',

openslides/agenda/static/js/agenda/site.js

@@ -82,9 +82,10 @@ angular.module('OpenSlidesApp.agenda.site', [
     'gettext',
     'osTableFilter',
     'AgendaCsvExport',
+    'PdfCreate',
     function($scope, $filter, $http, $state, DS, operator, ngDialog, Agenda, TopicForm, AgendaTree, Projector,
         ProjectionDefault, AgendaContentProvider, PdfMakeDocumentProvider, gettextCatalog, gettext, osTableFilter,
-        AgendaCsvExport) {
+        AgendaCsvExport, PdfCreate) {
         // Bind agenda tree to the scope
         $scope.$watch(function () {
             return Agenda.lastModified();
@@ -266,7 +267,7 @@ angular.module('OpenSlidesApp.agenda.site', [
             var filename = gettextCatalog.getString('Agenda') + '.pdf';
             var agendaContentProvider = AgendaContentProvider.createInstance($scope.itemsFiltered);
             var documentProvider = PdfMakeDocumentProvider.createInstance(agendaContentProvider);
-            pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+            PdfCreate.download(documentProvider.getDocument(), filename);
         };
         $scope.csvExport = function () {
             var element = document.getElementById('downloadLinkCSV');

openslides/assignments/static/js/assignments/site.js

@@ -264,9 +264,10 @@ angular.module('OpenSlidesApp.assignments.site', [
     'osTableSort',
     'gettext',
     'phases',
+    'PdfCreate',
     function($scope, ngDialog, AssignmentForm, Assignment, Tag, Agenda, Projector, ProjectionDefault,
         gettextCatalog, AssignmentContentProvider, AssignmentCatalogContentProvider, PdfMakeDocumentProvider,
-        User, osTableFilter, osTableSort, gettext, phases) {
+        User, osTableFilter, osTableSort, gettext, phases, PdfCreate) {
         Assignment.bindAll({}, $scope, 'assignments');
         Tag.bindAll({}, $scope, 'tags');
         $scope.$watch(function () {
@@ -386,7 +387,7 @@ angular.module('OpenSlidesApp.assignments.site', [
                 AssignmentCatalogContentProvider.createInstance(assignmentContentProviderArray);
             var documentProvider =
                 PdfMakeDocumentProvider.createInstance(assignmentCatalogContentProvider);
-            pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+            PdfCreate.download(documentProvider.getDocument(), filename);
         };
     }
 ])
@@ -411,9 +412,10 @@ angular.module('OpenSlidesApp.assignments.site', [
     'PdfMakeDocumentProvider',
     'PdfMakeBallotPaperProvider',
     'gettextCatalog',
+    'PdfCreate',
     function($scope, $http, $filter, filterFilter, gettext, ngDialog, AssignmentForm, operator, Assignment,
         User, assignmentId, phases, Projector, ProjectionDefault, AssignmentContentProvider, BallotContentProvider,
-        PdfMakeDocumentProvider, PdfMakeBallotPaperProvider, gettextCatalog) {
+        PdfMakeDocumentProvider, PdfMakeBallotPaperProvider, gettextCatalog, PdfCreate) {
         var assignment = Assignment.get(assignmentId);
         User.bindAll({}, $scope, 'users');
         Assignment.loadRelations(assignment, 'agenda_item');
@@ -593,7 +595,7 @@ angular.module('OpenSlidesApp.assignments.site', [
             var filename = gettextCatalog.getString("Election") + "_" + $scope.assignment.title + ".pdf";
             var assignmentContentProvider = AssignmentContentProvider.createInstance(assignment);
             var documentProvider = PdfMakeDocumentProvider.createInstance(assignmentContentProvider);
-            pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+            PdfCreate.download(documentProvider.getDocument(), filename);
         };
 
         //creates the ballotpaper as pdf
@@ -609,7 +611,7 @@ angular.module('OpenSlidesApp.assignments.site', [
             var filename = gettextCatalog.getString("Ballot") + "_" + pollNumber + "_" + $scope.assignment.title + ".pdf";
             var ballotContentProvider = BallotContentProvider.createInstance($scope, thePoll, pollNumber);
             var documentProvider = PdfMakeBallotPaperProvider.createInstance(ballotContentProvider);
-            pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+            PdfCreate.download(documentProvider.getDocument(), filename);
         };
 
         // Just mark some vote value strings for translation.

openslides/core/static/css/app.css

@@ -1133,6 +1133,39 @@ img {
     font-size: 90%;
 }
 
+/** Pdf creation status bar **/
+#pdf-status {
+    position: fixed;
+    bottom: 0;
+    width: 100%;
+    z-index: 100;
+}
+#pdf-status-container {
+    margin: 0 auto 0 auto;
+    padding: 0px 20px;
+    max-width: 1400px;
+}
+#pdf-status-container > div {
+    margin-bottom: 10px;
+    padding: 10px 20px;
+    border-radius: 6px;
+    box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
+}
+#pdf-status .generating {
+    color: #222;
+    background-color: #bed4de;
+    border-color: #46b8da;
+}
+#pdf-status .error {
+    color: #a94442;
+    background-color: #f2dede;
+    border-color: #ebccd1;
+}
+#pdf-status .finished {
+    color: #3c763d;
+    background-color: #dff0d8;
+    border-color: #d6e9c6;
+}
 
 /** General helper classes **/
 

openslides/core/static/js/core/pdf-worker.js

@@ -0,0 +1,37 @@
+/* Worker for creating PDFs in a separate thread. The creation of larger PDFs
+ * needs (currently) a lot of time and we don't want to block the UI.
+ */
+
+// Setup fake environment for pdfMake
+var document = {
+    'createElementNS': function () {
+        return {};
+    },
+};
+var window = this;
+
+// PdfMake and Fonts
+importScripts('/static/js/workers/pdf-worker-libs.js');
+
+// Set default font family.
+// To use custom ttf font files you have to replace the vfs_fonts.js file.
+// See https://github.com/pdfmake/pdfmake/wiki/Custom-Fonts---client-side
+// "PdfFont" is used as generic name in core/pdf.js. Adjust the four
+// font style names only.
+pdfMake.fonts = {
+    PdfFont: {
+        normal: 'Roboto-Regular.ttf',
+        bold: 'Roboto-Medium.ttf',
+        italics: 'Roboto-Italic.ttf',
+        bolditalics: 'Roboto-Italic.ttf'
+    }
+};
+
+// Create PDF on message and return the base64 decoded document
+self.addEventListener('message', function(e) {
+    var data = JSON.parse(e.data);
+    var pdf = pdfMake.createPdf(data);
+    pdf.getBase64(function (base64) {
+        self.postMessage(base64);
+    });
+}, false);

openslides/core/static/js/core/pdf.js

@@ -15,22 +15,6 @@ angular.module('OpenSlidesApp.core.pdf', [])
             size: 8
         };
 
-        // Set and return default font family.
-        //
-        // To use custom ttf font files you have to replace the vfs_fonts.js file.
-        // See https://github.com/pdfmake/pdfmake/wiki/Custom-Fonts---client-side
-        PDFLayout.getFontName = function() {
-            pdfMake.fonts = {
-                Roboto: {
-                    normal: 'Roboto-Regular.ttf',
-                    bold: 'Roboto-Medium.ttf',
-                    italics: 'Roboto-Italic.ttf',
-                    bolditalics: 'Roboto-Italic.ttf'
-                }
-            };
-            return "Roboto";
-        };
-
         // page title
         PDFLayout.createTitle = function(title) {
             return {
@@ -229,7 +213,7 @@ angular.module('OpenSlidesApp.core.pdf', [])
                     pageSize: 'A4',
                     pageMargins: [80, 90, 80, 60],
                     defaultStyle: {
-                        font: PDFLayout.getFontName(),
+                        font: 'PdfFont',
                         fontSize: 10
                     },
                     header: header,
@@ -353,7 +337,7 @@ angular.module('OpenSlidesApp.core.pdf', [])
                     pageSize: 'A4',
                     pageMargins: [0, 0, 0, 0],
                     defaultStyle: {
-                        font: PDFLayout.getFontName(),
+                        font: 'PdfFont',
                         fontSize: 10
                     },
                     content: content,
@@ -387,9 +371,8 @@ angular.module('OpenSlidesApp.core.pdf', [])
          * Converter component for HTML->JSON for pdfMake
          * @constructor
          * @param {object} images   - Key-Value structure representing image.src/BASE64 of images
-         * @param {object} pdfMake  - the converter component enhances pdfMake
          */
-        var createInstance = function(images, pdfMake) {
+        var createInstance = function(images) {
             var slice = Function.prototype.call.bind([].slice),
                 map = Function.prototype.call.bind([].map),
 
@@ -820,6 +803,82 @@ angular.module('OpenSlidesApp.core.pdf', [])
         return {
             createInstance: createInstance
         };
-}]);
+}])
+
+.factory('PdfCreate', [
+    '$timeout',
+    'FileSaver',
+    function ($timeout, FileSaver) {
+        var stateChangeCallbacks = [];
+        var b64toBlob = function(b64Data) {
+            var byteCharacters = atob(b64Data);
+            var byteNumbers = new Array(byteCharacters.length);
+            for (var i = 0; i < byteCharacters.length; i++) {
+                byteNumbers[i] = byteCharacters.charCodeAt(i);
+            }
+            var byteArray = new Uint8Array(byteNumbers);
+            var blob = new Blob([byteArray]);
+            return blob;
+        };
+        var stateChange = function (state, filename, error) {
+            _.forEach(stateChangeCallbacks, function (cb) {
+                $timeout(function () {cb(state, filename, error);}, 1);
+            });
+        };
+        return {
+            download: function (pdfDocument, filename) {
+                stateChange('generating', filename);
+                var pdfWorker = new Worker('/static/js/workers/pdf-worker.js');
+
+                pdfWorker.addEventListener('message', function (event) {
+                    var blob = b64toBlob(event.data);
+                    stateChange('finished', filename);
+                    FileSaver.saveAs(blob, filename);
+                });
+                pdfWorker.addEventListener('error', function (event) {
+                    stateChange('error', filename, event.message);
+                });
+                pdfWorker.postMessage(JSON.stringify(pdfDocument));
+            },
+            registerStateChangeCallback: function (cb) {
+                if (cb && typeof cb === 'function') {
+                    stateChangeCallbacks.push(cb);
+                }
+            },
+        };
+    }
+])
+
+.directive('pdfGenerationStatus', [
+    '$timeout',
+    'PdfCreate',
+    function($timeout, PdfCreate) {
+        return {
+            restrict: 'E',
+            templateUrl: 'static/templates/pdf-status.html',
+            scope: {},
+            controller: function ($scope, $element, $attrs, $location) {
+                $scope.pdfs = {};
+
+                var createStateChange = function (state, filename, error) {
+                    $scope.pdfs[filename] = {
+                        state: state,
+                        errorMessage: error
+                    };
+                    if (state === 'finished') {
+                        $timeout(function () {
+                            $scope.close(filename);
+                        }, 3000);
+                    }
+                };
+                PdfCreate.registerStateChangeCallback(createStateChange);
+
+                $scope.close = function (filename) {
+                    delete $scope.pdfs[filename];
+                };
+            },
+        };
+    }
+]);
 
 }());

openslides/core/static/templates/index.html

@@ -212,6 +212,8 @@
   </div><!--end content-container-->
 </div><!--end content-->
 
+<pdf-generation-status></pdf-generation-status>
+
 </div><!--end wrapper-->
 
 <script src="/webclient/site/"></script>

openslides/core/static/templates/pdf-status.html

@@ -0,0 +1,24 @@
+<div id="pdf-status">
+  <div id="pdf-status-container">
+    <div ng-repeat="(filename, pdf) in pdfs" ng-class="pdf.state">
+
+      <span class="close fa fa-times fa-lg" ng-click="close(filename)"></span>
+      <span ng-if="pdf.state === 'generating'">
+        <i class="fa fa-spinner fa-pulse fa-lg spacer-right"></i>
+        <translate>Generating PDF file {{ filename }} ...</translate>
+      </span>
+
+      <span ng-if="pdf.state === 'finished'">
+        <i class="fa fa-check fa-lg spacer-right"></i>
+        <translate>PDF successfully generated.</translate>
+      </span>
+
+      <span ng-if="pdf.state === 'error'">
+        <i class="fa fa-exclamation-triangle fa-lg spacer-right"></i>
+        <translate>Error while generating PDF file</translate> {{ filename }}:
+        <span ng-if="pdf.errorMessage"><code>{{ pdf.errorMessage | translate }}</code></span>
+      </span>
+
+    </div>
+  </div>
+</div>

openslides/motions/static/js/motions/motion-services.js

@@ -15,8 +15,9 @@ angular.module('OpenSlidesApp.motions.motionservices', ['OpenSlidesApp.motions',
     'PollContentProvider',
     'gettextCatalog',
     '$http',
+    'PdfCreate',
     function (HTMLValidizer, Motion, User, PdfMakeConverter, PdfMakeDocumentProvider, PdfMakeBallotPaperProvider,
-            MotionContentProvider, PollContentProvider, gettextCatalog, $http) {
+            MotionContentProvider, PollContentProvider, gettextCatalog, $http, PdfCreate) {
         var obj = {};
 
         var $scope;
@@ -30,11 +31,11 @@ angular.module('OpenSlidesApp.motions.motionservices', ['OpenSlidesApp.motions',
             });
 
             $http.post('/core/encode_media/', JSON.stringify(image_sources)).success(function(data) {
-                var converter = PdfMakeConverter.createInstance(data.images, pdfMake);
+                var converter = PdfMakeConverter.createInstance(data.images);
                 var motionContentProvider = MotionContentProvider.createInstance(converter, $scope.motion, $scope, User, $http);
                 var documentProvider = PdfMakeDocumentProvider.createInstance(motionContentProvider);
                 var filename = gettextCatalog.getString("Motion") + "-" + $scope.motion.identifier + ".pdf";
-                pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+                PdfCreate.download(documentProvider.getDocument(), filename);
             });
         };
 
@@ -45,7 +46,7 @@ angular.module('OpenSlidesApp.motions.motionservices', ['OpenSlidesApp.motions',
             var filename = gettextCatalog.getString("Motion") + "-" + id + "-" + gettextCatalog.getString("ballot-paper") + ".pdf";
             var pollContentProvider = PollContentProvider.createInstance(title, id, gettextCatalog);
             var documentProvider = PdfMakeBallotPaperProvider.createInstance(pollContentProvider);
-            pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+            PdfCreate.download(documentProvider.getDocument(), filename);
         };
 
         obj.init = function (_scope) {

openslides/motions/static/js/motions/site.js

@@ -643,10 +643,11 @@ angular.module('OpenSlidesApp.motions.site', [
     'ProjectionDefault',
     'osTableFilter',
     'osTableSort',
+    'PdfCreate',
     function($scope, $state, $http, gettext, gettextCatalog, ngDialog, MotionForm, Motion,
                 Category, Tag, Workflow, User, Agenda, MotionBlock, MotionCsvExport, MotionDocxExport,
                 MotionContentProvider, MotionCatalogContentProvider, PdfMakeConverter, PdfMakeDocumentProvider,
-                HTMLValidizer, Projector, ProjectionDefault, osTableFilter, osTableSort) {
+                HTMLValidizer, Projector, ProjectionDefault, osTableFilter, osTableSort, PdfCreate) {
         Motion.bindAll({}, $scope, 'motions');
         Category.bindAll({}, $scope, 'categories');
         MotionBlock.bindAll({}, $scope, 'motionBlocks');
@@ -863,7 +864,7 @@ angular.module('OpenSlidesApp.motions.site', [
 
             //post-request to convert the images. Async.
             $http.post('/core/encode_media/', JSON.stringify(image_sources)).success(function(data) {
-                var converter = PdfMakeConverter.createInstance(data.images, pdfMake);
+                var converter = PdfMakeConverter.createInstance(data.images);
                 var motionContentProviderArray = [];
 
                 //convert the filtered motions to motionContentProviders
@@ -872,7 +873,8 @@ angular.module('OpenSlidesApp.motions.site', [
                 });
                 var motionCatalogContentProvider = MotionCatalogContentProvider.createInstance(motionContentProviderArray, $scope, User, Category);
                 var documentProvider = PdfMakeDocumentProvider.createInstance(motionCatalogContentProvider);
-                pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+
+                PdfCreate.download(documentProvider.getDocument(), filename);
             });
         };
 

openslides/users/static/js/users/site.js

@@ -425,9 +425,10 @@ angular.module('OpenSlidesApp.users.site', [
     'osTableFilter',
     'osTableSort',
     'gettext',
+    'PdfCreate',
     function($scope, $state, $http, ngDialog, UserForm, User, Group, PasswordGenerator, Projector, ProjectionDefault,
         UserListContentProvider, Config, UserAccessDataListContentProvider, PdfMakeDocumentProvider, gettextCatalog,
-        UserCsvExport, osTableFilter, osTableSort, gettext) {
+        UserCsvExport, osTableFilter, osTableSort, gettext, PdfCreate) {
         User.bindAll({}, $scope, 'users');
         Group.bindAll({where: {id: {'>': 1}}}, $scope, 'groups');
         $scope.$watch(function () {
@@ -623,7 +624,7 @@ angular.module('OpenSlidesApp.users.site', [
             var filename = gettextCatalog.getString("List of participants")+".pdf";
             var userListContentProvider = UserListContentProvider.createInstance($scope.usersFiltered, $scope.groups);
             var documentProvider = PdfMakeDocumentProvider.createInstance(userListContentProvider);
-            pdfMake.createPdf(documentProvider.getDocument()).download(filename);
+            PdfCreate.download(documentProvider.getDocument(), filename);
         };
         $scope.pdfExportUserAccessDataList = function () {
             var filename = gettextCatalog.getString("List of access data")+".pdf";
@@ -631,7 +632,7 @@ angular.module('OpenSlidesApp.users.site', [
                 $scope.usersFiltered, $scope.groups, Config);
             var documentProvider = PdfMakeDocumentProvider.createInstance(userAccessDataListContentProvider);
             var noFooter = true;
-            pdfMake.createPdf(documentProvider.getDocument(noFooter)).download(filename);
+            PdfCreate.download(documentProvider.getDocument(noFooter), filename);
         };
         // Export as a csv file
         $scope.csvExport = function () {