From e1b4c1fc68332e8c11f695d9afcb24c5cbc63792 Mon Sep 17 00:00:00 2001 From: FinnStutzenstein Date: Fri, 27 Jan 2017 13:38:49 +0100 Subject: [PATCH] Used worker for pdf generation. Moved pdfmake.createPdf() into a web worker thread to prevent blocking UI and max_script_runtime error in browser. Used gulp to manage separate worker files (pdf-worker and pdf-worker-lib). --- CHANGELOG | 1 + bower.json | 5 +- gulpfile.js | 27 ++++- openslides/agenda/static/js/agenda/site.js | 5 +- .../assignments/static/js/assignments/site.js | 12 ++- openslides/core/static/css/app.css | 33 ++++++ openslides/core/static/js/core/pdf-worker.js | 37 +++++++ openslides/core/static/js/core/pdf.js | 101 ++++++++++++++---- openslides/core/static/templates/index.html | 2 + .../core/static/templates/pdf-status.html | 24 +++++ .../static/js/motions/motion-services.js | 9 +- openslides/motions/static/js/motions/site.js | 8 +- openslides/users/static/js/users/site.js | 7 +- 13 files changed, 227 insertions(+), 44 deletions(-) create mode 100644 openslides/core/static/js/core/pdf-worker.js create mode 100644 openslides/core/static/templates/pdf-status.html diff --git a/CHANGELOG b/CHANGELOG index 2d971ddd8..6b4ff452d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -18,6 +18,7 @@ Core: - Added control for the resolution of the projectors. - Set the projector language in the settings. - Used pdfMake for clientside generation of PDFs. + Run pdf creation in background (in a web worker thread). - Introduced new table design for list views with serveral filters and CSV export. - Used session cookies and store filter settings in session storage. diff --git a/bower.json b/bower.json index dd18759eb..a5f0fda6c 100644 --- a/bower.json +++ b/bower.json @@ -39,10 +39,7 @@ }, "overrides": { "pdfmake": { - "main": [ - "build/pdfmake.js", - "build/vfs_fonts.js" - ] + "main": [] }, "pdfjs-dist": { "main": "build/pdf.combined.js" diff --git a/gulpfile.js b/gulpfile.js index fcb4c9e17..62da32e17 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -41,10 +41,13 @@ var output_directory = path.join('openslides', 'static'); * Default tasks to be run before start. */ -// Catches all JavaScript files from all core apps and concats them to one +// Catches all JavaScript files (excluded worker files) from all core apps and concats them to one // file js/openslides.js. In production mode the file is uglified. gulp.task('js', function () { - return gulp.src(path.join('openslides', '*', 'static', 'js', '**', '*.js')) + return gulp.src([ + path.join('openslides', '*', 'static', 'js', '**', '*.js'), + '!' + path.join('openslides', 'core', 'static', 'js', 'core', 'pdf-worker.js'), + ]) .pipe(sourcemaps.init()) .pipe(concat('openslides.js')) .pipe(sourcemaps.write()) @@ -68,6 +71,24 @@ gulp.task('js-libs', function () { .pipe(gulp.dest(path.join(output_directory, 'js'))); }); +// Catches all pdfmake files for pdf worker. +gulp.task('pdf-worker', function () { + return gulp.src([ + path.join('openslides', 'core', 'static', 'js', 'core', 'pdf-worker.js'), + ]) + .pipe(gulpif(argv.production, uglify())) + .pipe(gulp.dest(path.join(output_directory, 'js', 'workers'))); +}); +// pdfmake files +gulp.task('pdf-worker-libs', function () { + return gulp.src([ + path.join('bower_components', 'pdfmake', 'build', 'pdfmake.min.js'), + path.join('bower_components', 'pdfmake', 'build', 'vfs_fonts.js'), + ]) + .pipe(concat('pdf-worker-libs.js')) + .pipe(gulp.dest(path.join(output_directory, 'js', 'workers'))); +}); + // Catches all template files from all core apps and concats them to one // file js/openslides-templates.js. In production mode the file is uglified. gulp.task('templates', function () { @@ -191,6 +212,8 @@ gulp.task('translations', function () { gulp.task('default', [ 'js', 'js-libs', + 'pdf-worker', + 'pdf-worker-libs', 'templates', 'css-libs', 'fonts-libs', diff --git a/openslides/agenda/static/js/agenda/site.js b/openslides/agenda/static/js/agenda/site.js index b5d0bd9fa..c0ca5d19f 100644 --- a/openslides/agenda/static/js/agenda/site.js +++ b/openslides/agenda/static/js/agenda/site.js @@ -82,9 +82,10 @@ angular.module('OpenSlidesApp.agenda.site', [ 'gettext', 'osTableFilter', 'AgendaCsvExport', + 'PdfCreate', function($scope, $filter, $http, $state, DS, operator, ngDialog, Agenda, TopicForm, AgendaTree, Projector, ProjectionDefault, AgendaContentProvider, PdfMakeDocumentProvider, gettextCatalog, gettext, osTableFilter, - AgendaCsvExport) { + AgendaCsvExport, PdfCreate) { // Bind agenda tree to the scope $scope.$watch(function () { return Agenda.lastModified(); @@ -266,7 +267,7 @@ angular.module('OpenSlidesApp.agenda.site', [ var filename = gettextCatalog.getString('Agenda') + '.pdf'; var agendaContentProvider = AgendaContentProvider.createInstance($scope.itemsFiltered); var documentProvider = PdfMakeDocumentProvider.createInstance(agendaContentProvider); - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + PdfCreate.download(documentProvider.getDocument(), filename); }; $scope.csvExport = function () { var element = document.getElementById('downloadLinkCSV'); diff --git a/openslides/assignments/static/js/assignments/site.js b/openslides/assignments/static/js/assignments/site.js index 68aa632dd..e6d612603 100644 --- a/openslides/assignments/static/js/assignments/site.js +++ b/openslides/assignments/static/js/assignments/site.js @@ -264,9 +264,10 @@ angular.module('OpenSlidesApp.assignments.site', [ 'osTableSort', 'gettext', 'phases', + 'PdfCreate', function($scope, ngDialog, AssignmentForm, Assignment, Tag, Agenda, Projector, ProjectionDefault, gettextCatalog, AssignmentContentProvider, AssignmentCatalogContentProvider, PdfMakeDocumentProvider, - User, osTableFilter, osTableSort, gettext, phases) { + User, osTableFilter, osTableSort, gettext, phases, PdfCreate) { Assignment.bindAll({}, $scope, 'assignments'); Tag.bindAll({}, $scope, 'tags'); $scope.$watch(function () { @@ -386,7 +387,7 @@ angular.module('OpenSlidesApp.assignments.site', [ AssignmentCatalogContentProvider.createInstance(assignmentContentProviderArray); var documentProvider = PdfMakeDocumentProvider.createInstance(assignmentCatalogContentProvider); - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + PdfCreate.download(documentProvider.getDocument(), filename); }; } ]) @@ -411,9 +412,10 @@ angular.module('OpenSlidesApp.assignments.site', [ 'PdfMakeDocumentProvider', 'PdfMakeBallotPaperProvider', 'gettextCatalog', + 'PdfCreate', function($scope, $http, $filter, filterFilter, gettext, ngDialog, AssignmentForm, operator, Assignment, User, assignmentId, phases, Projector, ProjectionDefault, AssignmentContentProvider, BallotContentProvider, - PdfMakeDocumentProvider, PdfMakeBallotPaperProvider, gettextCatalog) { + PdfMakeDocumentProvider, PdfMakeBallotPaperProvider, gettextCatalog, PdfCreate) { var assignment = Assignment.get(assignmentId); User.bindAll({}, $scope, 'users'); Assignment.loadRelations(assignment, 'agenda_item'); @@ -593,7 +595,7 @@ angular.module('OpenSlidesApp.assignments.site', [ var filename = gettextCatalog.getString("Election") + "_" + $scope.assignment.title + ".pdf"; var assignmentContentProvider = AssignmentContentProvider.createInstance(assignment); var documentProvider = PdfMakeDocumentProvider.createInstance(assignmentContentProvider); - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + PdfCreate.download(documentProvider.getDocument(), filename); }; //creates the ballotpaper as pdf @@ -609,7 +611,7 @@ angular.module('OpenSlidesApp.assignments.site', [ var filename = gettextCatalog.getString("Ballot") + "_" + pollNumber + "_" + $scope.assignment.title + ".pdf"; var ballotContentProvider = BallotContentProvider.createInstance($scope, thePoll, pollNumber); var documentProvider = PdfMakeBallotPaperProvider.createInstance(ballotContentProvider); - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + PdfCreate.download(documentProvider.getDocument(), filename); }; // Just mark some vote value strings for translation. diff --git a/openslides/core/static/css/app.css b/openslides/core/static/css/app.css index c5748092d..1120fefe0 100644 --- a/openslides/core/static/css/app.css +++ b/openslides/core/static/css/app.css @@ -1133,6 +1133,39 @@ img { font-size: 90%; } +/** Pdf creation status bar **/ +#pdf-status { + position: fixed; + bottom: 0; + width: 100%; + z-index: 100; +} +#pdf-status-container { + margin: 0 auto 0 auto; + padding: 0px 20px; + max-width: 1400px; +} +#pdf-status-container > div { + margin-bottom: 10px; + padding: 10px 20px; + border-radius: 6px; + box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19); +} +#pdf-status .generating { + color: #222; + background-color: #bed4de; + border-color: #46b8da; +} +#pdf-status .error { + color: #a94442; + background-color: #f2dede; + border-color: #ebccd1; +} +#pdf-status .finished { + color: #3c763d; + background-color: #dff0d8; + border-color: #d6e9c6; +} /** General helper classes **/ diff --git a/openslides/core/static/js/core/pdf-worker.js b/openslides/core/static/js/core/pdf-worker.js new file mode 100644 index 000000000..24e453edf --- /dev/null +++ b/openslides/core/static/js/core/pdf-worker.js @@ -0,0 +1,37 @@ +/* Worker for creating PDFs in a separate thread. The creation of larger PDFs + * needs (currently) a lot of time and we don't want to block the UI. + */ + +// Setup fake environment for pdfMake +var document = { + 'createElementNS': function () { + return {}; + }, +}; +var window = this; + +// PdfMake and Fonts +importScripts('/static/js/workers/pdf-worker-libs.js'); + +// Set default font family. +// To use custom ttf font files you have to replace the vfs_fonts.js file. +// See https://github.com/pdfmake/pdfmake/wiki/Custom-Fonts---client-side +// "PdfFont" is used as generic name in core/pdf.js. Adjust the four +// font style names only. +pdfMake.fonts = { + PdfFont: { + normal: 'Roboto-Regular.ttf', + bold: 'Roboto-Medium.ttf', + italics: 'Roboto-Italic.ttf', + bolditalics: 'Roboto-Italic.ttf' + } +}; + +// Create PDF on message and return the base64 decoded document +self.addEventListener('message', function(e) { + var data = JSON.parse(e.data); + var pdf = pdfMake.createPdf(data); + pdf.getBase64(function (base64) { + self.postMessage(base64); + }); +}, false); diff --git a/openslides/core/static/js/core/pdf.js b/openslides/core/static/js/core/pdf.js index 4351df837..addd27aca 100644 --- a/openslides/core/static/js/core/pdf.js +++ b/openslides/core/static/js/core/pdf.js @@ -15,22 +15,6 @@ angular.module('OpenSlidesApp.core.pdf', []) size: 8 }; - // Set and return default font family. - // - // To use custom ttf font files you have to replace the vfs_fonts.js file. - // See https://github.com/pdfmake/pdfmake/wiki/Custom-Fonts---client-side - PDFLayout.getFontName = function() { - pdfMake.fonts = { - Roboto: { - normal: 'Roboto-Regular.ttf', - bold: 'Roboto-Medium.ttf', - italics: 'Roboto-Italic.ttf', - bolditalics: 'Roboto-Italic.ttf' - } - }; - return "Roboto"; - }; - // page title PDFLayout.createTitle = function(title) { return { @@ -229,7 +213,7 @@ angular.module('OpenSlidesApp.core.pdf', []) pageSize: 'A4', pageMargins: [80, 90, 80, 60], defaultStyle: { - font: PDFLayout.getFontName(), + font: 'PdfFont', fontSize: 10 }, header: header, @@ -353,7 +337,7 @@ angular.module('OpenSlidesApp.core.pdf', []) pageSize: 'A4', pageMargins: [0, 0, 0, 0], defaultStyle: { - font: PDFLayout.getFontName(), + font: 'PdfFont', fontSize: 10 }, content: content, @@ -387,9 +371,8 @@ angular.module('OpenSlidesApp.core.pdf', []) * Converter component for HTML->JSON for pdfMake * @constructor * @param {object} images - Key-Value structure representing image.src/BASE64 of images - * @param {object} pdfMake - the converter component enhances pdfMake */ - var createInstance = function(images, pdfMake) { + var createInstance = function(images) { var slice = Function.prototype.call.bind([].slice), map = Function.prototype.call.bind([].map), @@ -820,6 +803,82 @@ angular.module('OpenSlidesApp.core.pdf', []) return { createInstance: createInstance }; -}]); +}]) + +.factory('PdfCreate', [ + '$timeout', + 'FileSaver', + function ($timeout, FileSaver) { + var stateChangeCallbacks = []; + var b64toBlob = function(b64Data) { + var byteCharacters = atob(b64Data); + var byteNumbers = new Array(byteCharacters.length); + for (var i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + var byteArray = new Uint8Array(byteNumbers); + var blob = new Blob([byteArray]); + return blob; + }; + var stateChange = function (state, filename, error) { + _.forEach(stateChangeCallbacks, function (cb) { + $timeout(function () {cb(state, filename, error);}, 1); + }); + }; + return { + download: function (pdfDocument, filename) { + stateChange('generating', filename); + var pdfWorker = new Worker('/static/js/workers/pdf-worker.js'); + + pdfWorker.addEventListener('message', function (event) { + var blob = b64toBlob(event.data); + stateChange('finished', filename); + FileSaver.saveAs(blob, filename); + }); + pdfWorker.addEventListener('error', function (event) { + stateChange('error', filename, event.message); + }); + pdfWorker.postMessage(JSON.stringify(pdfDocument)); + }, + registerStateChangeCallback: function (cb) { + if (cb && typeof cb === 'function') { + stateChangeCallbacks.push(cb); + } + }, + }; + } +]) + +.directive('pdfGenerationStatus', [ + '$timeout', + 'PdfCreate', + function($timeout, PdfCreate) { + return { + restrict: 'E', + templateUrl: 'static/templates/pdf-status.html', + scope: {}, + controller: function ($scope, $element, $attrs, $location) { + $scope.pdfs = {}; + + var createStateChange = function (state, filename, error) { + $scope.pdfs[filename] = { + state: state, + errorMessage: error + }; + if (state === 'finished') { + $timeout(function () { + $scope.close(filename); + }, 3000); + } + }; + PdfCreate.registerStateChangeCallback(createStateChange); + + $scope.close = function (filename) { + delete $scope.pdfs[filename]; + }; + }, + }; + } +]); }()); diff --git a/openslides/core/static/templates/index.html b/openslides/core/static/templates/index.html index 1c41fdac8..dcf9edd04 100644 --- a/openslides/core/static/templates/index.html +++ b/openslides/core/static/templates/index.html @@ -212,6 +212,8 @@ + + diff --git a/openslides/core/static/templates/pdf-status.html b/openslides/core/static/templates/pdf-status.html new file mode 100644 index 000000000..0d13d6e30 --- /dev/null +++ b/openslides/core/static/templates/pdf-status.html @@ -0,0 +1,24 @@ +
+
+
+ + + + + Generating PDF file {{ filename }} ... + + + + + PDF successfully generated. + + + + + Error while generating PDF file {{ filename }}: + {{ pdf.errorMessage | translate }} + + +
+
+
diff --git a/openslides/motions/static/js/motions/motion-services.js b/openslides/motions/static/js/motions/motion-services.js index 30a83d55c..087b6b406 100644 --- a/openslides/motions/static/js/motions/motion-services.js +++ b/openslides/motions/static/js/motions/motion-services.js @@ -15,8 +15,9 @@ angular.module('OpenSlidesApp.motions.motionservices', ['OpenSlidesApp.motions', 'PollContentProvider', 'gettextCatalog', '$http', + 'PdfCreate', function (HTMLValidizer, Motion, User, PdfMakeConverter, PdfMakeDocumentProvider, PdfMakeBallotPaperProvider, - MotionContentProvider, PollContentProvider, gettextCatalog, $http) { + MotionContentProvider, PollContentProvider, gettextCatalog, $http, PdfCreate) { var obj = {}; var $scope; @@ -30,11 +31,11 @@ angular.module('OpenSlidesApp.motions.motionservices', ['OpenSlidesApp.motions', }); $http.post('/core/encode_media/', JSON.stringify(image_sources)).success(function(data) { - var converter = PdfMakeConverter.createInstance(data.images, pdfMake); + var converter = PdfMakeConverter.createInstance(data.images); var motionContentProvider = MotionContentProvider.createInstance(converter, $scope.motion, $scope, User, $http); var documentProvider = PdfMakeDocumentProvider.createInstance(motionContentProvider); var filename = gettextCatalog.getString("Motion") + "-" + $scope.motion.identifier + ".pdf"; - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + PdfCreate.download(documentProvider.getDocument(), filename); }); }; @@ -45,7 +46,7 @@ angular.module('OpenSlidesApp.motions.motionservices', ['OpenSlidesApp.motions', var filename = gettextCatalog.getString("Motion") + "-" + id + "-" + gettextCatalog.getString("ballot-paper") + ".pdf"; var pollContentProvider = PollContentProvider.createInstance(title, id, gettextCatalog); var documentProvider = PdfMakeBallotPaperProvider.createInstance(pollContentProvider); - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + PdfCreate.download(documentProvider.getDocument(), filename); }; obj.init = function (_scope) { diff --git a/openslides/motions/static/js/motions/site.js b/openslides/motions/static/js/motions/site.js index ca097038f..49f8b9dcc 100644 --- a/openslides/motions/static/js/motions/site.js +++ b/openslides/motions/static/js/motions/site.js @@ -643,10 +643,11 @@ angular.module('OpenSlidesApp.motions.site', [ 'ProjectionDefault', 'osTableFilter', 'osTableSort', + 'PdfCreate', function($scope, $state, $http, gettext, gettextCatalog, ngDialog, MotionForm, Motion, Category, Tag, Workflow, User, Agenda, MotionBlock, MotionCsvExport, MotionDocxExport, MotionContentProvider, MotionCatalogContentProvider, PdfMakeConverter, PdfMakeDocumentProvider, - HTMLValidizer, Projector, ProjectionDefault, osTableFilter, osTableSort) { + HTMLValidizer, Projector, ProjectionDefault, osTableFilter, osTableSort, PdfCreate) { Motion.bindAll({}, $scope, 'motions'); Category.bindAll({}, $scope, 'categories'); MotionBlock.bindAll({}, $scope, 'motionBlocks'); @@ -863,7 +864,7 @@ angular.module('OpenSlidesApp.motions.site', [ //post-request to convert the images. Async. $http.post('/core/encode_media/', JSON.stringify(image_sources)).success(function(data) { - var converter = PdfMakeConverter.createInstance(data.images, pdfMake); + var converter = PdfMakeConverter.createInstance(data.images); var motionContentProviderArray = []; //convert the filtered motions to motionContentProviders @@ -872,7 +873,8 @@ angular.module('OpenSlidesApp.motions.site', [ }); var motionCatalogContentProvider = MotionCatalogContentProvider.createInstance(motionContentProviderArray, $scope, User, Category); var documentProvider = PdfMakeDocumentProvider.createInstance(motionCatalogContentProvider); - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + + PdfCreate.download(documentProvider.getDocument(), filename); }); }; diff --git a/openslides/users/static/js/users/site.js b/openslides/users/static/js/users/site.js index d975102c6..8b89c4c36 100644 --- a/openslides/users/static/js/users/site.js +++ b/openslides/users/static/js/users/site.js @@ -425,9 +425,10 @@ angular.module('OpenSlidesApp.users.site', [ 'osTableFilter', 'osTableSort', 'gettext', + 'PdfCreate', function($scope, $state, $http, ngDialog, UserForm, User, Group, PasswordGenerator, Projector, ProjectionDefault, UserListContentProvider, Config, UserAccessDataListContentProvider, PdfMakeDocumentProvider, gettextCatalog, - UserCsvExport, osTableFilter, osTableSort, gettext) { + UserCsvExport, osTableFilter, osTableSort, gettext, PdfCreate) { User.bindAll({}, $scope, 'users'); Group.bindAll({where: {id: {'>': 1}}}, $scope, 'groups'); $scope.$watch(function () { @@ -623,7 +624,7 @@ angular.module('OpenSlidesApp.users.site', [ var filename = gettextCatalog.getString("List of participants")+".pdf"; var userListContentProvider = UserListContentProvider.createInstance($scope.usersFiltered, $scope.groups); var documentProvider = PdfMakeDocumentProvider.createInstance(userListContentProvider); - pdfMake.createPdf(documentProvider.getDocument()).download(filename); + PdfCreate.download(documentProvider.getDocument(), filename); }; $scope.pdfExportUserAccessDataList = function () { var filename = gettextCatalog.getString("List of access data")+".pdf"; @@ -631,7 +632,7 @@ angular.module('OpenSlidesApp.users.site', [ $scope.usersFiltered, $scope.groups, Config); var documentProvider = PdfMakeDocumentProvider.createInstance(userAccessDataListContentProvider); var noFooter = true; - pdfMake.createPdf(documentProvider.getDocument(noFooter)).download(filename); + PdfCreate.download(documentProvider.getDocument(noFooter), filename); }; // Export as a csv file $scope.csvExport = function () {