3.0.14

.gitlab-ci.yml

@@ -12,40 +12,36 @@ stages:
   - release
   - metadata
 
+before_script:
+  - pnpm install -g pnpm
+  - pnpm install -g @shipzone/npmci
+  - npmci npm prepare
+
+# ====================
+# security stage
+# ====================
 # ====================
 # security stage
 # ====================
-mirror:
-  stage: security
-  script:
-    - npmci git mirror
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
 auditProductionDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
-    - npmci npm prepare
+     - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm install --production --ignore-scripts
+     - npmci command pnpm audit --audit-level=high --prod
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=prod --production
   tags:
+    - lossless
     - docker
+  allow_failure: true
 
 auditDevDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
-    - npmci npm prepare
-    - npmci command npm install --ignore-scripts
     - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=dev
+    - npmci command pnpm audit --audit-level=high --dev
   tags:
+    - lossless
     - docker
   allow_failure: true
 
@@ -56,7 +52,6 @@ auditDevDependencies:
 testStable:
   stage: test
   script:
-    - npmci npm prepare
     - npmci node install stable
     - npmci npm install
     - npmci npm test
@@ -67,7 +62,6 @@ testStable:
 testBuild:
   stage: test
   script:
-    - npmci npm prepare
     - npmci node install stable
     - npmci npm install
     - npmci command npm run build
@@ -96,10 +90,9 @@ codequality:
   only:
     - tags
   script:
-    - npmci command npm install -g tslint typescript
+    - npmci command npm install -g typescript
     - npmci npm prepare
     - npmci npm install
-    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
   tags:
     - lossless
     - docker
@@ -119,11 +112,9 @@ trigger:
 pages:
   stage: metadata
   script:
-    - npmci node install lts
+    - npmci node install stable
-    - npmci command npm install -g @gitzone/tsdoc
-    - npmci npm prepare
     - npmci npm install
-    - npmci command tsdoc
+    - npmci command npm run buildDocs
   tags:
     - lossless
     - docker

.snyk

@@ -1,9 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.13.5
-# ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore:
-  SNYK-JS-LODASH-450202:
-    - '@pushrocks/smartnetwork > portscanner > async > lodash':
-        reason: None given
-        expires: '2019-09-13T09:39:26.276Z'
-patch: {}

.vscode/launch.json

@@ -2,28 +2,10 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "current file",
+      "command": "npm test",
-      "type": "node",
+      "name": "Run npm test",
       "request": "launch",
-      "args": [
+      "type": "node-terminal"
-        "${relativeFile}"
-      ],
-      "runtimeArgs": ["-r", "@gitzone/tsrun"],
-      "cwd": "${workspaceRoot}",
-      "protocol": "inspector",
-      "internalConsoleOptions": "openOnSessionStart"
-    },
-    {
-      "name": "test.ts",
-      "type": "node",
-      "request": "launch",
-      "args": [
-        "test/test.ts"
-      ],
-      "runtimeArgs": ["-r", "@gitzone/tsrun"],
-      "cwd": "${workspaceRoot}",
-      "protocol": "inspector",
-      "internalConsoleOptions": "openOnSessionStart"
     }
   ]
 }

npmextra.json

@@ -9,7 +9,7 @@
       "githost": "gitlab.com",
       "gitscope": "pushrocks",
       "gitrepo": "smartpdf",
-      "shortDescription": "Create PDFs fast and smoothly",
+      "description": "Create PDFs fast and smoothly",
       "npmPackagename": "@pushrocks/smartpdf",
       "license": "MIT"
     }

package.json

@@ -1,34 +1,39 @@
 {
   "name": "@pushrocks/smartpdf",
-  "version": "2.0.13",
+  "version": "3.0.14",
   "private": false,
   "description": "create pdfs on the fly",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
+  "type": "module",
   "author": "Lossless GmbH",
   "license": "MIT",
   "scripts": {
     "test": "(tstest test/ --web)",
-    "build": "(tsbuild --web)"
+    "build": "(tsbuild --web --allowimplicitany)",
+    "buildDocs": "tsdoc"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.28",
+    "@gitzone/tsbuild": "^2.1.65",
-    "@gitzone/tsrun": "^1.2.18",
+    "@gitzone/tsdoc": "^1.1.12",
-    "@gitzone/tstest": "^1.0.60",
+    "@gitzone/tsrun": "^1.2.35",
-    "@pushrocks/tapbundle": "^3.2.15",
+    "@gitzone/tstest": "^1.0.73",
-    "@types/node": "^17.0.8",
+    "@pushrocks/tapbundle": "^5.0.4",
-    "tslint": "^6.1.3",
+    "@types/node": "^18.7.18"
-    "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
-    "@pushrocks/smartfile": "^9.0.5",
+    "@pushrocks/smartdelay": "^2.0.13",
-    "@pushrocks/smartnetwork": "^2.0.10",
+    "@pushrocks/smartfile": "^10.0.5",
-    "@pushrocks/smartpromise": "^3.1.6",
+    "@pushrocks/smartnetwork": "^3.0.0",
-    "@pushrocks/smartpuppeteer": "^1.0.36",
+    "@pushrocks/smartpath": "^5.0.5",
+    "@pushrocks/smartpromise": "^3.1.7",
+    "@pushrocks/smartpuppeteer": "^2.0.2",
     "@pushrocks/smartunique": "^3.0.3",
-    "@types/express": "^4.17.13",
+    "@tsclass/tsclass": "^4.0.21",
-    "express": "^4.17.2",
+    "@types/express": "^4.17.14",
-    "pdf-merger-js": "^3.2.1"
+    "express": "^4.18.1",
+    "pdf-merger-js": "^3.4.0",
+    "pdf2json": "^2.0.0"
   },
   "files": [
     "ts/**/*",

test/test.ts

@@ -1,11 +1,11 @@
 import { expect, tap } from '@pushrocks/tapbundle';
-import * as smartpdf from '../ts/index';
+import * as smartpdf from '../ts/index.js';
 
 let testSmartPdf: smartpdf.SmartPdf;
 
 tap.test('should create a valid instance of smartpdf', async () => {
   testSmartPdf = new smartpdf.SmartPdf();
-  expect(testSmartPdf).to.be.instanceof(smartpdf.SmartPdf);
+  expect(testSmartPdf).toBeInstanceOf(smartpdf.SmartPdf);
 });
 
 tap.test('should start the instance', async () => {
@@ -13,7 +13,11 @@ tap.test('should start the instance', async () => {
 });
 
 tap.test('should create a pdf from html string', async () => {
-  await testSmartPdf.getPdfResultForHtmlString('hi');
+  await testSmartPdf.getA4PdfResultForHtmlString('hi');
+});
+
+tap.test('should create a pdf from html string', async () => {
+  await testSmartPdf.getA4PdfResultForHtmlString('hi');
 });
 
 tap.test('should create a pdf from website as A4', async () => {
@@ -27,25 +31,27 @@ tap.test('should create a pdf from website as single page PDF', async () => {
 tap.test('should create a valid PDFResult', async () => {
   const writePDfToDisk = async (urlArg: string, fileName: string) => {
     const pdfResult = await testSmartPdf.getFullWebsiteAsSinglePdf(urlArg);
-    expect(pdfResult.buffer).to.be.instanceOf(Buffer);
+    expect(pdfResult.buffer).toBeInstanceOf(Buffer);
     const fs = await import('fs');
 
     if (!fs.existsSync('.nogit/')) {
       fs.mkdirSync('.nogit/');
     }
-    fs.writeFileSync(`.nogit/${fileName}`, pdfResult.buffer);
+    fs.writeFileSync(`.nogit/${fileName}`, pdfResult.buffer as Buffer);
   };
-  await writePDfToDisk('https://maintainedby.lossless.com/', '1.pdf')
+  await writePDfToDisk('https://maintainedby.lossless.com/', '1.pdf');
-  await writePDfToDisk('https://lossless.com/', '2.pdf')
+  await writePDfToDisk('https://rendertron.lossless.one/render/https://layer.io', '2.pdf');
 });
 
 tap.test('should combine pdfs', async () => {
   const fs = await import('fs');
-  const buffer1 = fs.readFileSync('.nogit/1.pdf');
+  const pdf1 = await testSmartPdf.readFileToPdfObject('.nogit/1.pdf');
-  const buffer2 = fs.readFileSync('.nogit/2.pdf');
+  const pdf2 = await testSmartPdf.readFileToPdfObject('.nogit/2.pdf');
-  fs.writeFileSync(`.nogit/combined.pdf`, await testSmartPdf.mergePdfBuffers([buffer1, buffer2]));
+  fs.writeFileSync(
-
+    `.nogit/combined.pdf`,
-})
+    (await testSmartPdf.mergePdfs([pdf1, pdf2])).buffer as Buffer
+  );
+});
 
 tap.test('should be able to close properly', async () => {
   await testSmartPdf.stop();

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @pushrocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@pushrocks/smartpdf',
+  version: '3.0.14',
+  description: 'create pdfs on the fly'
+}

ts/index.ts

@@ -6,4 +6,4 @@ declare global {
 }
 
 // normal
-export * from './smartpdf.classes.smartpdf';
+export * from './smartpdf.classes.smartpdf.js';

ts/interfaces/index.ts

@@ -1 +0,0 @@
-export * from './interface.pdfresult';

ts/interfaces/interface.pdfresult.ts

@@ -1,5 +0,0 @@
-export interface IPdfResult {
-  name: string;
-  id: string;
-  buffer: Buffer;
-}

ts/smartpdf.classes.pdfcandidate.ts

@@ -1,4 +1,8 @@
-import * as plugins from './smartpdf.plugins';
+/*
+ * pdf candidate is a construct used internally for mapping html
+ * to pdf buffers delivered by puppeteer
+ */
+import * as plugins from './smartpdf.plugins.js';
 
 export class PdfCandidate {
   public htmlString: string;

ts/smartpdf.classes.smartpdf.ts

@@ -1,13 +1,19 @@
-import * as plugins from './smartpdf.plugins';
+import * as plugins from './smartpdf.plugins.js';
-import * as paths from './smartpdf.paths';
+import * as paths from './smartpdf.paths.js';
 import { Server } from 'http';
-import { PdfCandidate } from './smartpdf.classes.pdfcandidate';
+import { PdfCandidate } from './smartpdf.classes.pdfcandidate.js';
+import { IPdf } from '@tsclass/tsclass/dist_ts/business/pdf.js';
 
 declare const document: any;
 
-import * as interfaces from './interfaces';
-
 export class SmartPdf {
+  // STATIC
+  public static async create() {
+    const smartpdfInstance = new SmartPdf();
+    return smartpdfInstance;
+  }
+
+  // INSTANCE
   htmlServerInstance: Server;
   serverPort: number;
   headlessBrowser: plugins.smartpuppeteer.puppeteer.Browser;
@@ -20,6 +26,7 @@ export class SmartPdf {
   }
 
   async start(headlessBrowserArg?: plugins.smartpuppeteer.puppeteer.Browser) {
+    const done = plugins.smartpromise.defer();
     // lets set the external browser in case one is provided
     this.headlessBrowser = headlessBrowserArg;
     // setup puppeteer
@@ -43,7 +50,9 @@ export class SmartPdf {
     this.htmlServerInstance.listen(3210, 'localhost');
     this.htmlServerInstance.on('listening', () => {
       this._readyDeferred.resolve();
+      done.resolve();
     });
+    await done.promise;
   }
 
   // stop
@@ -63,18 +72,19 @@ export class SmartPdf {
   /**
    * returns a pdf for a given html string;
    */
-  async getPdfResultForHtmlString(htmlStringArg: string): Promise<interfaces.IPdfResult> {
+  async getA4PdfResultForHtmlString(htmlStringArg: string): Promise<plugins.tsclass.business.IPdf> {
     await this._readyDeferred.promise;
     const pdfCandidate = new PdfCandidate(htmlStringArg);
     this._candidates[pdfCandidate.pdfId] = pdfCandidate;
     const page = await this.headlessBrowser.newPage();
     await page.setViewport({
-      height: 842,
+      width: 794,
-      width: 595,
+      height: 1122,
-    })
+    });
     const response = await page.goto(`http://localhost:3210/${pdfCandidate.pdfId}`, {
       waitUntil: 'networkidle2',
     });
+    // await plugins.smartdelay.delayFor(1000);
     const headers = response.headers();
     if (headers['pdf-id'] !== pdfCandidate.pdfId) {
       console.log('Error! Headers do not match. For security reasons no pdf is being emitted!');
@@ -84,10 +94,10 @@ export class SmartPdf {
     }
 
     const pdfBuffer = await page.pdf({
-      format: 'a4',
+      width: 794,
+      height: 1122,
       printBackground: true,
       displayHeaderFooter: false,
-      preferCSSPageSize: true,
     });
     await page.close();
     delete this._candidates[pdfCandidate.pdfId];
@@ -96,12 +106,19 @@ export class SmartPdf {
     return {
       id: pdfCandidate.pdfId,
       name: `${pdfCandidate.pdfId}.js`,
+      metadata: {
+        textExtraction: await this.extractTextFromPdfBuffer(pdfBuffer),
+      },
       buffer: pdfBuffer,
     };
   }
 
-  async getPdfResultForWebsite(websiteUrl: string): Promise<interfaces.IPdfResult> {
+  async getPdfResultForWebsite(websiteUrl: string): Promise<plugins.tsclass.business.IPdf> {
     const page = await this.headlessBrowser.newPage();
+    await page.setViewport({
+      width: 1980,
+      height: 1200,
+    });
     await page.emulateMediaType('screen');
     const response = await page.goto(websiteUrl, { waitUntil: 'networkidle2' });
     const pdfId = plugins.smartunique.shortId();
@@ -112,53 +129,101 @@ export class SmartPdf {
       };
     });
     const pdfBuffer = await page.pdf({
-      format: 'a4',
+      height: documentHeight,
-      height: documentWidth,
       width: documentWidth,
       printBackground: true,
       displayHeaderFooter: false,
-      preferCSSPageSize: true,
     });
     await page.close();
     return {
       id: pdfId,
       name: `${pdfId}.js`,
+      metadata: {
+        textExtraction: await this.extractTextFromPdfBuffer(pdfBuffer),
+      },
       buffer: pdfBuffer,
     };
   }
 
-  async getFullWebsiteAsSinglePdf(websiteUrl: string) {
+  async getFullWebsiteAsSinglePdf(websiteUrl: string): Promise<plugins.tsclass.business.IPdf> {
     const page = await this.headlessBrowser.newPage();
+    await page.setViewport({
+      width: 1920,
+      height: 1200,
+    });
     page.emulateMediaType('screen');
     const response = await page.goto(websiteUrl, { waitUntil: 'networkidle2' });
     const pdfId = plugins.smartunique.shortId();
     const { documentHeight, documentWidth } = await page.evaluate(() => {
       return {
-        documentHeight: document.height,
+        documentHeight: document.body.scrollHeight,
-        documentWidth: document.width,
+        documentWidth: document.body.clientWidth,
       };
     });
+    await page.setViewport({
+      width: 1920,
+      height: documentHeight,
+    });
     const pdfBuffer = await page.pdf({
-      format: 'a4',
+      height: documentHeight,
-      height: documentWidth,
+      width: 1920,
-      width: documentWidth,
       printBackground: true,
       displayHeaderFooter: false,
-      preferCSSPageSize: true,
+      scale: 1,
+      pageRanges: '1',
     });
     await page.close();
     return {
       id: pdfId,
       name: `${pdfId}.js`,
+      metadata: {
+        textExtraction: await this.extractTextFromPdfBuffer(pdfBuffer),
+      },
       buffer: pdfBuffer,
     };
   }
 
-  public async mergePdfBuffers(pdfBuffers: Buffer[]): Promise<Buffer> {
+  public async mergePdfs(pdfArrayArg: plugins.tsclass.business.IPdf[]): Promise<IPdf> {
     const merger = new plugins.pdfMerger();
-    for (const buffer of pdfBuffers) {
+    for (const pdf of pdfArrayArg) {
-      merger.add(buffer);
+      merger.add(Buffer.from(pdf.buffer));
     }
-    return merger.saveAsBuffer();
+    const resultBuffer = await merger.saveAsBuffer();
+    return {
+      name: 'mergedPdf',
+      buffer: resultBuffer,
+      id: null,
+      metadata: null,
+    };
+  }
+
+  public async readFileToPdfObject(pathArg: string): Promise<plugins.tsclass.business.IPdf> {
+    const path = plugins.smartpath.transform.makeAbsolute(pathArg);
+    const parsedPath = plugins.path.parse(path);
+    const buffer = await plugins.smartfile.fs.toBuffer(path);
+    return {
+      name: parsedPath.base,
+      buffer,
+      id: null,
+      metadata: null,
+    };
+  }
+
+  public async extractTextFromPdfBuffer(pdfBufferArg: Buffer): Promise<string> {
+    const deferred = plugins.smartpromise.defer<string>();
+    const pdfParser: any = new plugins.pdf2json();
+    pdfParser.on('pdfParser_dataReady', (pdfData: any) => {
+      let finalText = '';
+      for (const page of pdfData.Pages) {
+        for (const text of page.Texts) {
+          for (const letter of text.R) {
+            finalText = finalText + letter.T;
+          }
+        }
+      }
+      deferred.resolve(finalText);
+    });
+    pdfParser.parseBuffer(pdfBufferArg);
+    return deferred.promise;
   }
 }

ts/smartpdf.paths.ts

@@ -1,3 +1,6 @@
-import * as plugins from './smartpdf.plugins';
+import * as plugins from './smartpdf.plugins.js';
 
-export const packageDir = plugins.path.join(__dirname, '../');
+export const packageDir = plugins.path.join(
+  plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
+  '../'
+);

ts/smartpdf.plugins.ts

@@ -6,15 +6,32 @@ export { http, path };
 
 // @pushrocks
 import * as smartfile from '@pushrocks/smartfile';
+import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartpromise from '@pushrocks/smartpromise';
+import * as smartpath from '@pushrocks/smartpath';
 import * as smartpuppeteer from '@pushrocks/smartpuppeteer';
 import * as smartnetwork from '@pushrocks/smartnetwork';
 import * as smartunique from '@pushrocks/smartunique';
 
-export { smartfile, smartpromise, smartpuppeteer, smartunique, smartnetwork };
+export {
+  smartfile,
+  smartdelay,
+  smartpromise,
+  smartpath,
+  smartpuppeteer,
+  smartunique,
+  smartnetwork,
+};
+
+// tsclass scope
+import * as tsclass from '@tsclass/tsclass';
+
+export { tsclass };
 
 // thirdparty
 import pdfMerger from 'pdf-merger-js';
+// @ts-ignore
+import pdf2json from 'pdf2json';
 import express from 'express';
 
-export { pdfMerger, express };
+export { pdfMerger, pdf2json, express };

tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "nodenext",
+    "allowSyntheticDefaultImports": true
+  }
+}

tslint.json

@@ -1,17 +0,0 @@
-{
-  "extends": ["tslint:latest", "tslint-config-prettier"],
-  "rules": {
-    "semicolon": [true, "always"],
-    "no-console": false,
-    "ordered-imports": false,
-    "object-literal-sort-keys": false,
-    "member-ordering": {
-      "options":{
-        "order": [
-          "static-method"
-        ]
-      }
-    }
-  },
-  "defaultSeverity": "warning"
-}