initial

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @push.rocks/commitinfo
+ */
+export const commitinfo = {
+  name: 'rendertron',
+  version: '2.0.61',
+  description: 'a rendering service for lossless gmbh'
+}

ts/index.ts

@@ -0,0 +1,22 @@
+import { logger } from './rendertron.logging.js';
+import { db } from './rendertron.db.js';
+import { Rendertron } from './rendertron.classes.rendertron.js';
+export {
+  Rendertron
+}
+
+let rendertronInstance: Rendertron;
+
+export const runCli = async () => {
+  logger.log('info', `Starting rendertron...`);
+  rendertronInstance = new Rendertron();
+  rendertronInstance.start();
+  logger.log('success', `Successfully started rendertron!`);
+};
+
+export const stop = async () => {
+  if (rendertronInstance) {
+    rendertronInstance.stop();
+  }
+  db.close();
+};

ts/rendertron.classes.prerendermanager.ts

@@ -0,0 +1,97 @@
+import { logger } from './rendertron.logging.js';
+import { PrerenderResult } from './rendertron.classes.prerenderresult.js';
+import * as plugins from './rendertron.plugins.js';
+
+export class PrerenderManager {
+  public smartssrInstance: plugins.smartssr.SmartSSR;
+  public smartrobotsInstance: plugins.smartrobots.Smartrobots;
+  public smartsitemapInstance: plugins.smartsitemap.SmartSitemap;
+
+  constructor() {}
+
+  /**
+   * starts the manager
+   */
+  public async start() {
+    this.smartssrInstance = new plugins.smartssr.SmartSSR();
+    this.smartrobotsInstance = new plugins.smartrobots.Smartrobots();
+    this.smartsitemapInstance = new plugins.smartsitemap.SmartSitemap();
+  }
+
+  /**
+   * stops the manager
+   */
+  public async stop() {}
+
+  public async getPrerenderResultForUrl(urlArg: string): Promise<string> {
+    const done = plugins.smartpromise.defer<string>();
+    const prerenderResult = await PrerenderResult.getPrerenderResultForUrl(this, urlArg).catch(
+      () => {
+        done.resolve(`Cannot render ${urlArg} due to internal error.`);
+        return null;
+      }
+    );
+    done.resolve(prerenderResult.renderResultString);
+    return done.promise;
+  }
+
+  /**
+   * prerenders a sitemap
+   */
+  public async prerenderDomain(domainArg: string) {
+    logger.log('info', `prerendering domain: ${domainArg}`);
+    await this.getPrerenderResultForUrl(`https://${domainArg}/`).catch((err) => {
+      logger.log('error', `failed to prerender ${domainArg}`);
+    });
+    await this.getPrerenderResultForUrl(`https://${domainArg}`).catch((err) => {
+      logger.log('error', `failed to prerender ${domainArg}`);
+    });
+    const robotsTxt = await this.smartrobotsInstance
+      .parseRobotsTxtFromUrl(`https://${domainArg}/robots.txt`)
+      .catch((err) => {
+        logger.log('warn', `no robots for ${domainArg}`);
+      });
+    if (!robotsTxt) {
+      return;
+    }
+    if (robotsTxt.sitemaps.length === 0) {
+      logger.log('warn', `robot-txt for ${domainArg} does bot sepcify any sitemaps`);
+    }
+    for (const sitemapUrl of robotsTxt.sitemaps) {
+      await this.prerenderSitemap(sitemapUrl);
+    }
+  }
+
+  public async prerenderSitemap(sitemapUrlArg: string) {
+    logger.log('info', `prerendering sitemap: ${sitemapUrlArg}`);
+    const parsedSitemap = await this.smartsitemapInstance.parseSitemapUrl(sitemapUrlArg);
+    if (!parsedSitemap.urlset?.url) {
+      return;
+    }
+    if (!(parsedSitemap.urlset.url instanceof Array)) {
+      await this.getPrerenderResultForUrl(parsedSitemap.urlset.url.loc);
+    } else {
+      for (const url of parsedSitemap.urlset.url) {
+        if (!url?.loc) {
+          continue;
+        }
+        await this.getPrerenderResultForUrl(url.loc);
+      }
+    }
+  }
+
+  public async cleanPrerenderResults() {
+    const allPrerenderResults = await PrerenderResult.getInstances<PrerenderResult>({});
+    for (const prerenderResult of allPrerenderResults) {
+      const extendedDate = plugins.smarttime.ExtendedDate.fromMillis(prerenderResult.timestamp);
+      const stillValid = extendedDate.lessTimePassedToNow({ hours: 24 });
+      if (!stillValid) {
+        logger.log(
+          'warn',
+          `deleted prerender result for ${prerenderResult.url} since it is older than 24 hours`
+        );
+        await prerenderResult.delete();
+      }
+    }
+  }
+}

ts/rendertron.classes.prerenderresult.ts

@@ -0,0 +1,94 @@
+import * as plugins from './rendertron.plugins.js';
+
+import { db } from './rendertron.db.js';
+import { PrerenderManager } from './rendertron.classes.prerendermanager.js';
+import { logger } from './rendertron.logging.js';
+
+/**
+ * allows for prerendering results
+ */
+@plugins.smartdata.Collection(() => {
+  return db;
+})
+export class PrerenderResult extends plugins.smartdata.SmartDataDbDoc<
+  PrerenderResult,
+  PrerenderResult
+> {
+  // STATIC
+  public static async getPrerenderResultForUrl(
+    managerArg: PrerenderManager,
+    urlArg: string,
+    forceNew: boolean = false
+  ): Promise<PrerenderResult> {
+    let prerenderResult = await PrerenderResult.getInstance<PrerenderResult>({
+      url: urlArg,
+    });
+
+    if (prerenderResult) {
+      const prerenderResultDate = new plugins.smarttime.ExtendedDate(prerenderResult.timestamp);
+      if (
+        prerenderResultDate.lessTimePassedToNow({
+          hours: 12,
+        })
+      ) {
+        logger.log('info', `Serving prerendered result for ${prerenderResult.url}`);
+        return prerenderResult;
+      } else {
+        logger.log(
+          'info',
+          `Outdated Prerender Result: Requesting newer result for ${prerenderResult.url}`
+        );
+        prerenderResult.needsRerendering = true;
+      }
+    }
+
+    if (!prerenderResult || prerenderResult.needsRerendering) {
+      const newPrerenderResult: PrerenderResult = await PrerenderResult.createPrerenderResultForUrl(
+        managerArg,
+        urlArg
+      ).catch((err) => {
+        return prerenderResult;
+      });
+      prerenderResult = newPrerenderResult;
+    }
+
+    return prerenderResult;
+  }
+
+  private static async createPrerenderResultForUrl(
+    managerArg: PrerenderManager,
+    urlArg: string
+  ): Promise<PrerenderResult> {
+    const renderedResultPromise = managerArg.smartssrInstance.renderPage(urlArg).catch(() => {
+      const errorMessage = `failed to render ${urlArg}`;
+      logger.log('error', errorMessage);
+    });
+    let prerenderResult = await PrerenderResult.getInstance<PrerenderResult>({
+      url: urlArg,
+    });
+    prerenderResult = prerenderResult || new PrerenderResult();
+    prerenderResult.url = urlArg;
+    prerenderResult.timestamp = Date.now();
+    prerenderResult.renderResultString = (await renderedResultPromise) || 'error';
+    prerenderResult.needsRerendering = false;
+    await prerenderResult.save();
+    return prerenderResult;
+  }
+
+  // INSTANCE
+  @plugins.smartdata.unI()
+  url: string;
+
+  @plugins.smartdata.svDb()
+  renderResultString: string;
+
+  @plugins.smartdata.svDb()
+  timestamp: number;
+
+  @plugins.smartdata.svDb()
+  needsRerendering: boolean = false;
+
+  constructor() {
+    super();
+  }
+}

ts/rendertron.classes.rendertron.ts

@@ -0,0 +1,90 @@
+import * as plugins from './rendertron.plugins.js';
+import * as paths from './rendertron.paths.js';
+import { logger } from './rendertron.logging.js';
+import { PrerenderResult } from './rendertron.classes.prerenderresult.js';
+import { PrerenderManager } from './rendertron.classes.prerendermanager.js';
+import { TaskManager } from './rendertron.taskmanager.js';
+
+export class Rendertron {
+  public projectinfo: plugins.projectinfo.ProjectInfo;
+  public serviceServerInstance: plugins.typedserver.utilityservers.UtilityServiceServer;
+  public prerenderManager: PrerenderManager;
+  public taskManager: TaskManager;
+
+  /**
+   * starts the financeflow instance
+   */
+  public async start() {
+    this.projectinfo = new plugins.projectinfo.ProjectInfo(paths.packageDir);
+    this.prerenderManager = new PrerenderManager();
+    this.taskManager = new TaskManager(this);
+    await this.prerenderManager.start();
+    await this.taskManager.start();
+    this.serviceServerInstance = new plugins.typedserver.utilityservers.UtilityServiceServer({
+      serviceDomain: 'rendertron.lossless.one',
+      serviceName: 'rendertron',
+      serviceVersion: this.projectinfo.npm.version,
+      addCustomRoutes: async (serverArg) => {
+        serverArg.addRoute(
+          '/render/*',
+          new plugins.typedserver.servertools.Handler('GET', async (req, res) => {
+            const requestedUrl = req.url.replace('/render/', '');
+            logger.log('info', `Got SSR request for ${requestedUrl}`);
+            if (requestedUrl.startsWith('https://url(')) {
+              logger.log('warn', `relative url error for ${requestedUrl}`);
+              res.status(500);
+              res.write('error due to relative protocol');
+              res.end();
+              return;
+            }
+            const originResponse = await plugins.smartrequest
+              .request(
+                requestedUrl,
+                {
+                  method: 'GET',
+                  keepAlive: false,
+                  // headers: req.headers,
+                },
+                true
+              )
+              .catch((error) => {
+                logger.log('warn', `the origin request errored for ${requestedUrl}`);
+                res.write(`rendertron encountered an error for ${requestedUrl}`);
+                res.end();
+              });
+            if (!originResponse) {
+              return;
+            }
+            for (const header of Object.keys(originResponse.headers)) {
+              res.setHeader(header, originResponse.headers[header]);
+            }
+            if (originResponse.headers['content-type']?.includes('text/html')) {
+              logger.log('info', `Piping ${requestedUrl} through smartssr.`);
+              res.write(await this.prerenderManager.getPrerenderResultForUrl(requestedUrl));
+              res.end();
+            } else {
+              logger.log('info', `Serving ${requestedUrl} directly.`);
+              for (const headerKey of Object.keys(originResponse.headers)) {
+                console.log(`${headerKey}: ${originResponse.headers[headerKey]}`);
+                res.set(headerKey, originResponse.headers[headerKey]);
+              }
+              originResponse.on('data', (data) => {
+                res.write(data);
+              });
+              originResponse.on('end', () => {
+                res.end();
+              });
+            }
+          })
+        );
+      },
+    });
+    await this.serviceServerInstance.start();
+  }
+
+  public async stop() {
+    this.serviceServerInstance ? await this.serviceServerInstance.stop() : null;
+    this.prerenderManager ? await this.prerenderManager.stop() : null;
+    this.taskManager ? await this.taskManager.stop() : null;
+  }
+}

ts/rendertron.db.ts

@@ -0,0 +1,11 @@
+import * as plugins from './rendertron.plugins.js';
+
+export const db = new plugins.smartdata.SmartdataDb({
+  mongoDbUrl:
+    'mongodb+srv://<username>:<password>@losslessone-main.zee8suk.mongodb.net/myFirstDatabase?retryWrites=true&w=majority',
+  mongoDbName: 'rendertron',
+  mongoDbPass: 'wxW4LBa3sxPjyXGf',
+  mongoDbUser: 'rendertron',
+});
+
+db.init();

ts/rendertron.logging.ts

@@ -0,0 +1,8 @@
+import * as plugins from './rendertron.plugins.js';
+import * as paths from './rendertron.paths.js';
+
+const projectinfo = new plugins.projectinfo.ProjectinfoNpm(paths.packageDir);
+
+import { commitinfo } from './00_commitinfo_data.js';
+
+export const logger = plugins.smartlog.Smartlog.createForCommitinfo(commitinfo);

ts/rendertron.paths.ts

@@ -0,0 +1,3 @@
+import * as plugins from './rendertron.plugins.js';
+
+export const packageDir = plugins.path.join(plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), '../');

ts/rendertron.plugins.ts

@@ -0,0 +1,38 @@
+// node native scope
+import * as path from 'path';
+
+export { path };
+
+// @api.global scope
+import * as typedserver from '@api.global/typedserver';
+
+export { typedserver };
+
+// @push.rocks/projectinfo
+import * as projectinfo from '@push.rocks/projectinfo';
+import * as smartdata from '@push.rocks/smartdata';
+import * as smartdelay from '@push.rocks/smartdelay';
+import * as smartlog from '@push.rocks/smartlog';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrequest from '@push.rocks/smartrequest';
+import * as smartrobots from '@push.rocks/smartrobots';
+import * as smartsitemap from '@push.rocks/smartsitemap';
+import * as smartssr from '@push.rocks/smartssr';
+import * as smarttime from '@push.rocks/smarttime';
+import * as taskbuffer from '@push.rocks/taskbuffer';
+
+export {
+  projectinfo,
+  smartdata,
+  smartdelay,
+  smartlog,
+  smartpath,
+  smartpromise,
+  smartrequest,
+  smartrobots,
+  smartsitemap,
+  smartssr,
+  smarttime,
+  taskbuffer,
+};

ts/rendertron.taskmanager.ts

@@ -0,0 +1,60 @@
+import { logger } from './rendertron.logging.js';
+import { Rendertron } from './rendertron.classes.rendertron.js';
+import * as plugins from './rendertron.plugins.js';
+
+export class TaskManager {
+  rendertronRef: Rendertron;
+  public taskmanager: plugins.taskbuffer.TaskManager;
+
+  constructor(rendertronRefArg: Rendertron) {
+    this.rendertronRef = rendertronRefArg;
+    this.taskmanager = new plugins.taskbuffer.TaskManager();
+    this.taskmanager.addAndScheduleTask(
+      new plugins.taskbuffer.Task({
+        name: 'prerenderLocalDomains',
+        taskFunction: async () => {
+          logger.log('info', `starting domain prerender in 5 seconds`);
+          await plugins.smartdelay.delayFor(5000);
+          // get projects from lele-pubapiclient
+          const localDomains = []; // TODO: get from coreflow
+          for (const project of localDomains) {
+            logger.log('info', `Prerending project ${project.name} with url ${project.url}`);
+            const startTime = Date.now();
+            await this.rendertronRef.prerenderManager.prerenderDomain(
+              project.url.replace('https://', '')
+            );
+            logger.log(
+              'info',
+              `Prerended project ${project.name} with url ${project.url} in ${
+                Date.now() - startTime
+              }ms`
+            );
+          }
+        },
+      }),
+      '0 */30 * * * *'
+    );
+    this.taskmanager.addAndScheduleTask(
+      new plugins.taskbuffer.Task({
+        name: 'CleanupPrerenderResults',
+        taskFunction: async () => {
+          logger.log('info', `starting to delete old PrerenderResults in 5 seconds`);
+          await plugins.smartdelay.delayFor(2000);
+          await this.rendertronRef.prerenderManager.cleanPrerenderResults();
+          logger.log('success', `cleaned old prerender results`);
+        },
+      }),
+      '0 0 1 * * *'
+    );
+  }
+
+  public async start() {
+    this.taskmanager.start();
+    logger.log('info', 'triggering initial prerender task outside of schedule');
+    this.taskmanager.triggerTaskByName('prerenderLocalDomains');
+  }
+
+  public async stop() {
+    this.taskmanager.stop();
+  }
+}