This commit is contained in:
2025-01-01 07:33:33 +01:00
commit 396cfe70de
25 changed files with 10518 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
node_modules/
+20
View File
@@ -0,0 +1,20 @@
.nogit/
# artifacts
coverage/
public/
pages/
# installs
node_modules/
# caches
.yarn/
.cache/
.rpt2_cache
# builds
dist/
dist_*/
# custom
+11
View File
@@ -0,0 +1,11 @@
{
"version": "0.2.0",
"configurations": [
{
"command": "npm test",
"name": "Run npm test",
"request": "launch",
"type": "node-terminal"
}
]
}
+26
View File
@@ -0,0 +1,26 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"schema": {
"type": "object",
"properties": {
"npmci": {
"type": "object",
"description": "settings for npmci"
},
"gitzone": {
"type": "object",
"description": "settings for gitzone",
"properties": {
"projectType": {
"type": "string",
"enum": ["website", "element", "service", "npm", "wcc"]
}
}
}
}
}
}
]
}
+35
View File
@@ -0,0 +1,35 @@
# gitzone dockerfile_service
## STAGE 1 // BUILD
FROM registry.gitlab.com/hosttoday/ht-docker-node:npmci as node1
COPY ./ /app
WORKDIR /app
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN rm -rf node_modules && npm install
RUN npm run build
# gitzone dockerfile_service
## STAGE 2 // install production
FROM registry.gitlab.com/hosttoday/ht-docker-node:npmci as node2
WORKDIR /app
COPY --from=node1 /app /app
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN rm -r node_modules/ && npm install --production
### Healthchecks
RUN npm install -g @servezone/healthy
HEALTHCHECK --interval=30s --timeout=30s --start-period=30s --retries=3 CMD [ "healthy" ]
EXPOSE 80
# Add Tini
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini
ENTRYPOINT ["/tini", "--"]
# CMD
CMD ["npm", "start"]
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env node
process.env.CLI_CALL = 'true';
import * as cliTool from './ts/index.js';
cliTool.runCli();
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env node
process.env.CLI_CALL = 'true';
const cliTool = await import('./dist_ts/index.js');
cliTool.runCli();
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env node
process.env.CLI_CALL = 'true';
import * as tsrun from '@git.zone/tsrun';
tsrun.runPath('./cli.child.js', import.meta.url);
+19
View File
@@ -0,0 +1,19 @@
Copyright (c) 2016 Task Venture Capital GmbH (hello@task.vc)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+23
View File
@@ -0,0 +1,23 @@
{
"npmci": {
"npmGlobalTools": [],
"dockerRegistryRepoMap": {
"code.foss.global": "serve.zone/corerender"
},
"dockerBuildargEnvMap": {
"NPMCI_TOKEN_NPM2": "NPMCI_TOKEN_NPM2"
},
"npmRegistryUrl": "verdaccio.lossless.digital"
},
"gitzone": {
"projectType": "service",
"module": {
"githost": "code.foss.global",
"gitscope": "serve.zone",
"gitrepo": "corerender",
"description": "a rendering service for serve.zone that preserves styles for web components",
"npmPackagename": "@serve.zone/corerender",
"license": "MIT"
}
}
}
+57
View File
@@ -0,0 +1,57 @@
{
"name": "corerender",
"version": "2.0.61",
"description": "a rendering service for serve.zone that preserves styles for web components",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"author": "Lossless GmbH",
"license": "UNLICENSED",
"scripts": {
"test": "(tstest test/)",
"start": "(node --max_old_space_size=200 ./cli.js)",
"startTs": "(node cli.ts.js)",
"watch": "tswatch service",
"build": "(tsbuild --web --allowimplicitany)",
"buildDocs": "tsdoc"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.1.66",
"@git.zone/tsrun": "^1.2.44",
"@git.zone/tstest": "^1.0.77",
"@git.zone/tswatch": "^2.0.7",
"@push.rocks/tapbundle": "^5.0.8"
},
"dependencies": {
"@api.global/typedserver": "^3.0.53",
"@push.rocks/projectinfo": "^5.0.1",
"@push.rocks/smartdata": "^5.0.14",
"@push.rocks/smartdelay": "^3.0.1",
"@push.rocks/smartlog": "^3.0.7",
"@push.rocks/smartpath": "^5.0.5",
"@push.rocks/smartpromise": "^4.0.2",
"@push.rocks/smartrequest": "^2.0.15",
"@push.rocks/smartrobots": "^1.0.2",
"@push.rocks/smartsitemap": "^2.0.1",
"@push.rocks/smartssr": "^1.0.40",
"@push.rocks/smartstate": "^2.0.6",
"@push.rocks/smarttime": "^4.0.1",
"@push.rocks/taskbuffer": "^3.0.10"
},
"private": true,
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
"readme.md"
],
"browserslist": [
"last 1 chrome versions"
],
"type": "module"
}
+9816
View File
File diff suppressed because it is too large Load Diff
+32
View File
@@ -0,0 +1,32 @@
# @losslessone/services/servezone/rendertron
a rendering service for lossless gmbh
## Availabililty and Links
* [npmjs.org (npm package)](https://www.npmjs.com/package/@losslessone_private/rendertron)
* [gitlab.com (source)](https://gitlab.com/losslessone/services/servezone/rendertron)
* [github.com (source mirror)](https://github.com/losslessone/services/servezone/rendertron)
* [docs (typedoc)](https://losslessone/services/servezone.gitlab.io/rendertron/)
## Status for master
Status Category | Status Badge
-- | --
GitLab Pipelines | [![pipeline status](https://gitlab.com/losslessone/services/servezone/rendertron/badges/master/pipeline.svg)](https://lossless.cloud)
GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/losslessone/services/servezone/rendertron/badges/master/coverage.svg)](https://lossless.cloud)
npm | [![npm downloads per month](https://badgen.net/npm/dy/@losslessone_private/rendertron)](https://lossless.cloud)
Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/losslessone/services/servezone/rendertron)](https://lossless.cloud)
TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@losslessone_private/rendertron)](https://lossless.cloud)
PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@losslessone_private/rendertron)](https://lossless.cloud)
BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@losslessone_private/rendertron)](https://lossless.cloud)
## Usage
Use TypeScript for best in class intellisense.
For further information read the linked docs at the top of this readme.
## Legal
> UNLICENSED licensed | **©** [Task Venture Capital GmbH](https://task.vc)
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
+20
View File
@@ -0,0 +1,20 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as rendertron from '../ts/index.js';
let testRendertron: rendertron.Rendertron
tap.test('should start rendertron', async (tools) => {
testRendertron = new rendertron.Rendertron();
await testRendertron.start();
});
tap.test('should prerender a page', async () => {
await testRendertron.prerenderManager.getPrerenderResultForUrl('https://lossless.com')
})
tap.test('stop rendertron', async () => {
await testRendertron.stop();
});
tap.start();
+8
View File
@@ -0,0 +1,8 @@
/**
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: 'rendertron',
version: '2.0.61',
description: 'a rendering service for lossless gmbh'
}
+22
View File
@@ -0,0 +1,22 @@
import { logger } from './rendertron.logging.js';
import { db } from './rendertron.db.js';
import { Rendertron } from './rendertron.classes.rendertron.js';
export {
Rendertron
}
let rendertronInstance: Rendertron;
export const runCli = async () => {
logger.log('info', `Starting rendertron...`);
rendertronInstance = new Rendertron();
rendertronInstance.start();
logger.log('success', `Successfully started rendertron!`);
};
export const stop = async () => {
if (rendertronInstance) {
rendertronInstance.stop();
}
db.close();
};
+97
View File
@@ -0,0 +1,97 @@
import { logger } from './rendertron.logging.js';
import { PrerenderResult } from './rendertron.classes.prerenderresult.js';
import * as plugins from './rendertron.plugins.js';
export class PrerenderManager {
public smartssrInstance: plugins.smartssr.SmartSSR;
public smartrobotsInstance: plugins.smartrobots.Smartrobots;
public smartsitemapInstance: plugins.smartsitemap.SmartSitemap;
constructor() {}
/**
* starts the manager
*/
public async start() {
this.smartssrInstance = new plugins.smartssr.SmartSSR();
this.smartrobotsInstance = new plugins.smartrobots.Smartrobots();
this.smartsitemapInstance = new plugins.smartsitemap.SmartSitemap();
}
/**
* stops the manager
*/
public async stop() {}
public async getPrerenderResultForUrl(urlArg: string): Promise<string> {
const done = plugins.smartpromise.defer<string>();
const prerenderResult = await PrerenderResult.getPrerenderResultForUrl(this, urlArg).catch(
() => {
done.resolve(`Cannot render ${urlArg} due to internal error.`);
return null;
}
);
done.resolve(prerenderResult.renderResultString);
return done.promise;
}
/**
* prerenders a sitemap
*/
public async prerenderDomain(domainArg: string) {
logger.log('info', `prerendering domain: ${domainArg}`);
await this.getPrerenderResultForUrl(`https://${domainArg}/`).catch((err) => {
logger.log('error', `failed to prerender ${domainArg}`);
});
await this.getPrerenderResultForUrl(`https://${domainArg}`).catch((err) => {
logger.log('error', `failed to prerender ${domainArg}`);
});
const robotsTxt = await this.smartrobotsInstance
.parseRobotsTxtFromUrl(`https://${domainArg}/robots.txt`)
.catch((err) => {
logger.log('warn', `no robots for ${domainArg}`);
});
if (!robotsTxt) {
return;
}
if (robotsTxt.sitemaps.length === 0) {
logger.log('warn', `robot-txt for ${domainArg} does bot sepcify any sitemaps`);
}
for (const sitemapUrl of robotsTxt.sitemaps) {
await this.prerenderSitemap(sitemapUrl);
}
}
public async prerenderSitemap(sitemapUrlArg: string) {
logger.log('info', `prerendering sitemap: ${sitemapUrlArg}`);
const parsedSitemap = await this.smartsitemapInstance.parseSitemapUrl(sitemapUrlArg);
if (!parsedSitemap.urlset?.url) {
return;
}
if (!(parsedSitemap.urlset.url instanceof Array)) {
await this.getPrerenderResultForUrl(parsedSitemap.urlset.url.loc);
} else {
for (const url of parsedSitemap.urlset.url) {
if (!url?.loc) {
continue;
}
await this.getPrerenderResultForUrl(url.loc);
}
}
}
public async cleanPrerenderResults() {
const allPrerenderResults = await PrerenderResult.getInstances<PrerenderResult>({});
for (const prerenderResult of allPrerenderResults) {
const extendedDate = plugins.smarttime.ExtendedDate.fromMillis(prerenderResult.timestamp);
const stillValid = extendedDate.lessTimePassedToNow({ hours: 24 });
if (!stillValid) {
logger.log(
'warn',
`deleted prerender result for ${prerenderResult.url} since it is older than 24 hours`
);
await prerenderResult.delete();
}
}
}
}
+94
View File
@@ -0,0 +1,94 @@
import * as plugins from './rendertron.plugins.js';
import { db } from './rendertron.db.js';
import { PrerenderManager } from './rendertron.classes.prerendermanager.js';
import { logger } from './rendertron.logging.js';
/**
* allows for prerendering results
*/
@plugins.smartdata.Collection(() => {
return db;
})
export class PrerenderResult extends plugins.smartdata.SmartDataDbDoc<
PrerenderResult,
PrerenderResult
> {
// STATIC
public static async getPrerenderResultForUrl(
managerArg: PrerenderManager,
urlArg: string,
forceNew: boolean = false
): Promise<PrerenderResult> {
let prerenderResult = await PrerenderResult.getInstance<PrerenderResult>({
url: urlArg,
});
if (prerenderResult) {
const prerenderResultDate = new plugins.smarttime.ExtendedDate(prerenderResult.timestamp);
if (
prerenderResultDate.lessTimePassedToNow({
hours: 12,
})
) {
logger.log('info', `Serving prerendered result for ${prerenderResult.url}`);
return prerenderResult;
} else {
logger.log(
'info',
`Outdated Prerender Result: Requesting newer result for ${prerenderResult.url}`
);
prerenderResult.needsRerendering = true;
}
}
if (!prerenderResult || prerenderResult.needsRerendering) {
const newPrerenderResult: PrerenderResult = await PrerenderResult.createPrerenderResultForUrl(
managerArg,
urlArg
).catch((err) => {
return prerenderResult;
});
prerenderResult = newPrerenderResult;
}
return prerenderResult;
}
private static async createPrerenderResultForUrl(
managerArg: PrerenderManager,
urlArg: string
): Promise<PrerenderResult> {
const renderedResultPromise = managerArg.smartssrInstance.renderPage(urlArg).catch(() => {
const errorMessage = `failed to render ${urlArg}`;
logger.log('error', errorMessage);
});
let prerenderResult = await PrerenderResult.getInstance<PrerenderResult>({
url: urlArg,
});
prerenderResult = prerenderResult || new PrerenderResult();
prerenderResult.url = urlArg;
prerenderResult.timestamp = Date.now();
prerenderResult.renderResultString = (await renderedResultPromise) || 'error';
prerenderResult.needsRerendering = false;
await prerenderResult.save();
return prerenderResult;
}
// INSTANCE
@plugins.smartdata.unI()
url: string;
@plugins.smartdata.svDb()
renderResultString: string;
@plugins.smartdata.svDb()
timestamp: number;
@plugins.smartdata.svDb()
needsRerendering: boolean = false;
constructor() {
super();
}
}
+90
View File
@@ -0,0 +1,90 @@
import * as plugins from './rendertron.plugins.js';
import * as paths from './rendertron.paths.js';
import { logger } from './rendertron.logging.js';
import { PrerenderResult } from './rendertron.classes.prerenderresult.js';
import { PrerenderManager } from './rendertron.classes.prerendermanager.js';
import { TaskManager } from './rendertron.taskmanager.js';
export class Rendertron {
public projectinfo: plugins.projectinfo.ProjectInfo;
public serviceServerInstance: plugins.typedserver.utilityservers.UtilityServiceServer;
public prerenderManager: PrerenderManager;
public taskManager: TaskManager;
/**
* starts the financeflow instance
*/
public async start() {
this.projectinfo = new plugins.projectinfo.ProjectInfo(paths.packageDir);
this.prerenderManager = new PrerenderManager();
this.taskManager = new TaskManager(this);
await this.prerenderManager.start();
await this.taskManager.start();
this.serviceServerInstance = new plugins.typedserver.utilityservers.UtilityServiceServer({
serviceDomain: 'rendertron.lossless.one',
serviceName: 'rendertron',
serviceVersion: this.projectinfo.npm.version,
addCustomRoutes: async (serverArg) => {
serverArg.addRoute(
'/render/*',
new plugins.typedserver.servertools.Handler('GET', async (req, res) => {
const requestedUrl = req.url.replace('/render/', '');
logger.log('info', `Got SSR request for ${requestedUrl}`);
if (requestedUrl.startsWith('https://url(')) {
logger.log('warn', `relative url error for ${requestedUrl}`);
res.status(500);
res.write('error due to relative protocol');
res.end();
return;
}
const originResponse = await plugins.smartrequest
.request(
requestedUrl,
{
method: 'GET',
keepAlive: false,
// headers: req.headers,
},
true
)
.catch((error) => {
logger.log('warn', `the origin request errored for ${requestedUrl}`);
res.write(`rendertron encountered an error for ${requestedUrl}`);
res.end();
});
if (!originResponse) {
return;
}
for (const header of Object.keys(originResponse.headers)) {
res.setHeader(header, originResponse.headers[header]);
}
if (originResponse.headers['content-type']?.includes('text/html')) {
logger.log('info', `Piping ${requestedUrl} through smartssr.`);
res.write(await this.prerenderManager.getPrerenderResultForUrl(requestedUrl));
res.end();
} else {
logger.log('info', `Serving ${requestedUrl} directly.`);
for (const headerKey of Object.keys(originResponse.headers)) {
console.log(`${headerKey}: ${originResponse.headers[headerKey]}`);
res.set(headerKey, originResponse.headers[headerKey]);
}
originResponse.on('data', (data) => {
res.write(data);
});
originResponse.on('end', () => {
res.end();
});
}
})
);
},
});
await this.serviceServerInstance.start();
}
public async stop() {
this.serviceServerInstance ? await this.serviceServerInstance.stop() : null;
this.prerenderManager ? await this.prerenderManager.stop() : null;
this.taskManager ? await this.taskManager.stop() : null;
}
}
+11
View File
@@ -0,0 +1,11 @@
import * as plugins from './rendertron.plugins.js';
export const db = new plugins.smartdata.SmartdataDb({
mongoDbUrl:
'mongodb+srv://<username>:<password>@losslessone-main.zee8suk.mongodb.net/myFirstDatabase?retryWrites=true&w=majority',
mongoDbName: 'rendertron',
mongoDbPass: 'wxW4LBa3sxPjyXGf',
mongoDbUser: 'rendertron',
});
db.init();
+8
View File
@@ -0,0 +1,8 @@
import * as plugins from './rendertron.plugins.js';
import * as paths from './rendertron.paths.js';
const projectinfo = new plugins.projectinfo.ProjectinfoNpm(paths.packageDir);
import { commitinfo } from './00_commitinfo_data.js';
export const logger = plugins.smartlog.Smartlog.createForCommitinfo(commitinfo);
+3
View File
@@ -0,0 +1,3 @@
import * as plugins from './rendertron.plugins.js';
export const packageDir = plugins.path.join(plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), '../');
+38
View File
@@ -0,0 +1,38 @@
// node native scope
import * as path from 'path';
export { path };
// @api.global scope
import * as typedserver from '@api.global/typedserver';
export { typedserver };
// @push.rocks/projectinfo
import * as projectinfo from '@push.rocks/projectinfo';
import * as smartdata from '@push.rocks/smartdata';
import * as smartdelay from '@push.rocks/smartdelay';
import * as smartlog from '@push.rocks/smartlog';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartrobots from '@push.rocks/smartrobots';
import * as smartsitemap from '@push.rocks/smartsitemap';
import * as smartssr from '@push.rocks/smartssr';
import * as smarttime from '@push.rocks/smarttime';
import * as taskbuffer from '@push.rocks/taskbuffer';
export {
projectinfo,
smartdata,
smartdelay,
smartlog,
smartpath,
smartpromise,
smartrequest,
smartrobots,
smartsitemap,
smartssr,
smarttime,
taskbuffer,
};
+60
View File
@@ -0,0 +1,60 @@
import { logger } from './rendertron.logging.js';
import { Rendertron } from './rendertron.classes.rendertron.js';
import * as plugins from './rendertron.plugins.js';
export class TaskManager {
rendertronRef: Rendertron;
public taskmanager: plugins.taskbuffer.TaskManager;
constructor(rendertronRefArg: Rendertron) {
this.rendertronRef = rendertronRefArg;
this.taskmanager = new plugins.taskbuffer.TaskManager();
this.taskmanager.addAndScheduleTask(
new plugins.taskbuffer.Task({
name: 'prerenderLocalDomains',
taskFunction: async () => {
logger.log('info', `starting domain prerender in 5 seconds`);
await plugins.smartdelay.delayFor(5000);
// get projects from lele-pubapiclient
const localDomains = []; // TODO: get from coreflow
for (const project of localDomains) {
logger.log('info', `Prerending project ${project.name} with url ${project.url}`);
const startTime = Date.now();
await this.rendertronRef.prerenderManager.prerenderDomain(
project.url.replace('https://', '')
);
logger.log(
'info',
`Prerended project ${project.name} with url ${project.url} in ${
Date.now() - startTime
}ms`
);
}
},
}),
'0 */30 * * * *'
);
this.taskmanager.addAndScheduleTask(
new plugins.taskbuffer.Task({
name: 'CleanupPrerenderResults',
taskFunction: async () => {
logger.log('info', `starting to delete old PrerenderResults in 5 seconds`);
await plugins.smartdelay.delayFor(2000);
await this.rendertronRef.prerenderManager.cleanPrerenderResults();
logger.log('success', `cleaned old prerender results`);
},
}),
'0 0 1 * * *'
);
}
public async start() {
this.taskmanager.start();
logger.log('info', 'triggering initial prerender task outside of schedule');
this.taskmanager.triggerTaskByName('prerenderLocalDomains');
}
public async stop() {
this.taskmanager.stop();
}
}
+14
View File
@@ -0,0 +1,14 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"useDefineForClassFields": false,
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
},
"exclude": [
"dist_*/**/*.d.ts"
]
}