import * as plugins from '../plugins.js'; import { Server } from './classes.server.js'; import { Handler } from './classes.handler.js'; export const setupRobots = async (smartexpressRefArg: Server, domainArg: string) => { smartexpressRefArg.addRouteBefore( '/robots.txt', new Handler('GET', async (req, res) => { res.type('text/plain'); res.send(` User-agent: Googlebot-News Disallow: /account Disallow: /login User-agent: * Disallow: /account Disallow: /login ${ smartexpressRefArg.options.blockWaybackMachine ? ` User-Agent: ia_archiver Disallow: / ` : `` } Sitemap: https://${domainArg}/sitemap Sitemap: https://${domainArg}/sitemap-news `); }) ); };