34 lines
696 B
TypeScript
34 lines
696 B
TypeScript
import * as plugins from '../plugins.js';
|
|
import { Server } from './classes.server.js';
|
|
import { Handler } from './classes.handler.js';
|
|
|
|
export const setupRobots = async (smartexpressRefArg: Server, domainArg: string) => {
|
|
smartexpressRefArg.addRouteBefore(
|
|
'/robots.txt',
|
|
new Handler('GET', async (req, res) => {
|
|
res.type('text/plain');
|
|
res.send(`
|
|
User-agent: Googlebot-News
|
|
Disallow: /account
|
|
Disallow: /login
|
|
|
|
User-agent: *
|
|
Disallow: /account
|
|
Disallow: /login
|
|
|
|
${
|
|
smartexpressRefArg.options.blockWaybackMachine
|
|
? `
|
|
User-Agent: ia_archiver
|
|
Disallow: /
|
|
`
|
|
: ``
|
|
}
|
|
|
|
Sitemap: https://${domainArg}/sitemap
|
|
Sitemap: https://${domainArg}/sitemap-news
|
|
`);
|
|
})
|
|
);
|
|
};
|