34 lines
708 B
TypeScript
34 lines
708 B
TypeScript
|
import * as plugins from '../typedserver.plugins.js';
|
||
|
|
import { Server } from './classes.server.js';
|
||
|
|
import { Handler } from './classes.handler.js';
|
||
|
|
|
||
|
|
export const setupRobots = async (smartexpressRefArg: Server, domainArg: string) => {
|
||
|
|
smartexpressRefArg.addRouteBefore(
|
||
|
|
'/robots.txt',
|
||
|
|
new Handler('GET', async (req, res) => {
|
||
|
|
res.type('text/plain');
|
||
|
|
res.send(`
|
||
|
|
User-agent: Googlebot-News
|
||
|
|
Disallow: /account
|
||
|
|
Disallow: /login
|
||
|
|
|
||
|
|
User-agent: *
|
||
|
|
Disallow: /account
|
||
|
|
Disallow: /login
|
||
|
|
|
||
|
|
${
|
||
|
|
smartexpressRefArg.options.blockWaybackMachine
|
||
|
|
? `
|
||
|
|
User-Agent: ia_archiver
|
||
|
|
Disallow: /
|
||
|
|
`
|
||
|
|
: ``
|
||
|
|
}
|
||
|
|
|
||
|
|
Sitemap: https://${domainArg}/sitemap
|
||
|
|
Sitemap: https://${domainArg}/sitemap-news
|
||
|
|
`);
|
||
|
|
})
|
||
|
|
);
|
||
|
|
};
|