fix(core): update

ts/servertools/tools.robots.ts

@@ -0,0 +1,33 @@
+import * as plugins from '../typedserver.plugins.js';
+import { Server } from './classes.server.js';
+import { Handler } from './classes.handler.js';
+
+export const setupRobots = async (smartexpressRefArg: Server, domainArg: string) => {
+  smartexpressRefArg.addRouteBefore(
+    '/robots.txt',
+    new Handler('GET', async (req, res) => {
+      res.type('text/plain');
+      res.send(`
+User-agent: Googlebot-News
+Disallow: /account
+Disallow: /login
+
+User-agent: *
+Disallow: /account
+Disallow: /login
+
+${
+  smartexpressRefArg.options.blockWaybackMachine
+    ? `
+User-Agent: ia_archiver
+Disallow: /
+`
+    : ``
+}
+
+Sitemap: https://${domainArg}/sitemap
+Sitemap: https://${domainArg}/sitemap-news
+`);
+    })
+  );
+};