fix(core): update

ts/index.ts

@@ -0,0 +1,24 @@
+import * as plugins from './smartrobots.plugins';
+
+export class Smartrobots {
+  private webrequestInstance = new plugins.webrequest.WebRequest();
+
+  public async parseRobotsTxtFromUrl(urlArg: string) {
+    const robotsTxtString = await (
+      await this.webrequestInstance.request(urlArg, {
+        method: 'GET',
+      })
+    ).text();
+    const parsedResult = this.parseRobotsTxt(robotsTxtString);
+    return parsedResult;
+  }
+
+  public async parseRobotsTxt(robotsTxt: string) {
+    const lineArray = robotsTxt.split('\n');
+    return {
+      sitemaps: lineArray
+        .filter((lineArg) => lineArg.startsWith('Sitemap: '))
+        .map((lineArg) => lineArg.replace('Sitemap: ', '')),
+    };
+  }
+}

ts/smartrobots.plugins.ts

@@ -0,0 +1,6 @@
+// @pushrocks scope
+import * as webrequest from '@pushrocks/webrequest';
+
+export {
+  webrequest
+}