import * as plugins from './smartrobots.plugins';

export class Smartrobots {
  private webrequestInstance = new plugins.webrequest.WebRequest();

  public async parseRobotsTxtFromUrl(urlArg: string) {
    const robotsTxtString = await (
      await this.webrequestInstance.request(urlArg, {
        method: 'GET',
      })
    ).text();
    const parsedResult = this.parseRobotsTxt(robotsTxtString);
    return parsedResult;
  }

  public async parseRobotsTxt(robotsTxt: string) {
    const lineArray = robotsTxt.split('\n');
    return {
      sitemaps: lineArray
        .filter((lineArg) => lineArg.startsWith('Sitemap: '))
        .map((lineArg) => lineArg.replace('Sitemap: ', '')),
    };
  }
}