25 lines
702 B
TypeScript
25 lines
702 B
TypeScript
import * as plugins from './smartrobots.plugins';
|
|
|
|
export class Smartrobots {
|
|
private webrequestInstance = new plugins.webrequest.WebRequest();
|
|
|
|
public async parseRobotsTxtFromUrl(urlArg: string) {
|
|
const robotsTxtString = await (
|
|
await this.webrequestInstance.request(urlArg, {
|
|
method: 'GET',
|
|
})
|
|
).text();
|
|
const parsedResult = this.parseRobotsTxt(robotsTxtString);
|
|
return parsedResult;
|
|
}
|
|
|
|
public async parseRobotsTxt(robotsTxt: string) {
|
|
const lineArray = robotsTxt.split('\n');
|
|
return {
|
|
sitemaps: lineArray
|
|
.filter((lineArg) => lineArg.startsWith('Sitemap: '))
|
|
.map((lineArg) => lineArg.replace('Sitemap: ', '')),
|
|
};
|
|
}
|
|
}
|