let link = "https://www.skroutz.gr/c/"; const https = require('https'); const cheerio = require('cheerio') const db = require('../db'); class Crawler { constructor(){ } getPage(code, category) { return new Promise((res, rej) => { https.get(link + category + ".html?from=catspan&keyphrase=" + code, (resp) => { let data = ''; // A chunk of data has been recieved. resp.on('data', (chunk) => { data += chunk; }); // The whole response has been received. Print out the result. resp.on('end', () => { res(data); }); }).on("error", (err) => { console.log("Error: " + err.message); rej(err); }); }) } findPosition(data, player = "Toner Sales") { return new Promise((res,rej) => { const $ = cheerio.load(data); let items = $('li.cf.card.click-to-shop.js-product-card'); let len = items.length; console.log(data); for (var i=0;i < len; i++) { let txt = $(items.get(i)).find('a.shop.icon.js-shop-info-link').text(); let price = $(items.get(i)).find('a.sku-link.js-sku-link').text(); console.log(txt + '-' + player); if(txt == player) { res({ position: i+1, price: price }); return; } } rej("[!]Not found"); }) } newWithData(name, enemy, data) { return { data: { [enemy]: data }, price: 0.0, position: 0, enemies: [enemy] } } update(name, enemy, data) { return db.get(name).then((item) => { item.data[enemy] = data; if(data.position < item.position) { item.alert = 1; } item.lastDate = new Date().toISOString(); return db.put(name, item); }) .catch((e) => { console.log("Error"); console.log(e); db.put(name, this.newWithData(name, enemy, data)); }); } updateItem(name, data) { return db.get(name).then((item) => { item.position = data.position; item.price = data.price; item.alert = 0; item.lastDate = (new Date().toISOString()); return db.put(name, item).then(() => console.log(item)); }); } collect(name, value) { console.log("Item: " + name + " - " + value.category) return this.getPage(name, value.category) .then((page) => { let parr = []; let start = Promise.resolve([]); start = start.then(() => { return this.findPosition(page) }) .then((data) => { return this.updateItem(name, data) }); for(var i in value.enemies){ let temp = value.enemies[i]; start = start .then(() => this.findPosition(page, temp)) .then((newdata) => this.update(name, temp, newdata)); } return start; }).catch((e) => console.log("Error" + e)); } } module.exports = new Crawler();