skroutzi-api/routes/crawler.js

let link = "https://www.skroutz.gr/c/";

const https = require('https');
const cheerio = require('cheerio')
const db = require('../db');


class Crawler {
	constructor(){ 
	}

	getPage(code, category) {
		return new Promise((res, rej) => {
			https.get(link + category + ".html?from=catspan&keyphrase=" + code, (resp) => {
			  let data = '';

			  // A chunk of data has been recieved.
			  resp.on('data', (chunk) => {
			    data += chunk;
			  });

			  // The whole response has been received. Print out the result.
			  resp.on('end', () => {
			    res(data);
			  });

			}).on("error", (err) => {
				console.log("Error: " + err.message);
				rej(err);
			});		
		})
	}

	findPosition(data, player = "Toner Sales") {
		return new Promise((res,rej) => {
			const $ = cheerio.load(data);
			let items = $('li.cf.card.click-to-shop.js-product-card');
			let len = items.length;
			console.log(data);
			for (var i=0;i < len; i++) {
				let txt = $(items.get(i)).find('a.shop.icon.js-shop-info-link').text();
				let price = $(items.get(i)).find('a.sku-link.js-sku-link').text();
				console.log(txt + '-' + player);
				if(txt == player) {
					res({
						position: i+1, 
						price: price
					});
					return;
				}
			}
			rej("[!]Not found");
		})
	}

	newWithData(name, enemy, data) {
		return {
			data: {
				[enemy]: data 
			},
			price: 0.0,
			position: 0,
			enemies: [enemy]
		}
	}

	update(name, enemy, data) {
		return db.get(name).then((item) => {
			item.data[enemy] = data;
			if(data.position < item.position) { 
				item.alert = 1;
			}
			item.lastDate = new Date().toISOString();
			return db.put(name, item);
		})
		.catch((e) => {
			console.log("Error");
			console.log(e);
			db.put(name, this.newWithData(name, enemy, data));
		});
	}

	updateItem(name, data) {
		return db.get(name).then((item) => {
			item.position = data.position;
			item.price = data.price;
			item.alert = 0;
			item.lastDate = (new Date().toISOString());
			return db.put(name, item).then(() => console.log(item));
		});
	}

	collect(name, value) {
		console.log("Item: " + name + " - " + value.category)
		return this.getPage(name, value.category)
		.then((page) => {
			let parr = [];
			let start = Promise.resolve([]);
			start = start.then(() => {
							return this.findPosition(page)
						})
						.then((data) => {
							return this.updateItem(name, data)
						});
			for(var i in value.enemies){
			 let temp = value.enemies[i];
			 	start = start
			 	.then(() => this.findPosition(page, temp))
			 	.then((newdata) => this.update(name, temp, newdata));
			}
			return start;
		}).catch((e) => console.log("Error" + e));
	}

}


module.exports = new Crawler();