+ foo bar
+
+```
+```
+{
+ "name": "myText",
+ "elem": ".text",
+ "data": "text"
+}
+```
+Result : `{ myText: 'foo bar' }`
+Note: This is default
+
+###Element content, include tags
+
+```
+
+ foo bar
+
+```
+```
+{
+ "name": "myHtml",
+ "elem": ".html",
+ "data": "html"
+}
+```
+Result : `{ myHtml: 'foo bar' }`
+
+###Element content, only text nodes
+
+```
+
+ foo bar
+
+```
+```
+{
+ "name": "myTextNode",
+ "elem": ".text-node",
+ "data": "txtn"
+}
+```
+Result : `{ myTextNode: 'bar' }`
+
+###Element attribute(s)
+
+```
+
+```
+```
+{
+ "name": "myAttr",
+ "elem": ".attr",
+ "data": [ "attr", "src" ]
+}
+```
+Result : `{ myAttr: 'http://foo.bar/img.jpg' }`
+
+Note: *attr* also supports more arguments, returning an array (see *data* below)
+
+###Element data field(s)
+
+```
+
+```
+```
+{
+ "name": "myData",
+ "elem": ".data",
+ "data": [ "data", "small", "big" ]
+}
+```
+Result : `{ myData: [ 'small.jpg', 'big.jpg' ] }`
+
+Note: *data* also supports one argument, returning a string (see *attr* above)
+
+###Special case
+Sometimes a node has content you want as well as being a container! Take a look at this scenario:
+```
+
+ Click!
+ ..
+
+```
+```
+{
+ "name": "myItem",
+ "elem": ".item",
+ "data": "object",
+ "kids": [
+ {
+ "name": "title",
+ "data": [ "attr", "meta-title" ]
+ },
+ {
+ "name": "url",
+ "elem": "a",
+ "data": [ "attr", "href" ]
+ },
+ ..
+ ]
+}
+```
+Result : `{ myItem: { title: 'foobar', url: 'http://foo.bar' } }`
+Note: The first child has no *elem* defined, so it uses the parent element.
+
+#Parser
+
+###Custom tasks
+You can inject your own custom tasks or override those included in the core.
+```
+// my-tasks.js
+export { _myTask as myTask }
+..
+function _myTask(args, value) {
+ return 'foobar';
+}
+..
+```
+```
+// app.js
+import { Parser } from "edderkopp";
+import tasks from './my-tasks';
+
+// Inject all tasks defined in my-tasks.js
+Parser.injectTasks(tasks);
+
+// Or you can inject an anonymous task
+Parser.injectTasks({
+ anonTask: function(args, value) {
+ return 'foobar';
+ }
+});
+```
+Note: If you feel your custom task should be included in core feel free to make a pull request or create an issue.
diff --git a/config.js b/config.js
deleted file mode 100644
index ba90420..0000000
--- a/config.js
+++ /dev/null
@@ -1,88 +0,0 @@
-var fs = require('fs');
-var URI = require('URIjs');
-var log = require('./log');
-
-var _config = './config';
-var _shops;
-
-var Config = function() {}
-
-// Inject config path
-Config.prototype.setPath = function(path) {
- if (fs.existsSync(path)) {
- _config = path;
- } else {
- throw new Error('Path "' + path + '" don\'t exist');
- }
-}
-
-// Load all shops
-Config.prototype.getShops = function() {
- if (!_shops) {
- log.verbose('[Config] Get config for all shops');
- var files = getFiles(_config + '/sites/shops');
- var _shops = {};
- for (var i = 0; i < files.length; i++) {
- var obj = JSON.parse(fs.readFileSync(files[i]).toString());
- _shops[obj.key] = obj;
- _shops[obj.id] = obj;
-
- }
- }
- return _shops;
-}
-
-Config.prototype.getByUrl = function(url) {
- var hostname = new URI(url).hostname();
- var files = getFiles(_config + '/sites');
- for (var i = 0; i < files.length; i++) {
- var obj = JSON.parse(fs.readFileSync(files[i]).toString());
- if (hostname == new URI(obj.url).hostname()) {
- log.verbose('[Config] Get config for ' + obj.name);
- return obj;
- }
- }
- log.error('[Config] Missing config for ' + url);
- return false;
-}
-
-Config.prototype.parseFile = function(file) {
- return JSON.parse(fs.readFileSync(file).toString());
-}
-
-//Config.prototype.load = function(target, key) {
-// log.verbose('[Config] Load ' + target + ': ' + key);
-// init();
-//
-// var file = _config[target] + '/' + key + '.json';
-// if (fs.existsSync(file)) {
-// return this[target][key] = JSON.parse(fs.readFileSync(file).toString());
-// } else {
-// log.error('[Config] Missing ' + file);
-// return false;
-// }
-//}
-
-
-function getContent(file) {
- var obj = JSON.parse(fs.readFileSync(path + '/' + file).toString());
- self[target][obj.key] = obj;
- self[target][obj.id] = obj;
-}
-
-function getFiles(path) {
- var files = [];
- fs.readdirSync(path).forEach(function(file) {
- if (fs.statSync(path + '/' + file).isDirectory()) {
- files = files.concat(getFiles(path + '/' + file));
- } else {
- files.push(path + '/' + file);
- }
- //log.verbose(file);
- //var obj = JSON.parse(fs.readFileSync(path + '/' + file).toString());
- //self[target][obj.key] = obj;
- //self[target][obj.id] = obj;
- });
- return files;
-}
-module.exports = Config;
diff --git a/customHttpConnector.js b/customHttpConnector.js
deleted file mode 100644
index 228ebfb..0000000
--- a/customHttpConnector.js
+++ /dev/null
@@ -1,41 +0,0 @@
-var HttpConnector = require('elasticsearch/src/lib/connectors/http');
-var qs = require('querystring');
-var inherits = require('util').inherits;
-var fs = require('fs');
-
-function CustomHttpConnector(host, config) {
- HttpConnector.call(this, host, config);
-}
-
-inherits(CustomHttpConnector, HttpConnector);
-
-CustomHttpConnector.prototype.makeReqParams = function (params) {
- params = params || {};
- var host = this.host;
-
- var reqParams = {
- method: params.method || 'GET',
- protocol: host.protocol + ':',
- auth: host.auth,
- hostname: host.host,
- port: host.port,
- path: (host.path || '') + (params.path || ''),
- headers: host.getHeaders(params.headers),
- agent: this.agent,
- rejectUnauthorized: true,
- ca: fs.readFileSync('carlsen.crt', 'utf8')
- };
-
- if (!reqParams.path) {
- reqParams.path = '/';
- }
-
- var query = host.getQuery(params.query);
- if (query) {
- reqParams.path = reqParams.path + '?' + qs.stringify(query);
- }
-
- return reqParams;
-};
-
-module.exports = CustomHttpConnector;
diff --git a/download.js b/download.js
deleted file mode 100644
index 3e7fef1..0000000
--- a/download.js
+++ /dev/null
@@ -1,55 +0,0 @@
-var request = require('request');
-var log = require('./log');
-var Promise = require('es6-promise').Promise;
-
-var Download = function() {}
-
-Download.prototype = Object.create(require('events').EventEmitter.prototype);
-
-Download.prototype.get = function(url, obj) {
- //var self = this;
- var t = process.hrtime();
- var options = {
- url: url,
- headers: {
- 'User-Agent': 'request'
- },
- timeout: 60000
- }
- return new Promise(function (fulfill, reject) {
- request(options, function (error, response, html) {
- if (error !== null) {
- if (error.code === 'ETIMEDOUT') {
- log.warn('[Download] Timeout of ' + options.timeout * 0.001 + 's reached for ' + url);
- obj ? fulfill(obj) : fulfill();
- } else {
- log.error('[Download] ' + error.toString() + ' (' + url + ')');
- reject(error.toString());
- }
- } else if (response.statusCode !== 200) {
- log.error('[Download] Status code ' + response.statusCode + ' (' + url + ')');
- if (response.statusCode == 404 || response.statusCode == 500) {
- obj ? fulfill(obj) : fulfill();
- } else {
- reject(response.statusCode);
- }
- } else if (html){
- t = process.hrtime(t);
- var diff = (t[0] + t[1] * 1e-9).toFixed(2);
- log.verbose('[Download] ' + url + ' (' + diff + 's)');
- if (obj) {
- obj.html = html;
- fulfill(obj);
- } else {
- fulfill(html);
- }
- } else {
- log.error('[Download] WEIRD!! No errors AND no html');
- reject();
- }
- });
- });
-
-}
-
-module.exports = Download;
diff --git a/edderkopp.js b/edderkopp.js
deleted file mode 100644
index cc398c5..0000000
--- a/edderkopp.js
+++ /dev/null
@@ -1,113 +0,0 @@
-var log = require('./log');
-var Config = require('./config');
-var Queue = require('./queue');
-var Download = require('./download');
-var Parser = require('./parser');
-var Elasticsearch = require('./elasticsearch');
-var Solr = require('./solr');
-
-var Edderkopp = function(options) {
- this.config = new Config();
- this.queue = new Queue();
- this.download = new Download();
- this.parser = new Parser();
- this.site = null; // Current site
-
- if (options) {
- if (options.configPath) {
- this.config.setPath(options.configPath);
- }
- if (options.logLevel) {
- log.transports.console.level = options.logLevel;
- }
- }
-};
-
-//Edderkopp.prototype.initIndex = function(target) {
-// if (!this.config.loadAuth(target)) {
-// return;
-// }
-//
-// if (target == 'elasticsearch') {
-// this.elasticsearch = new Elasticsearch(this.config.elasticsearch);
-// } else if (target == 'solr') {
-//
-// }
-//}
-//
-//Edderkopp.prototype.checkIndex = function() {
-// this.initIndex('elasticsearch');
-// if (this.elasticsearch) {
-// this.elasticsearch = new Elasticsearch(this.config.elasticsearch);
-// this.elasticsearch.info();
-// }
-//}
-
-// DEPRECIATED
-Edderkopp.prototype.initSite = function(site) {
- log.verbose('[Edderkopp] Init site: ' + site);
-
- if (!this.config.loadSite(site)) { return false; }
-
- this.parser.init({
- url: this.config.sites[site].url,
- targets: this.config.sites[site].targets,
- blacklist: this.config.sites[site].blacklist || {}
- });
- return true;
-}
-// DEPRECIATED
-Edderkopp.prototype.getSite = function(site) {
- log.verbose('[Edderkopp] Get site: ' + site);
-
- if (!this.initSite(site)) {
- return;
- }
- this.initIndex('elasticsearch');
-
- var self = this;
- this.download.get(this.config.site.url);
- this.download.on('finished', function(response) {
- self.parser.load(response);
- if (self.parser.isTarget()) {
- var data = self.parser.getData();
- data.url = response.url;
- if (self.elasticsearch) {
- var id = 'shop_id' + self.config.site.id + 'product_id' + data.shop_product_id;
- self.elasticsearch.exists('shopz', 'product', id, function(response) {
- if (response) {
- log.warn('[Edderkopp] Document exists: ' + id);
- } else {
- log.verbose('[Edderkopp] Adding document: ' + id);
- self.elasticsearch.create(data, 'shopz', 'product', id);
- }
-
- });
- } else {
- console.log(data);
- }
- }
-
- if (self.queue.active) {
- var links = self.parser.getLinks();
- if (links) {
- self.queue.add(links);
- }
- }
-
- var url = self.queue.get();
- if (url) {
- // Delay
- if (self.config.site.delay) {
- setTimeout(function() {
- self.download.get(url);
- }, self.config.site.delay)
- } else {
- self.download.get(url);
- }
-
- }
- });
-}
-
-module.exports = Edderkopp;
diff --git a/edderkopp.png b/edderkopp.png
deleted file mode 100644
index 63ac236..0000000
Binary files a/edderkopp.png and /dev/null differ
diff --git a/elasticsearch.js b/elasticsearch.js
deleted file mode 100644
index a2ebe74..0000000
--- a/elasticsearch.js
+++ /dev/null
@@ -1,85 +0,0 @@
-var elasticsearch = require('elasticsearch');
-var CustomHttpConnector = require('./customHttpConnector');
-var log = require('./log');
-
-var Elasticsearch = function(host) {
- this.client = new elasticsearch.Client({
- host: host,
- //log: {
- //level: 'info'
- //},
- keepAlive: true,
- apiVerison: "1.3",
- connectionClass: CustomHttpConnector
- });
-}
-
-Elasticsearch.prototype = Object.create(require('events').EventEmitter.prototype);
-
-
-Elasticsearch.prototype.info = function() {
- var options = {
- requestTimeout: 1000
- };
- this.client.info(options, function (error, response, status) {
- console.log(error);
- console.log(status);
- console.log(response);
- });
-
-}
-Elasticsearch.prototype.exists = function(index, type, id, cb) {
- var options = {
- index: index,
- type: type,
- id: id
- };
- this.client.exists(options, function (error, response) {
- cb(response);
- });
-}
-
-Elasticsearch.prototype.create = function(doc, index, type, id) {
- var self = this;
- var options = {
- index: index,
- type: type,
- body: doc
- };
- if (id !== undefined) {
- options.id = id;
- }
- this.client.create(options, function (error, response) {
- if (error) {
- log.error('[Elasticsearch] ' + error);
- } else {
- log.verbose('[Elasticsearch] Document id:' + id + ' added');
- //self.emit('added', response);
- }
- });
-}
-
-Elasticsearch.prototype.update = function(doc, index, type, id) {
- var self = this;
- var options = {
- index: index,
- type: type,
- body: {
- doc: doc,
- doc_as_upsert: true
- }
- };
- if (id !== undefined) {
- options.id = id;
- }
- this.client.update(options, function (error, response) {
- if (error) {
- log.error('[Elasticsearch] ' + error);
- } else {
- log.verbose('[Elasticsearch] Document id:' + id + ' added');
- //self.emit('added', response);
- }
- });
-}
-
-module.exports = Elasticsearch;
diff --git a/example/site.html b/example/site.html
new file mode 100644
index 0000000..815b418
--- /dev/null
+++ b/example/site.html
@@ -0,0 +1,69 @@
+
+
+
+
+
+
+
+
+
diff --git a/example/site.json b/example/site.json
new file mode 100644
index 0000000..b8c503f
--- /dev/null
+++ b/example/site.json
@@ -0,0 +1,126 @@
+{
+ "id": 1234,
+ "name": "site",
+ "pages": {
+ "somePage": [
+ {
+ "name": "item",
+ "data": "object",
+ "kids": [
+ {
+ "name": "title",
+ "elem": "head meta[property=og\\:title]",
+ "data": [ "attr", "content"]
+ },
+ {
+ "elem": ".item",
+ "kids": [
+ {
+ "name": "id",
+ "data": [ "data", "id"],
+ "task": [
+ [ "match", "\\D*'([\\d]+)\\D*", 1 ],
+ [ "parseInt" ]
+ ]
+ },
+ {
+ "name": "image",
+ "elem": ".image-main img, .image-extra img",
+ "data": [ "attr", "src" ],
+ "task": [
+ [ "replace", "thumb", "big" ],
+ [ "prepend", "https://foo.bar/" ]
+ ]
+ },
+ {
+ "name": "description",
+ "elem": ".description",
+ "data": "html"
+ },
+ {
+ "name": "short",
+ "elem": ".short"
+ },
+ {
+ "elem": ".wrapper",
+ "kids": [
+ {
+ "name": "subtitle",
+ "elem": "meta[itemprop=title]",
+ "data": [ "attr", "content" ]
+ },
+ {
+ "name": "image",
+ "elem": ".image-wrapped img",
+ "data": [ "attr", "src" ],
+ "task": [
+ [ "replace", "thumb", "big" ],
+ [ "prepend", "https://bar.foo/" ],
+ [ "append", "?foo=bar" ]
+ ]
+ },
+ {
+ "name": "textOnly",
+ "elem": ".text-only",
+ "data": "txtn"
+ }
+ ]
+ },
+ {
+ "name": "special",
+ "elem": ".special",
+ "data": "object",
+ "kids": [
+ {
+ "name": "first",
+ "data": [ "data", "title" ]
+ },
+ {
+ "name": "second",
+ "elem": "p"
+ }
+ ]
+ },
+ {
+ "name": "objects",
+ "data": "object",
+ "kids": [
+ {
+ "name": "objectWrapper",
+ "elem": ".objects .group",
+ "data": "array",
+ "kids": [
+ {
+ "name": "group",
+ "data": [ "attr", "content" ]
+ },
+ {
+ "name": "comment",
+ "elem": ".comment"
+ },
+ {
+ "name": "details",
+ "elem": ".details",
+ "data": "array",
+ "kids": [
+ {
+ "name": "color",
+ "elem": ".color"
+ },
+ {
+ "name": "material",
+ "elem": ".material"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+}
diff --git a/getPage.js b/getPage.js
deleted file mode 100755
index f40b153..0000000
--- a/getPage.js
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env node
-var log = require('./log');
-var Edderkopp = require('./edderkopp');
-
-//log.transports.console.level = 'verbose';
-log.transports.console.level = 'debug';
-log.transports.console.prettyPrint = true;
-
-// Check argv
-var url = process.argv[2];
-if (url === undefined) {
- var thisFile = process.argv[1].split('/').pop();
- log.info('Usage: ' + thisFile + '
+ 
+
+
+
+ 
+ 
+
+
+
+
+
+ Once upon a time there was an item
+that lived happily ever after
+
+ Get this but not the wrapping span tags or strong tags
+
+
+
+
+
+ 
+
+
+
+ Don't get this element
+ This is what I want!
+ Don't get this element
+
+
+
+
+ I'm not special
+
+
+
+
+
+ Square comment
+
+
+ Blue
+ Metal
+
+
+ Green
+ Wood
+
+
+
+ Round comment
+
+
+ Yellow
+ Plastic
+
+
+ Purple
+ Rubber
+ ').append($(this)).html();
- log.debug('[Parser] No link: ' + outerHTML);
- return;
- }
- url = url.trim();
- log.silly('[Parser] Found url: ' + url);
-
- // Skip blacklisted paths
- if (blacklist.paths !== undefined) {
- for (var i = 0; i < blacklist.paths.length; i++) {
- if (url.indexOf(blacklist.paths[i]) !== -1) {
- log.debug('[Parser] Skip ignored path (' + blacklist.paths[i] + '): ' + url);
- return;
- }
- }
- }
-
- var uri = URI(url);
-
- // Prepend domain to relative path (Example: )
- if (uri.is('relative')) {
- uri.protocol(siteUri.protocol()).hostname(siteUri.hostname());
- }
-
- // Skip external links
- if (uri.hostname() !== siteUri.hostname()) {
- log.debug('[Parser] Skip external link: ' + url);
- return;
- }
-
- // Skip media files
- if (uri.suffix().match(/jpg|jpeg|png|gif|bmp|svg|pdf/i) !== null) {
- log.debug('[Parser] Skip media file: ' + url);
- return;
- }
-
- // Remove anchor (Example: http://domain/page.html#anchor)
- uri.hash('');
-
- // Remove trailing slash (Example: http://domain/section/) -> important?
-
- url = uri.normalize().toString();
- log.silly('[Parser] Processed: ' + url);
-
- // Skip previously downloaded pages
- var md5 = crypto.createHash('md5').update(url).digest('hex');
- if (cache[md5] === undefined) {
- cache[md5] = true;
- log.silly('[Parser] Send to queue');
- } else {
- log.silly('[Parser] Already fetched');
- return;
- }
-
- // Skip previously downloaded filenames
- if (uri.filename()) {
- var md5file = crypto.createHash('md5').update(uri.filename()).digest('hex');
- if (cacheFile[md5file] === undefined) {
- cacheFile[md5file] = [ url ];
- } else {
- cacheFile[md5file].push(url);
- log.warn(cacheFile[md5file]);
- return;
- }
- }
-
- links.push(url);
- });
-
- if (links.length) {
- log.verbose('[Parser] ' + links.length + ' new links');
- return links;
- } else {
- log.verbose('[Parser] No new links');
- return false;
- }
-
-}
-
-Parser.prototype.getData = function(obj) {
- log.verbose('[Parser] Parse content of ' + obj.url);
- if (!obj.html) {
- log.error('[Parser] No html for: ' + obj.url);
- return obj;
- }
- $ = cheerio.load(obj.html);
- obj.web = {};
- pageParser(null, obj.config.targets, obj.web);
- log.debug(obj.web);
- return obj;
-}
-
-function pageParser($container, targets, data, depth) {
- depth = depth || 1;
- var logPrefix = '[Parser] ' + Array(depth).join(" ") + '';
- log.silly(logPrefix + 'Depth: ' + depth);
- for (var i = 0; i < targets.length; i++) {
- var target = targets[i];
-
- // Support single and multiple elements. Ex: '.foobar' and ['.foo', '.bar']
- var elements = Array.isArray(target.elem) ? target.elem : [ target.elem ];
-
- var $elem;
- if (target.ifelse) {
- // Only use the first found element in array
- for (var l = 0; l < elements.length; l++) {
- $elem = $container !== null ? $(elements[l], $container) : $(elements[l]);
- if ($elem.length) { break; }
- }
- } else {
- $elem = $container !== null ? $(elements.join(','), $container) : $(elements.join(','));
- }
-
- var msg = target.type + ': ' + target.elem + (target.name ? ' (' + target.name + ')' : '');
- if ($elem.length === 0) {
- if (target.miss) {
- var $missing = $container !== null ? $(target.miss, $container) : $(target.miss);
- if ($missing.length) {
- continue;
- }
- } else if (target.optional) {
- continue;
- }
- log.warn('[Parser] Couldn\'t find ' + msg);
- continue;
- }
-
- log.silly(logPrefix + 'Found ' + msg);
- var key = target.name;
- if (target.type == 'container') {
- if (!key) {
- pageParser($elem, target.children, data, depth + 1);
- } else if ($elem.length > 1) {
- log.silly(logPrefix + 'Name: ' + key);
- $elem.each(function() {
- if (data[key] === undefined) {
- data[key] = [];
- }
- var obj = {};
- data[key].push(obj);
- pageParser($(this), target.children, obj, depth + 1);
- });
- } else {
- data[key] = {};
- log.silly(logPrefix + 'Name: ' + key);
- pageParser($elem, target.children, data[key], depth + 1);
- }
- } else if (target.type == 'data') {
- // Get value(s) from the data attribute, a custom attribute or content of tag
- var values = [];
- $elem.each(function() {
- if (target.data) {
- // Ex:
- var data = Array.isArray(target.data) ? target.data : [ target.data ];
- for (var j = 0; j < data.length; j++) {
- values.push($(this).data(data[j]));
- }
- } else if (target.attr) {
- // Ex: 
, foo
- values.push($(this).attr(target.attr));
- } else if (target.text) {
- // Ex: , , foo
+ for (let i = 1; i < rule.data.length; i++) {
+ values.push($(this).attr(rule.data[i]));
+ }
+ break;
+ case 'data':
+ // Get content from data
+ // Ex:
valueskip this
- values.push($(this).contents().filter(function() { return this.nodeType == 3; } ).text()); // 3 = TEXT_NODE
- } else {
- // Ex: value 1
value 2
value
- var value = target.tags ? $(this).html() : $(this).text();
- values.push(value.trim());
- }
- });
-
- // Run functions defined in config on found values
- if (target.func) {
- var functions = Array.isArray(target.func) ? target.func : [ target.func ];
- for (var j = 0; j < functions.length; j++) {
- var name = functions[j].name;
- var args = functions[j].args;
- for (var k = 0; k < values.length; k++) {
- var value = values[k];
- values[k] = _functions[name](value, args);
- log.silly(logPrefix + 'Run function: ' + name + (args ? ' (' + value + ', ' + JSON.stringify(args) + ')' : ''));
- }
- }
- }
-
- // Store found and processed values in data structure
- if (values.length > 1) {
- // Support joining of values
- if (target.glue) {
- log.silly(logPrefix + 'Glue: ' + values.length + ' items joined with "' + target.glue + '"');
- values = values.join(target.glue);
- }
- data[key] = data[key] ? Array.concat(data[key], values) : values; // join values with same name
- } else if (values.length) {
- data[key] = data[key] ? values.concat(data[key]) : values.pop(); // join values with same name
- }
- }
- }
-}
-
-// Parse functions
-var _functions = {
- regexp: function(value, args) {
- var matches = value.match(new RegExp(args[0]));
- if (matches) {
- return args[1] ? matches[args[1]] : matches;
- } else {
- return null;
- }
- },
- prepend: function(value, text) {
- return text + value;
- },
- append: function(value, text) {
- return value + text;
- },
- join: function(value, args) {
- var str = '';
- for (var i = 0; i < args.length; i++) {
- if (args[i].charAt(0) === '$') {
- if (Array.isArray(value)) {
- var index = parseInt(args[i].substr(1));
- str += value[index];
- }
- else {
- str += value;
- }
- }
- else {
- str += args[i];
- }
- }
- return str;
- },
- replace: function(value, args) {
- // Check if pattern is a regex or string
- var pattern = args[2] && args[2] == 'regexp' ? new RegExp(args[0], 'g') : args[0];
- return value.replace(pattern, args[1]);
- },
- toInt: function(value) {
- return /^\d+$/.test(value) ? parseInt(value, 10) : false;
- },
- parsePrice: function(price, args) {
- // Example inputs: "kr 2.347,95", "969 NOK", "625 kr.", "449.0" (number)
- price = price.toString().replace(/[^\d,.]/g, ''); // strip everything except numbers, "," and "."
- var match = price.match(/^([\d,.]+)[.,](\d{1,2})$/); // split price on decimals if they exist
- if (match) {
- price = match[1].replace(/[,.]/g, ''); // strip "," and "." from the part before the decimals
- price = Math.round(price + '.' + match[2]); // add decimals and round
- } else if (price) {
- price = parseInt(price.replace(/[,.]/g, ''), 10); // strip "," and "." and convert to int
- } else {
- price = null;
- }
- if (args && args.addVat) {
- price = Math.round(price * 1.25); // Add MVA/VAT and round
- }
- return price;
- },
- htmlEntitiesDecode: function(value) {
- return he.decode(value);
- }
-}
-
-
-
-module.exports = Parser;
diff --git a/queue.js b/queue.js
deleted file mode 100644
index 1dc0cbc..0000000
--- a/queue.js
+++ /dev/null
@@ -1,79 +0,0 @@
-var log = require('./log');
-
-var Queue = function(options) {
- this._stack = {
- get: [], // items at current level
- add: [] // items for next level
- }
- this.active = true;
-
- // Handle options and defaults
- options = options || {};
- this._maxLevel = options.maxLevel || false;
- this._maxItems = options.maxItems || false
-
- // Internal
- this._currentLevel = 0;
- this._currentItem = 0;
-}
-
-Queue.prototype.add = function(items) {
- log.verbose('[Queue] Received ' + items.length + ' items');
- if (!this._maxLevel || this._currentLevel < this._maxLevel) {
-
- // Support items not in array
- if (!Array.isArray(items)) {
- items = [ items ];
- }
-
- // Add items to the stack
- var n = items.length;
- var added = 0;
- for (var i = 0; i < n; i++) {
- if (!this._maxItems || this._currentItem < this._maxItems) {
- this._stack.add.push(items[i]);
- this._currentItem++;
- added++;
- log.debug('[Queue] Added item: ' + items[i]);
- } else {
- this.active = false;
- log.debug('[Queue] Reached max items limit of ' + this._maxItems);
- break;
- }
- }
- log.verbose('[Queue] Added ' + added + ' items');
- } else {
- log.verbose('[Queue] No items added');
- }
-}
-
-Queue.prototype.get = function() {
- log.verbose('[Queue] Get item');
- if (this._stack.get.length) {
- return this._stack.get.pop();
- } else {
- if (!this._maxLevel || this._currentLevel < this._maxLevel) {
- // Set next level
- this._currentLevel++;
- log.verbose('[Queue] Starting level ' + this._currentLevel);
-
- // Switch stack
- this._stack.get = this._stack.add;
- this._stack.add = [];
-
- if (this._stack.get.length) {
- return this._stack.get.pop();
- } else {
- this.active = false;
- log.verbose('[Queue] No more items in queue');
- return false;
- }
- } else {
- this.active = false;
- log.verbose('[Queue] Reached max level limit of ' + this._maxLevel);
- return false;
- }
- }
-}
-
-module.exports = Queue;
diff --git a/solr.js b/solr.js
deleted file mode 100644
index 9732815..0000000
--- a/solr.js
+++ /dev/null
@@ -1,33 +0,0 @@
-var solr = require('solr-client')
-var log = require('./log');
-
-var Solr = function(host, port, index) {
- if (index === undefined) {
- log.warn('[Solr] No index provided');
- return;
- }
- host = host || '192.168.1.104';
- port = port || '8983';
- this.client = solr.createClient({
- host: host,
- port: port,
- core: index
- });
-}
-
-Solr.prototype = Object.create(require('events').EventEmitter.prototype);
-
-Solr.prototype.add = function(doc) {
- var self = this;
- this.client.add(doc, function(error, response) {
- if (error) {
- log.error('[Solr] ' + error);
- } else {
- self.client.commit();
- log.verbose('[Solr] Document added');
- self.emit('added', response);
- }
- });
-}
-
-module.exports = Solr;
\ No newline at end of file
diff --git a/src/config.js b/src/config.js
new file mode 100644
index 0000000..7faf775
--- /dev/null
+++ b/src/config.js
@@ -0,0 +1,64 @@
+import fs from 'fs';
+import URI from 'urijs';
+import log from './log';
+
+export default { setPath, get };
+
+let _path = __dirname + '/../config';
+
+function setPath(path) {
+ _path = path;
+}
+
+function get(arg) {
+ if (Number.isInteger(arg)) {
+ return getById(arg);
+ } else if (arg.indexOf('http') !== -1) {
+ return getByUrl(arg);
+ } else if (/^[^/]+\.json$/.test(arg)) {
+ return parse(_path + '/' + arg);
+ } else {
+ return parse(arg);
+ }
+}
+
+function getById(id) {
+ let files = getFiles(_path);
+ for (let i = 0; i < files.length; i++) {
+ if (files[i].match(/-(\d+)/).pop() == id) {
+ return parse(files[i]);
+ }
+ }
+ return false;
+}
+
+function getByUrl(url) {
+ let hostname = new URI(url).hostname();
+ let files = getFiles(_path);
+ for (let i = 0; i < files.length; i++) {
+ let config = parse(files[i]);
+ if (hostname == new URI(config.url).hostname()) {
+ // log.verbose('[Config] Get config for ' + obj.name);
+ return config;
+ }
+ }
+ // log.error('[Config] Missing config for ' + url);
+ return false;
+}
+
+// Recursivly find all files
+function getFiles(path) {
+ var files = [];
+ fs.readdirSync(path).forEach( (file) => {
+ if (fs.statSync(path + '/' + file).isDirectory()) {
+ files = files.concat(getFiles(path + '/' + file));
+ } else {
+ files.push(path + '/' + file);
+ }
+ });
+ return files;
+}
+
+function parse(file) {
+ return JSON.parse(fs.readFileSync(file).toString());
+}
diff --git a/src/download.js b/src/download.js
new file mode 100644
index 0000000..fcc3e01
--- /dev/null
+++ b/src/download.js
@@ -0,0 +1,31 @@
+import request from "request";
+import log from './log';
+
+export default function(url, timeout) {
+ timeout = timeout ? timeout * 1000 : 60000;
+ const t0 = process.hrtime();
+ const options = {
+ url: url,
+ headers: {
+ 'User-Agent': USER_AGENT
+ },
+ timeout: timeout
+ }
+ return new Promise(function (fulfill, reject) {
+ request(options, function (error, response, html) {
+ if (error !== null) {
+ reject(error);
+ } else if (response.statusCode !== 200) {
+ reject('Error! Response code: ' + response.statusCode);
+ } else if (html){
+ var diff = process.hrtime(t0);
+ fulfill({
+ html: html,
+ time: (diff[0] + diff[1] * 1e-9).toFixed(2)
+ });
+ } else {
+ reject('This should not happen');
+ }
+ });
+ });
+}
diff --git a/src/index.js b/src/index.js
new file mode 100644
index 0000000..821f3a6
--- /dev/null
+++ b/src/index.js
@@ -0,0 +1,9 @@
+import config from './config';
+import download from "./download"
+import Parser from "./parser"
+import log from "./log"
+
+global.VERSION = '1.0.0-alpha';
+global.USER_AGENT = 'Edderkopp/' + VERSION;
+
+export { config, download, Parser, log };
diff --git a/log.js b/src/log.js
similarity index 69%
rename from log.js
rename to src/log.js
index f375ea6..93196fc 100644
--- a/log.js
+++ b/src/log.js
@@ -1,4 +1,14 @@
-var winston = require('winston');
+import winston from 'winston';
+/**
+ * Config levels:
+ * silly: 0,
+ * debug: 1,
+ * verbose: 2,
+ * info: 3,
+ * warn: 4,
+ * error: 5
+ */
+
winston.emitErrs = true;
var log = new winston.Logger({
@@ -22,13 +32,8 @@ var log = new winston.Logger({
exitOnError: false
});
-module.exports = log;
+log.setLevel = function(level) {
+ this.transports.console.level = level;
+};
-//npmConfig.levels = {
-// silly: 0,
-// debug: 1,
-// verbose: 2,
-// info: 3,
-// warn: 4,
-// error: 5
-//};
+export default log;
diff --git a/src/parser.js b/src/parser.js
new file mode 100644
index 0000000..e69d825
--- /dev/null
+++ b/src/parser.js
@@ -0,0 +1,135 @@
+import URI from 'urijs';
+import cheerio from "cheerio";
+import log from './log';
+import * as tasks from './parser.tasks';
+
+export default class Parser {
+
+ constructor(html) {
+ this.$ = cheerio.load(html);
+ }
+
+ getData(rules) {
+ return this._recParse(rules);
+ }
+
+ // Recursively parse DOM
+ _recParse(rules, data, $container) {
+ let $ = this.$;
+ data = data || {};
+ for (let i = 0; i < rules.length; i++) {
+ const rule = rules[i];
+ if (rule.name) {
+ const $elem = rule.elem ? $(rule.elem, $container) : $container;
+ if (rule.data == 'array') {
+ data[rule.name] = [];
+ $elem.each((i, e) => {
+ let obj = {};
+ data[rule.name].push(obj);
+ this._recParse(rule.kids, obj, $(e));
+ });
+ } else if (rule.data == 'object') {
+ data[rule.name] = {};
+ this._recParse(rule.kids, data[rule.name], $elem);
+ } else {
+ if ($elem.length > 0) {
+ const values = this._getContent($elem, rule);
+ // Join values with same name
+ data[rule.name] = data[rule.name] ? [].concat(data[rule.name], values) : values;
+ } else if (!rule.null){
+ log.warn('Element not found: ' + rule.elem);
+ }
+ }
+ } else if (rule.elem) {
+ this._recParse(rule.kids, data, $(rule.elem, $container));
+ }
+ }
+ return data;
+ }
+
+ // Get values
+ _getContent($elem, rule) {
+ let $ = this.$;
+ let values = [];
+ const dataType = Array.isArray(rule.data) ? rule.data[0] : rule.data;
+ $elem.each(function() {
+ switch (dataType) {
+ case 'html':
+ // Get all content including tags
+ // Ex: paragraph 1
paragraph 2
paragraph 3
+ values.push($(this).html().trim()); + break; + case 'txtn': + // Get only text nodes + // Ex: skip this get this skip this + values.push($(this).contents().filter(function() { + return this.nodeType == 3; // 3 = TEXT_NODE + }).text().trim()); + break; + case 'attr': + // Get content from attribute + // Ex:
+ for (let i = 1; i < rule.data.length; i++) {
+ values.push($(this).data(rule.data[i]));
+ }
+ break;
+ case 'text':
+ default:
+ // Get only text (strip away tags)
+ values.push($(this).text().trim());
+ }
+ });
+
+ // Run tasks on values
+ if (rule.task) {
+ let task;
+ if (typeof rule.task == 'string') {
+ // "task": "foobar"
+ task = [ [ rule.task ] ];
+ } else if (!Array.isArray(rule.task[0])) {
+ // "task": [ "foobar", "arg1", "arg2" ]
+ task = [ rule.task ];
+ } else {
+ // "task": [
+ // [ "foobar1", "arg1", "arg2" ],
+ // [ "foobar2", "arg1", "arg2" ]
+ // ]
+ task = rule.task;
+ }
+ for (let i = 0; i < task.length; i++) {
+ for (let j = 0; j < values.length; j++) {
+ let name = task[i][0];
+ let args = task[i].slice(1);
+ if (tasks[name]) {
+ values[j] = tasks[name](args, values[j]);
+ } else {
+ log.warn('task not exist: ' + name);
+ }
+ }
+ }
+ }
+
+ if (values.length == 1) {
+ values = values.pop();
+ }
+
+ return values;
+ }
+
+ // Support custom tasks
+ static injectTasks(customTasks) {
+ for (var prop in customTasks) {
+ if (tasks[prop]) {
+ log.warn('Overriding task: ' + prop);
+ }
+ tasks[prop] = customTasks[prop];
+ }
+ }
+}
diff --git a/src/parser.tasks.js b/src/parser.tasks.js
new file mode 100644
index 0000000..289789b
--- /dev/null
+++ b/src/parser.tasks.js
@@ -0,0 +1,63 @@
+export {
+ _match as match,
+ _prepend as prepend,
+ _append as append,
+ _join as join,
+ _replace as replace,
+ _parseInt as parseInt
+};
+
+// "task": [ "match", "\\/(\\d+)\\.", 1 ]
+function _match(args, value) {
+ var matches = value.match(new RegExp(args[0]));
+ if (matches) {
+ return args[1] ? matches[args[1]] : matches;
+ } else {
+ return null;
+ }
+}
+
+// "task": [ "prepend", "http://foo.bar/" ]
+function _prepend(args, value) {
+ return args[0] + value;
+}
+
+// "task": [ "append", "&foo=bar" ]
+function _append(args, value) {
+ return value + args[0];
+}
+
+// "task": [ "join", "http://foo.bar/", "$1" ]
+// "task": [ "join", "$1", "$3", "(foobar)", "$2" ]
+function _join(args, value) {
+ var str = '';
+ for (var i = 0; i < args.length; i++) {
+ if (args[i].charAt(0) === '$') {
+ str += value[args[i].substr(1)];
+ } else {
+ str += args[i];
+ }
+ }
+ return str;
+}
+
+// "task": [ "replace", "foo", "bar" ]
+// "task": [ "replace", "[\\r\\n\\t\\s]+", "", "regexp" ]
+function _replace(args, value) {
+ if (typeof args[0] == 'string' && typeof args[1] == 'string') {
+ args[0] = [ args[0] ];
+ args[1] = [ args[1] ];
+ }
+ var pattern;
+ for (var i = 0; i < args[0].length; i++) {
+ pattern = args[2] && args[2] == 'regexp' ? new RegExp(args[0][i], 'g') : args[0][i];
+ value = value.replace(pattern, args[1][i]);
+ }
+ return value;
+}
+
+// "task": [ "parseInt" ]
+function _parseInt(args, value) {
+ value = value ? value.replace(/[^\d]/g, '') : null;
+ return value ? parseInt(value, 10) : null;
+}