파싱해서 DB로 저장해보기.

이쪽이 전문분야가 아니라 많이 부족합니다.

아~ 이렇게도 가능 하구나 하고 참조만 하세요.

(그누보드로 돌리면 리자님한테 혼납니다.)

[code]

var Spider = require('node-spider');

var mysql = require('mysql');

var conn = mysql.createConnection({

    host    :'localhost',

    port : 3306,

    user : 'DB아이디',

    password : '비번',

    database:'데이터베이스'

});

conn.connect();

if(!String.prototype.trim) {

  String.prototype.trim = function () {

    return this.replace(/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, '');

  };

}

function addslashes(string) {

    return string.replace(/\\/g, '\\\\').

        replace(/\u0008/g, '\\b').

        replace(/\t/g, '\\t').

        replace(/\n/g, '\\n').

        replace(/\f/g, '\\f').

        replace(/\r/g, '\\r').

        replace(/'/g, '\\\'').

        replace(/"/g, '\\"');

}

var spider = new Spider({

    // How many requests can be run in parallel

    concurrent: 5,

    // How long to wait after each request

    delay: 0,

    // A stream to where internal logs are sent, optional

    logs: process.stderr,

    // Re-visit visited URLs, false by default

    allowDuplicates: false,

    // If `true` all queued handlers will be try-catch'd, errors go to `error` callback

    catchErrors: true,

    // Called when there's an error, throw will be used if none is provided

    error: function(err, url) {

console.log(err);

    },

    // Called when there are no more requests

    done: function() {

    },

    //- All options are passed to `request` module, for example:

    headers: { 'user-agent': 'node-spider' },

    //encoding: 'utf8'

encoding: null

});

var handleRequest = function(doc) {

// doc.res

var content = doc.$('div.con_inner').text().trim();

var title = doc.$('#head_title').text().trim();

var contentHTML = doc.$('div.con_inner').html();

var member =  doc.$('li#info_name').find('span.member').text().trim();

var PageUrl = doc.url;

if(title) title = addslashes(title);

if(member) member = addslashes(member);

if(content) content = addslashes(content);

if(contentHTML) contentHTML = addslashes(contentHTML);

if(content) {

// new page crawled

console.log('---시작---');

console.log(PageUrl); // page url

conn.query("insert into `_sc` set `member` = '"+member+"', `title` = '"+title+"', `content` = '"+content+"', `html_content` = '"+contentHTML+"', `url` = '"+PageUrl+"', `rdate` = now()");

console.log('---종료---');

}

    // uses cheerio, check its docs for more info

    doc.$('a').each(function(i, elem) {

        // do stuff with element

        var href = elem.attribs.href.split('#')[0];

        var url = doc.resolve(href);

if(url.indexOf('/cm_free/') > -1) { // 자게 주소 패턴이 있을경우만..

// crawl more

spider.queue(url, handleRequest);

}

    });

};

// start crawling

spider.queue('http://sir.kr/cm_free', handleRequest);

[/code]