/micro-scraper

Node.js 爬虫示例 （for：百度百科）

Primary LanguageJavaScript

micro-scraper

Node.js 爬虫示例 （for：百度百科）

app.js

var request = require('request'),
    cheerio = require('cheerio'),
    http = require('http'),
    url = require('url');

var host = 'http://baike.baidu.com/view/39744.htm';//可修改为其他的百科地址

var html = [];
setInterval(scraper(host), 1000*60*15);//15 分钟更新一次
function scraper (host) {
  request(host, function (error, response, data) {
    if (!error && response.statusCode == 200) {
      var $ = cheerio.load(data);
      var title = $('.title').first().text(),
          header = [],
          nav = [],
          body = [];
      //删除无用数据
      $('.title').remove();
      $('.pic-info').remove();
      $('.count').remove();
      $('sup').remove();
      //筛选有用数据
      $('#lemmaContent-0 .headline-1').each(function (i) {
        var str = '',
            $next = $(this).next();
        while (!$next.hasClass('headline-1')&&(!$next.next().hasClass('clear'))) {
          if ($next.hasClass('headline-2')) {
            str += "<p><strong>" + $next.text() + "</strong></p>";
          } else {
            str += "<p>" + $next.text() + "</p>";
          }
          $next = $next.next();
        }
        header.push($(this).find('.headline-content').text());
        nav.push("<span><a href='/" + i + "'>" + header[i] + "</a></span>");
        body.push(str);
      });

      var len = $('#catalog-holder-0 .catalog-item').length;//获取 “目录” 条文数
      for (var i = 0; i < len;  i++) {
        html[i] = "" +
        "<!DOCTYPE html>" +
        "<html>" +
        "<head>" +
        "<meta charset='UTF-8' />" +
        "<title>" + title + "</title>" +
        "<style type='text/css'>" +
        "body{width:600px;margin:2em auto;font-family:'Microsoft YaHei';}" +
        "p{line-height:24px;margin:1em 0;}" +
        "header{border-bottom:1px solid #cccccc;font-size:2em;font-weight:bold;padding-bottom:.2em;}" +
        "nav{float:left;font-family:'Microsoft YaHei';margin-left:-12em;width:9em;text-align:right;}" +
        "nav a{display:block;text-decoration:none;padding:.7em 1em;color:#000000;}" +
        "nav a:hover{background-color:#003f00;color:#f9f9f9;-webkit-transition:color .2s linear;}" +
        "</style>" +
        "</head>" +
        "<body>" +
        "<header>" + header[i] + "</header>" +
        "<nav>" + nav.join('') + "</nav>" +
        "<article>" + body[i] + "</article>" +
        "</body>" +
        "</html>";
      }
    }
  });
}

http.createServer(function (req, res) {
  var path = url.parse(req.url).pathname;
  path = path == '/' ? 0 : parseInt(path.slice(1));
  res.writeHead(200, {"Content-Type":"text/html"});
  res.end(html[path]);
}).listen(3000);

console.log('Server running at localhost:3000');