[关闭]
@jeffjade 2016-10-10T15:04:14.000000Z 字数 6977 阅读 1419

Nodejs 各种小脚本

Node.js

Nodejs 遍历文件夹 获取 文件夹下所有文件列表

  1. var fs = require('fs');
  2. var root_path=process.argv[2];
  3. var w_file='res.lst';
  4. function getAllFiles(root){
  5.   var res = [] , files = fs.readdirSync(root);
  6.   files.forEach(function(file){
  7.     var pathname = root+'/'+file
  8.     , stat = fs.lstatSync(pathname);
  10.     if (!stat.isDirectory()){
  11.        res.push(pathname.replace(root_path,'.'));
  12.     } else {
  13.        res = res.concat(getAllFiles(pathname));
  14.     }
  15.   });
  16.   return res
  17. }
  18. var w_content=getAllFiles(root_path).join('\n');
  19. fs.readFile(root_path+w_file,function(err , data){
  20.    if(err && err.errno==33){
  21.      fs.open(w_file,"w",0666,function(e,fd){
  22.        if(e) throw e;
  23.        fs.write(fd,w_content,0,'utf8',function(e){
  24.         if(e) throw e;
  25.         fs.closeSync(fd);
  26.        })
  27.      });
  28.    } else{
  29.      fs.writeFile(root_path+w_file,w_content,function(e){
  30.         if(e) throw e
  31.      })
  32.    }
  33. })

参见： http://www.cnblogs.com/litao229/archive/2012/01/04/2312393.html

NodeJs 按行读取文件

  1. var fs = require('fs'),
  2.     readline = require('readline');
  4. var rd = readline.createInterface({
  5.     input: fs.createReadStream('./test.txt'),
  6.     output: process.stdout,
  7.     terminal: false
  8. });
  10. var i = 1;
  11. rd.on('line', function(line) {
  12.     console.log('第'+ i +'行：'+ line);
  13.     write(line);
  14.     i++;
  15. });

Nodejs 过滤单个页面中不合法链接：

  1. var http = require("http"),
  2.     fs = require('fs'),
  3.     cheerio = require("cheerio");
  5. // var root_path=process.argv[2];
  7. var url = "http://www.jeffjade.com/2016/03/30/104-front-end-tutorial/"
  8. var saveLinkArr = {}
  10. // Utility function that downloads a URL and invokes callback with the data.
  11. function download(url, callback) {
  12.     http.get(url, function(res) {
  13.         var data = "";
  15.         res.on('data', function(chunk) {
  16.             data += chunk;
  17.         });
  19.         res.on("end", function() {
  20.             callback(data);
  21.         });
  22.     }).on("error", function() {
  23.         callback(null);
  24.     });
  25. }
  27. function writeOut(resData){
  28.     var filepath = './jeffJadeLinkData.js'
  29.     var resJson = JSON.stringify(resData , null, 2 )
  30.     fs.writeFile( filepath, resJson , function(e){
  31.         if(e) throw e
  32.     });
  33. }
  35. download(url, function(data) {
  36.     if (data) {
  37.         var $ = cheerio.load(data);
  39.         $("tbody a").each(function(i, e) {
  40.             console.log(i, $(e).attr("href"));
  41.             saveLinkArr[i] = $(e).attr("href")
  42.         });
  44.         writeOut( saveLinkArr )
  45.         console.log("done");
  46.     } else {
  47.         console.log("download error !")
  48.     };
  49. });
  51. // ==================================================
  53. var http = require('http'),
  54.     parse = require('url').parse;
  56. function urlget(url, callback) {
  57.     var info = parse(url),
  58.         path = info.pathname + (info.search || ''),
  59.         options = {
  60.             host: info.hostname,
  61.             port: info.port || 80,
  62.             path: path,
  63.             method: 'GET'
  64.         };
  66.     var req = null, request_timeout = null;
  67.     request_timeout = setTimeout(function() {
  68.         request_timeout = null; 
  69.         req.abort();
  70.         callback(new Error('Request timeout'));
  71.     }, 5000);
  73.     req = http.request(options, function(res) {
  74.         clearTimeout(request_timeout);
  75.         var chunks = [], length = 0, response_timeout = null;
  76.         response_timeout = setTimeout(function() {
  77.             response_timeout = null;
  78.             req.abort();
  79.             callback(new Error('Response timeout'));
  80.         }, 5000);
  81.         res.on('data', function(chunk) {
  82.             length += chunk.length;
  83.             chunks.push(chunk);
  84.         }).on('end', function() {
  85.             if(response_timeout) {
  86.                 // node0.5.x及以上：req.abort()会触发res的end事件
  87.                 clearTimeout(response_timeout);
  88.                 var data = new Buffer(length);
  89.                 // ... 此处忽略相同部分的代码 ...
  90.             }
  91.         }).on('error', function(err) {
  92.             // ... 此处忽略相同部分的代码 ...
  93.         }).on('aborted', function() {
  94.             if(response_timeout) {
  95.                 // node0.5.x及以上：当res有效的时候，
  96.                 // req.abort()会触发res的aborted事件
  97.                 callback(new Error('Response aborted'), res);
  98.             }
  99.         });
  100.     }).on('error', function(err) {
  101.         callback(err);
  102.     });
  103.     req.end();
  104. };
  107. // https://fengmk2.com/ppt/those-things-using-nodejs/#slide-10

===============================

  1. var http = require('http'),
  2.     parse = require('url').parse,
  3.     fs = require('fs'),
  4.     needFilterList = require('./jeffJadeLinkData.json');
  6. function requestUrl(url, callback) {
  7.     var info = parse(url),
  8.         path = info.pathname + (info.search || ''),
  9.         options = {
  10.             host: info.hostname,
  11.             port: info.port || 80,
  12.             path: path,
  13.             method: 'GET'
  14.         };
  16.     var req = null,
  17.         request_timeout = null;
  19.     request_timeout = setTimeout(function() {
  20.         request_timeout = null;
  21.         req.abort();
  22.         callback(url, new Error('Request timeout'));
  23.     }, 5000);
  25.     req = http.request(options, function(res) {
  26.         clearTimeout(request_timeout);
  28.         var chunks = [],
  29.             length = 0,
  30.             response_timeout = null;
  32.         response_timeout = setTimeout(function() {
  33.             response_timeout = null;
  34.             req.abort();
  35.             callback(url, new Error('Response timeout'));
  36.         }, 5000);
  38.         res.on('data', function(chunk) {
  39.             length += chunk.length;
  40.             chunks.push(chunk);
  41.         }).on('end', function() {
  42.             if (response_timeout) {
  43.                 // node0.5.x及以上：req.abort()会触发res的end事件
  44.                 clearTimeout(response_timeout);
  45.                 var data = new Buffer(length);
  46.                 // ... 此处忽略相同部分的代码 ...
  47.             }
  48.         }).on('error', function(err) {
  49.             // ... 此处忽略相同部分的代码 ...
  50.             callback(url, err);
  51.         }).on('aborted', function() {
  52.             if (response_timeout) {
  53.                 // node 0.5.x及以上：当res有效的时候，
  54.                 // req.abort()会触发res的aborted事件
  55.                 callback(url, new Error('Response aborted'), res);
  56.             }
  57.         });
  58.     }).on('error', function(err) {
  59.         console.log('what? error ? \n' + url)
  60.         // callback(url, err);
  61.     });
  63.     req.end();
  64. };
  66. /*===============================================================Read Line: */
  67. var errUrlList = []
  69. function callback(errUrl, errInfo, errRes) {
  70.     console.log("err url: " + errUrl)
  71.     errUrlList.push(errUrl)
  72.     writeOut(errUrlList)
  73. }
  75. function jadeFilterLink() {
  76.     for (var key in needFilterList) {
  77.         console.log("now turn:" + needFilterList[key])
  78.         requestUrl(needFilterList[key], callback)
  79.     }
  80. }
  82. function writeOut(resData) {
  83.     var filepath = './err_url_list.json'
  84.     var resJson = JSON.stringify(resData, null, 2)
  85.     fs.writeFile(filepath, resJson, function(e) {
  86.         if (e) throw e
  87.     });
  88. }
  90. jadeFilterLink()
  1. var http = require("http"),
  2.     fs = require('fs'),
  3.     cheerio = require("cheerio"),
  4.     parse = require('url').parse;
  6. // var root_path=process.argv[2];
  8. var url = "http://www.jeffjade.com/2016/03/30/104-front-end-tutorial/",
  9.     saveLinkArr = {},
  10.     errUrlList = []
  12. // Utility function that downloads a URL and invokes callback with the data.
  13. function download(url, callback) {
  14.     http.get(url, function(res) {
  15.         var data = "";
  17.         res.on('data', function(chunk) {
  18.             data += chunk;
  19.         });
  21.         res.on("end", function() {
  22.             callback(data);
  23.         });
  24.     }).on("error", function() {
  25.         callback(null);
  26.     });
  27. }
  29. download(url, function(data) {
  30.     if (data) {
  31.         var $ = cheerio.load(data);
  33.         $("tbody a").each(function(i, e) {
  34.             console.log(i, $(e).attr("href"));
  35.             saveLinkArr[i] = $(e).attr("href")
  36.         });
  38.         linkCount = saveLinkArr.length
  39.         jadeFilterLink(saveLinkArr)
  40.         console.log("url-link done !");
  41.     } else {
  42.         console.log("download error !")
  43.     };
  44. });
  46. function requestUrl(url, callback) {
  47.     var info = parse(url),
  48.         path = info.pathname + (info.search || ''),
  49.         options = {
  50.             host: info.hostname,
  51.             port: info.port || 80,
  52.             path: path,
  53.             method: 'GET'
  54.         },
  55.         req = null,
  56.         request_timeout = null;
  58.     request_timeout = setTimeout(function() {
  59.         request_timeout = null;
  60.         req.abort();
  61.         callback(new Error('Request timeout'), url);
  62.     }, 60000);
  64.     req = http.request(options, function(res) {
  65.         clearTimeout(request_timeout);
  66.         var chunks = [],
  67.             length = 0;
  68.         res.on('data', function(chunk) {
  69.             length += chunk.length;
  70.             chunks.push(chunk);
  71.         }).on('end', function() {
  72.             var data = new Buffer(length);
  73.             for (var i = 0, pos = 0, l = chunks.length; i < l; i++) {
  74.                 chunks[i].copy(data, pos);
  75.                 pos += chunks[i].length;
  76.             }
  77.             res.body = data;
  78.             callback(null, url);
  79.         }).on('error', function(err) {
  80.             callback(err, url);
  81.         });
  82.     }).on('error', function(err) {
  83.         // node0.5.x及以上，调用req.abort()会触发一次“socket hang up” error
  84.         // 所以需要判断是否超时，如果是超时，则无需再回调异常结果
  85.         if (request_timeout) {
  86.             clearTimeout(request_timeout);
  87.             callback(err, url);
  88.         }
  89.     });
  90.     req.end();
  91. };
  93. function callback(err, errUrl) {
  94.     if (err){
  95.         console.log(err + ' ' + errUrl)
  96.         errUrlList.push(errUrl + ' ' + err)
  97.         outPrint(errUrlList)
  98.     }
  99. }
  101. function jadeFilterLink(needFilterList) {
  102.     for (var key in needFilterList) {
  103.         // console.log("now turn:" + needFilterList[key])
  104.         (function(key) {
  105.             requestUrl(needFilterList[key], callback);
  106.         })(key)
  107.     }
  108. }
  110. // requestUrl("https://www.google.com/" , callback)
  112. function outPrint(resData) {
  113.     var filepath = './err_url_list.json'
  114.     var resJson = JSON.stringify(resData, null, 2)
  115.     fs.writeFile(filepath, resJson, function(e) {
  116.         if (e) throw e
  117.     });
  118. }
添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注