她就是我在密码里做的
基本上做网页抓取
我不知道这里发生了什么,是这个Web服务器把我踢出去了(看起来不像),还是我的代码写得很糟糕,v8引擎无法处理?(更有可能)
这是我的代码,如果有人想尝试运行大量的链接,我用https://hackertarget.com/extract-links/获取链接,然后把它放在一个文本文件中。
var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var axios = require('axios');
const fs = require('fs');
const readline = require('readline');
const timeout = require('./timeout')
var main = [];
async function processLineByLine() {
const rl = readline.createInterface({
input: fs.createReadStream('C:/Users/T440/Documents/crawl/links.txt'),
crlfDelay: Infinity
});
for await (const line of rl) {
if (line.startsWith('https://www.example.com')) {
var encodeLink = encodeURI(line);
const response = await axios.get(encodeLink).catch((err)=>{
var Status_ErrorsCatchaxios = {
"status Code": err.response.status ?error.response.status:"No status code available",
"Page title:": $('title').text()?$('title').text():'No title avaialble',
"Original Link": encodeLink?encodeLink:"No Original Link Available",
"errorCode":err
}
main.push(Status_ErrorsCatchaxios)
})
try {
const body = response.data;
if (response.status === 200) {
// Parse the document body
var $ = cheerio.load(body);
var Status_200 = {
"status Code": response.status,
"Page title:": $('title').text(),
"Original Link": encodeLink,
}
main.push(Status_200)
}
if (response.status === 302 || response.status === 404 || response.status === 500) {
// Parse the document body
var Status_Errors = {
"status Code": response.status,
"Page title:": $('title').text(),
"Original Link": encodeLink,
}
main.push(Status_Errors)
}
fs.writeFile("C:/Users/T440/Documents/crawl/output.json", JSON.stringify(main), (err) => {
if (err) console.log(err);
console.log("Successfully Written to File.");
});
} catch (error) {
var Status_ErrorsCatch = {
"status Code": response.status ?error.response.status:"No status code available",
"Page title:": $('title').text()?$('title').text():'No title avaialble',
"Original Link": encodeLink?encodeLink:"No Original Link Available",
}
main.push(Status_ErrorsCatch)
}
}
}
}
processLineByLine()
我相信我也有同样的问题。最初发现这个问题:cheerio v8“泄漏”内存。我认为这是Cheerio/V8中的内存泄漏。我还没有找到解决问题的方法,但我尝试了以下方法:
1和2的效果最好。我相信这只是我的代码,因为这是一个老问题。