cherrio.load(html)
moajs
unescape
Mongodb.github.io/node-mongodb-native/3.0/api
const {MongoClient}=require('mongodb').MongoClient
const db= await MongoClient.connect('mogondb://localhost:27017/acfun')
await db.collection('articles').findOneAndUpdate({
acfunid:id
},{
content.content,
articleContentHtml: articleContent,
},{
upsert:true,returnNewValue:true
}
})
边学边讲,编码问题没有解决，存文章到mongodb也没有解决。
1. 工具
node_redis
ioredis
2.
Redis=require('ioredis')
const redis= new Redis()
redis.set 与命令相同
(async()=>{
const abc = await redis.get('abc')
const hgetall = await redis.hgetall('myhash1')
const smembers = await redis.smembers('mset')
let set = new Set(...smembers )
})()
.then(r=>{
})
.catch(e=>{}）
3. bluebird 异步代码
4. 爬虫，有目标，制定爬取计划，随机爬取，防止IP被封禁。
5.redis_service
const Redis = require('ioredis')
const redis = new Redis();
const ACFUN_ID_SET_REDIS_KEY:'acfun_id_set';
async function generateAcfunIdsToRedis(min,max){
for(let i= min;i<max;i++){
for(let j= 0;j<10000;j++){ -- 解决IO瓶颈
arr.push(i*10000+j)
}
await redis.sadd(ACFUN_ID_SET_REDIS_KEY,...arr)
}
}
async function getRandomAcfunIds(count){
}
moule.export={
generateAcfunIdsToRedis,
}
6.spider
const RedisService = require('./redis_service')
switch(process.argv[2]){
case 'generate_ids':
RedisService.generateAcfunIdsToRedis(Number(process.argv[3]),Number(process.argv[4]))
.then(r=>{
console.log('done')
process.exit(0)
}
)
.catch(e=>{
console.log(1)
process.exit(1)
}
}
7. 爬虫工具：axios, cherrio
8. for(let id of ids){ -- foreach
}
1. redis-cli
info
info memory
2. set abc 123
get abc
flushall
del abc
expire abc 10
ttl abc
3. 聊天室 socket, redis
4. set结构 --- O(1)
sadd mset 1
smembers mset
自动去重
set inventory 10
get inventory
incrby inventory -1
sadd successfully_ids aaa
sismember successfully_ids aaa
scard mset --元素个数
srem mset 1
spop --随机取值并移除
srangmember mset -- 随机取值
5. sorted set --- O(log(N))
zadd sortset1 score member
zadd sortset1 100 a
zadd sortset1 200 b -100 c 0 d
zrangebyscore sortset1 0 100 -- 正序
zrevrangebysocre sortset 100 0 -- 倒序zrangebyscore sortset1 0 100 WITHSCORES
6. Hash set -- O(1)
hset myhash1 abc 123 def 456 ghi 789
hgetall myhash1
hget myhash1 abc
hdel myhash1 abc
hlen myhash1
hexists myhash1 abc
hkeys myhash1
hvals myhash1
expire myhash1 10
总结：
内存数据库
移植性
原子性
分布式
集群部署
