package com.javaweb.spider.test;
|
|
import com.google.common.collect.Maps;
|
import com.javaweb.common.core.domain.ICallBack;
|
import com.javaweb.spider.backend.FastSpiderBackendService;
|
import com.javaweb.spider.config.SpiderConstants;
|
import com.javaweb.spider.domain.ExitWayEnum;
|
import com.javaweb.spider.domain.SpiderConfig;
|
|
import java.util.LinkedHashMap;
|
import java.util.Map;
|
import java.util.concurrent.CopyOnWriteArrayList;
|
|
public class Test {
|
|
public static void main(String[] args) {
|
SpiderConfig config=SpiderConfig.create()
|
.setEntryUrls(SpiderConstants.URL_CNBLOGS)//设置入口地址
|
.setTargetRegex(SpiderConstants.TARGET_URL_CNBLOGS)//设置目标url正则
|
.addField("title","标题","//a[@id=cb_post_title_url]/html()")//配置数据提取规则
|
.setExitWay(ExitWayEnum.URL_COUNT)
|
.setCount(3)//爬取10条数据就结束
|
.setShowLog(1)//关闭爬取内容日志展示
|
/*.setSleepTime(500L) //睡眠时间2秒
|
.setCascade(true) //开启子页面链接发现
|
.setRetryTimes(2) //失败重试次数
|
.setCharset("utf8") // 设置字符集
|
.addPipeline(new Pipeline() { // 这里单纯的打印输出结果;可以自定义保存到数据库等。
|
@Override
|
public void process(ResultItems resultItems, Task task) {
|
Map<String, Object> map = resultItems.getAll();
|
for (Map.Entry<String, Object> entry : map.entrySet()) {
|
System.out.println(entry.getKey() + " : " + entry.getValue());
|
}
|
}
|
})
|
.setThreadCount(2L)*/;//开启2个线程
|
|
TestCallBack callBack=new TestCallBack();
|
FastSpiderBackendService spider=new FastSpiderBackendService(config,callBack);
|
spider.start();
|
}
|
|
public static class TestCallBack implements ICallBack {
|
Map params= Maps.newConcurrentMap();
|
@Override
|
public void onSuccess() {
|
System.out.println(">>>>>>>>>>>>>job done>>>>>>>>>>>>>>");
|
CopyOnWriteArrayList<LinkedHashMap<String, String>> datas=(CopyOnWriteArrayList<LinkedHashMap<String, String>>)params.get("datas");
|
System.out.println(">>>>>>>>>>>>>"+datas.size()+">>>>>>>>>>>>>>>");
|
}
|
|
@Override
|
public void onFail() {
|
|
}
|
@Override
|
public Map setParams(Map map) {
|
params.clear();
|
params.putAll(map);
|
return params;
|
}
|
}
|
}
|
