package com.javaweb.spider.domain;
|
|
import com.javaweb.spider.config.SpiderConstants;
|
import com.javaweb.spider.fast.FastConfigContext;
|
import com.javaweb.spider.util.FackUserAgentUtil;
|
import lombok.Data;
|
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.lang3.StringUtils;
|
import com.javaweb.common.annotation.Excel;
|
import com.javaweb.common.core.domain.BaseEntity;
|
import org.assertj.core.util.Lists;
|
import us.codecraft.webmagic.pipeline.Pipeline;
|
import us.codecraft.webmagic.proxy.Proxy;
|
|
import java.util.ArrayList;
|
import java.util.HashMap;
|
import java.util.List;
|
import java.util.Map;
|
|
/**
|
* 爬虫配置对象 spider_config
|
*
|
* @author wujiyue
|
* @date 2019-11-11
|
*/
|
@Data
|
public class SpiderConfig extends BaseEntity
|
{
|
private static final long serialVersionUID = 1L;
|
|
/** 爬虫配置ID */
|
private Long id;
|
/** 爬虫编码 */
|
@Excel(name = "爬虫编码")
|
private String spiderCode;
|
/** 爬虫名称 */
|
@Excel(name = "爬虫名称")
|
private String spiderName;
|
|
/** 入口地址 */
|
private String entryUrls;
|
|
/** 目标URL正则 */
|
private String targetRegex;
|
|
/** 存储的表名 */
|
@Excel(name = "存储的表名")
|
private String tableName;
|
|
/** 网站根域名 */
|
@Excel(name = "网站根域名")
|
private String domain;
|
|
|
/** 字符集 */
|
@Excel(name = "字符集")
|
private String charset = "utf8";
|
|
/** 睡眠时间(ms) */
|
@Excel(name = "睡眠时间(ms)")
|
private Long sleepTime= 1000L;
|
|
/** 重试次数 */
|
@Excel(name = "重试次数")
|
private Integer retryTimes = 2;
|
|
/** 线程数量 */
|
@Excel(name = "线程数量")
|
private Long threadCount = 1L;
|
|
/** 使用代理 */
|
@Excel(name = "使用代理")
|
private Integer useProxy =0 ;
|
@Excel(name = "打印日志")
|
private Integer showLog=1;
|
public Integer getShowLog(){
|
return this.showLog;
|
}
|
/**
|
* 退出方式{DURATION:爬虫持续的时间,URL_COUNT:抓取到的url数量}
|
*/
|
private String exitWay = ExitWayEnum.URL_COUNT.toString();
|
/**
|
* 对应退出方式，当exitWay = URL_COUNT时，该值表示url数量，当exitWay = DURATION时，该值表示爬虫持续的时间
|
*/
|
private int count;
|
|
private Integer cascade=0;//默认0表示支持入口URL发现目标URL，而不从目标URL级联收集URL
|
|
public Integer getCascade(){
|
return this.cascade;
|
}
|
|
private List<Cookie> cookies = new ArrayList<>();
|
private Map<String, String> headers = new HashMap<>();
|
private String ua = FackUserAgentUtil.getUserAgent();
|
/** 入口地址集合 */
|
private List<String> entryUrlsList= Lists.newArrayList();
|
|
private List<SpiderField> fieldsList=Lists.newArrayList();
|
|
public List<SpiderField> getFieldsList(){
|
return this.fieldsList;
|
}
|
|
public String getTargetRegex(){
|
return this.targetRegex;
|
}
|
|
public Integer getUseProxy(){
|
return this.useProxy;
|
}
|
|
public List<Proxy> getProxyList(){
|
return this.proxyList;
|
}
|
|
private List<Pipeline> pipelineList=Lists.newArrayList();
|
|
public List<Pipeline> getPipelineList(){
|
return this.pipelineList;
|
}
|
|
/**
|
* 是否转存图片，当选择true时会自动过滤原文中的img链接，调用端可选择将图片下载后替换掉原来的图片
|
*/
|
private boolean convertImg = false;
|
private List<Proxy> proxyList = new ArrayList<>();
|
|
private String userId;//扩展字段，无数据库字段对应。对应于爬取某个人的博客的作者uid
|
|
public String getUserId(){
|
return this.userId;
|
}
|
|
public String getDomain(){
|
return this.domain;
|
}
|
|
public Long getThreadCount(){
|
return this.threadCount;
|
}
|
|
public String getEntryUrls(){
|
return this.entryUrls;
|
}
|
|
public static SpiderConfig create(){
|
return new SpiderConfig();
|
}
|
|
public List<String> getEntryUrlsList(){
|
return this.entryUrlsList;
|
}
|
|
public String getSpiderCode(){
|
return this.spiderCode;
|
}
|
|
public String getSpiderName(){
|
return this.spiderName;
|
}
|
|
public String getExitWay(){
|
return this.exitWay;
|
}
|
|
public void setExitWay(String exitWay) {
|
this.exitWay = exitWay;
|
}
|
|
public SpiderConfig setExitWay(ExitWayEnum exitWay) {
|
this.exitWay = exitWay.toString();
|
this.count = exitWay.getDefaultCount();
|
return this;
|
}
|
|
|
public SpiderConfig setConvertImg(boolean convertImg) {
|
this.convertImg = convertImg;
|
return this;
|
}
|
|
public int getCount(){
|
return this.count;
|
}
|
|
public SpiderConfig setCount(int count) {
|
this.count = count;
|
return this;
|
}
|
|
public Map<String, String> getHeaders(){
|
return this.headers;
|
}
|
|
public SpiderConfig setHeader(String key, String value) {
|
Map<String, String> headers = this.getHeaders();
|
headers.put(key, value);
|
return this;
|
}
|
|
public SpiderConfig setHeader(String headersStr) {
|
if (StringUtils.isNotEmpty(headersStr)) {
|
String[] headerArr = headersStr.split("\r\n");
|
for (String s : headerArr) {
|
String[] header = s.split("=");
|
setHeader(header[0], header[1]);
|
}
|
}
|
return this;
|
}
|
|
public List<Cookie> getCookies(){
|
return this.cookies;
|
}
|
|
public SpiderConfig setCookie(String domain, String key, String value) {
|
List<Cookie> cookies = this.getCookies();
|
cookies.add(new Cookie(domain, key, value));
|
return this;
|
}
|
|
public SpiderConfig setCookie(String cookiesStr) {
|
if (StringUtils.isNotEmpty(cookiesStr)) {
|
List<Cookie> cookies = this.getCookies();
|
String[] cookieArr = cookiesStr.split(";");
|
for (String aCookieArr : cookieArr) {
|
String[] cookieNode = aCookieArr.split("=");
|
if (cookieNode.length <= 1) {
|
continue;
|
}
|
cookies.add(new Cookie(cookieNode[0].trim(), cookieNode[1].trim()));
|
}
|
}
|
return this;
|
}
|
|
|
private void addProxy(Proxy proxy) {
|
if (this.useProxy == 0 || null == this.useProxy || null == proxy) {
|
return;
|
}
|
proxyList.add(proxy);
|
}
|
|
public SpiderConfig setProxy(String proxyStr) {
|
if (this.useProxy == 0 || null == this.useProxy || proxyStr == null) {
|
return this;
|
}
|
String[] proxyArr = proxyStr.split(",");
|
for (String s : proxyArr) {
|
String[] proxy = s.split(":");
|
if (proxy.length == 2) {
|
this.addProxy(new Proxy(proxy[0], Integer.parseInt(proxy[1])));
|
} else if (proxy.length == 4) {
|
this.addProxy(new Proxy(proxy[0], Integer.parseInt(proxy[1]), proxy[2], proxy[3]));
|
}
|
}
|
return this;
|
}
|
|
|
public SpiderConfig setEntryUrls(String entryUrls) {
|
this.entryUrls = entryUrls;
|
if(StringUtils.isNotEmpty(entryUrls)){
|
String[] arr=entryUrls.split(",");
|
for(String s:arr){
|
if(StringUtils.isNotEmpty(s)){
|
entryUrlsList.add(s);
|
}
|
}
|
}
|
return this;
|
}
|
public SpiderConfig addEntryUrl(String entryUrl) {
|
this.entryUrlsList.add(entryUrl);
|
return this;
|
}
|
public void setEntryUrlsList(List<String> entryUrlsList) {
|
this.entryUrlsList = entryUrlsList;
|
}
|
|
public SpiderConfig setFieldsList(List<SpiderField> fieldsList) {
|
if(CollectionUtils.isEmpty(fieldsList)){
|
return this;
|
}
|
this.fieldsList.addAll(fieldsList);
|
return this;
|
}
|
|
/**
|
* 增加一个爬取字段
|
* @param field
|
* @return
|
*/
|
public SpiderConfig addField(SpiderField field) {
|
if(field==null||StringUtils.isEmpty(field.getField())){
|
return this;
|
}
|
this.fieldsList.add(field);
|
return this;
|
}
|
/**
|
* 增加一个数据管道
|
* @param pipeline
|
* @return
|
*/
|
public SpiderConfig addPipeline(Pipeline pipeline) {
|
if(pipeline==null){
|
return this;
|
}
|
this.pipelineList.add(pipeline);
|
return this;
|
}
|
|
/**
|
* 增加一个xpath提取规则字段
|
* @param field
|
* @param fieldName
|
* @param xpath
|
* @return
|
*/
|
public SpiderConfig addField(String field,String fieldName,String xpath) {
|
SpiderField spiderField=new SpiderField();
|
spiderField.setField(field);
|
spiderField.setFieldName(fieldName);
|
spiderField.setExtractType(SpiderConstants.FIELD_EXTRACT_TYPE_XPATH);
|
spiderField.setExtractBy(xpath);
|
this.fieldsList.add(spiderField);
|
return this;
|
}
|
|
public SpiderConfig setTargetRegex(String targetRegex) {
|
this.targetRegex = targetRegex;
|
return this;
|
}
|
|
public SpiderConfig setThreadCount(Long threadCount) {
|
this.threadCount = threadCount;
|
return this;
|
}
|
|
public SpiderConfig setShowLog(Integer showLog) {
|
this.showLog = showLog;
|
return this;
|
}
|
|
public SpiderConfig setSleepTime(Long sleepTime) {
|
this.sleepTime = sleepTime;
|
return this;
|
}
|
|
public SpiderConfig setDomain(String domain) {
|
this.domain = domain;
|
return this;
|
}
|
|
public SpiderConfig setCharset(String charset) {
|
this.charset = charset;
|
return this;
|
}
|
|
public SpiderConfig setRetryTimes(Integer retryTimes) {
|
this.retryTimes = retryTimes;
|
return this;
|
}
|
|
public SpiderConfig setUseProxy(Integer useProxy) {
|
this.useProxy = useProxy;
|
return this;
|
}
|
|
public SpiderConfig setCascade(Integer cascade) {
|
this.cascade = cascade;
|
return this;
|
}
|
|
/**
|
* 爬取某个人博客时候博主的用户id
|
* @param userId
|
* @return
|
*/
|
public SpiderConfig setUserId(String userId) {
|
this.userId = userId;
|
return FastConfigContext.replaceConfigUid(this);
|
}
|
public Long getId() {
|
return id;
|
}
|
public void setId(Long id) {
|
this.id = id;
|
}
|
|
public void setSpiderCode(String spiderCode) {
|
this.spiderCode = spiderCode;
|
}
|
|
|
public void setSpiderName(String spiderName) {
|
this.spiderName = spiderName;
|
}
|
public String getTableName() {
|
return tableName;
|
}
|
public void setTableName(String tableName) {
|
this.tableName = tableName;
|
}
|
|
public void setCookies(List<Cookie> cookies) {
|
this.cookies = cookies;
|
}
|
public void setHeaders(Map<String, String> headers) {
|
this.headers = headers;
|
}
|
public String getUa() {
|
return ua;
|
}
|
public void setUa(String ua) {
|
this.ua = ua;
|
}
|
|
public void setPipelineList(List<Pipeline> pipelineList) {
|
this.pipelineList = pipelineList;
|
}
|
|
public void setProxyList(List<Proxy> proxyList) {
|
this.proxyList = proxyList;
|
}
|
|
public String getCharset() {
|
return charset;
|
}
|
public Long getSleepTime() {
|
return sleepTime;
|
}
|
public Integer getRetryTimes() {
|
return retryTimes;
|
}
|
|
public boolean isConvertImg() {
|
return convertImg;
|
}
|
|
}
|
