package com.javaweb.spider.resolver;
|
|
import com.javaweb.common.utils.StringUtils;
|
import com.javaweb.spider.config.SpiderConstants;
|
import com.javaweb.spider.domain.SpiderConfig;
|
import com.javaweb.spider.domain.SpiderField;
|
import com.javaweb.spider.domain.SpiderFiledRule;
|
import org.apache.commons.collections.CollectionUtils;
|
import org.jsoup.nodes.Document;
|
import org.jsoup.nodes.Element;
|
import org.jsoup.select.Elements;
|
import us.codecraft.webmagic.Page;
|
import us.codecraft.webmagic.selector.Html;
|
|
import java.util.Comparator;
|
import java.util.List;
|
import java.util.regex.Pattern;
|
import java.util.stream.Collectors;
|
|
/**
|
* 解析处理普通的Html网页
|
*/
|
public class DefaultResolver implements Resolver {
|
|
@Override
|
public void process(Page page, SpiderConfig spiderConfig) {
|
List<SpiderField> fields = spiderConfig.getFieldsList();
|
|
Html pageHtml = page.getHtml();
|
Document pageDoc =page.getHtml().getDocument();
|
String link = page.getRequest().getUrl();
|
|
if(!spiderConfig.getEntryUrlsList().contains(link) || Pattern.matches(spiderConfig.getTargetRegex(), link)) {
|
//入口页面无需取值，入口页面只是用于发现目标url，除非入口页面匹配目标url的正则
|
for (SpiderField field : fields) {
|
String column=field.getField();//有可能取得同一个字段需要多个不同规则
|
//先判断该字段是否有值，如果没有在继续处理，有了就不用处理了
|
String checkVal = String.valueOf(page.getResultItems().getAll().get(column));
|
if(StringUtils.isNotEmpty(checkVal)){
|
continue;
|
}
|
String type = field.getExtractType();
|
if (StringUtils.isEmpty(type)||type.equals(SpiderConstants.FIELD_EXTRACT_TYPE_XPATH)) {
|
this.xpath_put(page, pageHtml, field);
|
} else if (type.equals(SpiderConstants.FIELD_EXTRACT_TYPE_CSS)) {
|
this.css_put(page, pageDoc, field);
|
} else if (type.equals(SpiderConstants.FIELD_EXTRACT_TYPE_CONSTANT)) {
|
this.constant_put(page,field);
|
} else {
|
}
|
}
|
page.putField("link", link);
|
}
|
if (StringUtils.isNotEmpty(spiderConfig.getTargetRegex())) {
|
if(spiderConfig.getCascade()==1 || spiderConfig.getEntryUrlsList().contains(link)){
|
//级联发现或者是入口URL才收集目标URL
|
page.addTargetRequests(page.getHtml().links().regex(spiderConfig.getTargetRegex()).all());
|
}
|
}
|
}
|
|
/**
|
* 处理爬取后原始的文本内容，比如替换截取等操作最终得到自己想要的字符串
|
* @param sourceValue
|
* @param field
|
* @return
|
*/
|
private String processValue(String sourceValue,SpiderField field){
|
if(StringUtils.isEmpty(sourceValue) || CollectionUtils.isEmpty(field.getFieldRules())){
|
return sourceValue;
|
}
|
List<SpiderFiledRule> rules = field.getFieldRules();
|
rules=rules.stream().sorted(Comparator.comparing(SpiderFiledRule::getSort)).collect(Collectors.toList());
|
for(SpiderFiledRule rule:rules){
|
if(SpiderConstants.FIELD_PROCESS_TYPE_REPLACE.equals(rule.getProcessType())){
|
sourceValue=sourceValue.replace(rule.getReplacereg(),StringUtils.isEmpty(rule.getReplacement())?"":rule.getReplacement());
|
}else if(SpiderConstants.FIELD_PROCESS_TYPE_SUBSTRING_AFTER.equals(rule.getProcessType())){
|
sourceValue=StringUtils.trim_end_exclu(sourceValue, rule.getSubstrTarget());
|
}else if(SpiderConstants.FIELD_PROCESS_TYPE_SUBSTRING_BEFORE.equals(rule.getProcessType())){
|
sourceValue=StringUtils.trim_before_exclu(sourceValue, rule.getSubstrTarget());
|
}else{}
|
}
|
return sourceValue;
|
}
|
private void xpath_put(Page page, Html pageHtml,SpiderField field) {
|
if (StringUtils.isNotEmpty(field.getExtractAttr())) {
|
String value=pageHtml.xpath(field.getExtractAttr()).get();
|
value= processValue(value,field);//处理替换
|
page.putField(field.getField(), value);
|
}
|
}
|
private void css_put(Page page,Document pageDoc,SpiderField field) {
|
String[] indexArr=field.getExtractIndex().split(",");
|
String resStr="";
|
if(!"1".equals(field.getExtractAttrFlag())){
|
//非根据属性名取值
|
for(String ix:indexArr){
|
if(StringUtils.isNotEmpty(ix)){
|
String tempRes="";
|
try{
|
tempRes= pageDoc.select(field.getExtractBy()).get(Integer.valueOf(ix)).html();
|
}catch (Exception ex){}
|
if(StringUtils.isNotEmpty(tempRes)){
|
resStr +=tempRes;
|
resStr +=",";
|
}
|
}
|
}
|
}else{
|
//根据属性名取值
|
for(String ix:indexArr) {
|
if (StringUtils.isNotEmpty(ix)) {
|
String tempRes="";
|
try {
|
tempRes= getAttributeByElement(pageDoc.select(field.getExtractBy()).get(Integer.valueOf(ix)),field.getExtractAttr());
|
}catch (Exception ex){}
|
if(StringUtils.isNotEmpty(tempRes)){
|
resStr +=tempRes;
|
resStr +=",";
|
}
|
}
|
}
|
|
}
|
if(resStr.endsWith(",")){
|
resStr=resStr.substring(0,resStr.length()-1);
|
}
|
//处理替换
|
resStr= processValue(resStr,field);
|
page.putField(field.getField(),resStr);
|
}
|
private void constant_put(Page page,SpiderField field) {
|
page.putField(field.getField(),field.getConstantValue());
|
}
|
|
private String getAttributeByElement(Element e, String attrName){
|
// 判断如果属性名是href或者src
|
String res = "";
|
if ("href".equals(attrName) || "src".equals(attrName)) {
|
// 因为要获取他们绝对路径
|
res = e.attr("abs:" + attrName);
|
} else {
|
//不是href或者src
|
res = e.attr(attrName);
|
}
|
return res;
|
}
|
}
|
