package com.javaweb.spider.service.impl; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import com.google.common.collect.Maps; import com.javaweb.common.annotation.DataScope; import com.javaweb.common.core.domain.AjaxResult; import com.javaweb.common.core.domain.ICallBack; import com.javaweb.common.utils.DateUtils; import com.javaweb.common.utils.spring.SpringUtils; import com.javaweb.framework.util.ShiroUtils; import com.javaweb.spider.backend.SpiderBackendService; import com.javaweb.spider.config.SpiderConstants; import com.javaweb.spider.domain.SpiderConfig; import com.javaweb.spider.domain.SpiderField; import com.javaweb.spider.mapper.SpiderConfigMapper; import com.javaweb.spider.mapper.SpiderFieldMapper; import com.javaweb.spider.processer.AbstractProcessor; import com.javaweb.spider.processer.DefalutProcessor; import com.javaweb.system.domain.SysUser; import org.apache.commons.lang3.StringUtils; import org.assertj.core.util.Lists; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; import com.javaweb.spider.mapper.SpiderMissionMapper; import com.javaweb.spider.domain.SpiderMission; import com.javaweb.spider.service.ISpiderMissionService; import com.javaweb.common.core.text.Convert; /** * 爬虫任务Service业务层处理 * * @author wujiyue * @date 2019-11-11 */ @Service public class SpiderMissionServiceImpl implements ISpiderMissionService { @Autowired private SpiderMissionMapper spiderMissionMapper; @Autowired private SpiderConfigMapper spiderConfigMapper; @Autowired private SpiderFieldMapper spiderFieldMapper; /** * 查询爬虫任务 * * @param missionId 爬虫任务ID * @return 爬虫任务 */ @Override public SpiderMission selectSpiderMissionById(Long missionId) { return spiderMissionMapper.selectSpiderMissionById(missionId); } /** * 查询爬虫任务列表 * * @param spiderMission 爬虫任务 * @return 爬虫任务 */ @Override @DataScope(deptAlias = "a",userAlias = "a") public List selectSpiderMissionList(SpiderMission spiderMission) { return spiderMissionMapper.selectSpiderMissionList(spiderMission); } /** * 新增爬虫任务 * * @param spiderMission 爬虫任务 * @return 结果 */ @Override public int insertSpiderMission(SpiderMission spiderMission) { SysUser user= ShiroUtils.getSysUser(); spiderMission.setUserId(user.getUserId().toString()); spiderMission.setDeptId(user.getDeptId().toString()); spiderMission.setStatus(SpiderConstants.SPIDER_MISSION_STATUS_WAIT); spiderMission.setCreateBy(user.getUserName()); spiderMission.setCreateTime(DateUtils.getNowDate()); return spiderMissionMapper.insertSpiderMission(spiderMission); } /** * 修改爬虫任务 * * @param spiderMission 爬虫任务 * @return 结果 */ @Override public int updateSpiderMission(SpiderMission spiderMission) { return spiderMissionMapper.updateSpiderMission(spiderMission); } /** * 删除爬虫任务对象 * * @param ids 需要删除的数据ID * @return 结果 */ @Override public int deleteSpiderMissionByIds(String ids) { return spiderMissionMapper.deleteSpiderMissionByIds(Convert.toStrArray(ids)); } /** * 删除爬虫任务信息 * * @param missionId 爬虫任务ID * @return 结果 */ @Override public int deleteSpiderMissionById(Long missionId) { return spiderMissionMapper.deleteSpiderMissionById(missionId); } @Override public AjaxResult runSpiderMission(String missionId) { SpiderMission mission=this.selectSpiderMissionById(Long.valueOf(missionId)); if(mission!=null){ if(SpiderConstants.SPIDER_MISSION_STATUS_RUNNING.equals(mission.getStatus())){ return AjaxResult.error("该任务正在运行中!"); } /* Long configId=mission.getSpiderConfigId(); SpiderConfig config = spiderConfigMapper.selectSpiderConfigById(configId); SpiderField queryForm=new SpiderField(); queryForm.setConfigId(config.getId()); List fields = spiderFieldMapper.selectSpiderFieldList(queryForm); config.setFieldsList(fields); String entryUrls=mission.getEntryUrls(); List urls= Lists.newArrayList(); if(StringUtils.isNotEmpty(entryUrls)){ String[] arr=entryUrls.split(","); for(String s:arr){ if(StringUtils.isNotEmpty(s)){ urls.add(s); } } } config.setEntryUrlsList(urls); config.setExitWay(mission.getExitWay()); Long c= mission.getExitWayCount(); if(c==null){ c=0L; } config.setCount(Integer.valueOf(c.toString())); AbstractProcessor processor=new DefalutProcessor(config,missionId.toString()); mission.setStatus(SpiderConstants.SPIDER_MISSION_STATUS_RUNNING); mission.setStartTime(new Date()); spiderMissionMapper.updateSpiderMission(mission); new Thread(new Runnable() { @Override public void run() { CopyOnWriteArrayList> datas = processor.execute(); mission.setEndTime(new Date()); mission.setStatus(SpiderConstants.SPIDER_MISSION_STATUS_DONE); mission.setSuccessNum(Long.valueOf(datas.size())); Long count=(mission.getEndTime().getTime()-mission.getStartTime().getTime())/1000; mission.setTimeCost(count.toString()); spiderMissionMapper.updateSpiderMission(mission); } }).start();*/ SpiderCallBack spiderCallBack= new SpiderCallBack(); SpiderBackendService spiderBackendService=new SpiderBackendService(missionId,spiderCallBack); spiderBackendService.start(); } return AjaxResult.success(); } public class SpiderCallBack implements ICallBack{ Map params= Maps.newConcurrentMap(); @Override public void onSuccess() { System.out.println(">>>>>>>>>>>>>done>>>>>>>>>>>>>>"); CopyOnWriteArrayList> datas=(CopyOnWriteArrayList>)params.get("datas"); System.out.println(">>>>>>>>>>>>>"+datas.size()+">>>>>>>>>>>>>>"); } @Override public void onFail() { } @Override public Map setParams(Map map) { params.clear(); params.putAll(map); return params; } } }