package com.javaweb.spider.controller; import java.util.List; import com.javaweb.common.utils.StringUtils; import com.javaweb.spider.domain.SpiderConfig; import com.javaweb.spider.service.ISpiderConfigService; import org.apache.commons.collections.CollectionUtils; import org.apache.shiro.authz.annotation.RequiresPermissions; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; import com.javaweb.common.annotation.Log; import com.javaweb.common.enums.BusinessType; import com.javaweb.spider.domain.SpiderMission; import com.javaweb.spider.service.ISpiderMissionService; import com.javaweb.common.core.controller.BaseController; import com.javaweb.common.core.domain.AjaxResult; import com.javaweb.common.utils.poi.ExcelUtil; import com.javaweb.common.core.page.TableDataInfo; /** * 爬虫任务Controller * * @author wujiyue * @date 2019-11-11 */ @Controller @RequestMapping("/spider/spiderMission") public class SpiderMissionController extends BaseController { private String prefix = "spider/spiderMission"; @Autowired private ISpiderMissionService spiderMissionService; @Autowired private ISpiderConfigService spiderConfigService; @RequiresPermissions("spider:spiderMission:view") @GetMapping() public String spiderMission() { return prefix + "/spiderMission"; } /** * 查询爬虫任务列表 */ @RequiresPermissions("spider:spiderMission:list") @PostMapping("/list") @ResponseBody public TableDataInfo list(SpiderMission spiderMission) { startPage(); List list = spiderMissionService.selectSpiderMissionList(spiderMission); return getDataTable(list); } /** * 导出爬虫任务列表 */ @RequiresPermissions("spider:spiderMission:export") @PostMapping("/export") @ResponseBody public AjaxResult export(SpiderMission spiderMission) { List list = spiderMissionService.selectSpiderMissionList(spiderMission); ExcelUtil util = new ExcelUtil(SpiderMission.class); return util.exportExcel(list, "spiderMission"); } /** * 新增爬虫任务 */ @GetMapping("/add") public String add() { return prefix + "/add"; } /** * 新增保存爬虫任务 */ @RequiresPermissions("spider:spiderMission:add") @Log(title = "爬虫任务", businessType = BusinessType.INSERT) @PostMapping("/add") @ResponseBody public AjaxResult addSave(SpiderMission spiderMission) { String entryUrls=spiderMission.getEntryUrls(); String[] arr=null; if(entryUrls.contains(",")){ //前台多条传过来用逗号分隔 arr=entryUrls.split(","); }else{ //前台多条传过来用换行分隔 arr=entryUrls.split("\r\n"); } if(arr==null||arr.length==0){ return AjaxResult.error("入口URL必填!多条用逗号分隔或者换行分隔!"); } boolean isUrlFlag=true; String tempUrl=""; String urlAll=""; for(String url:arr){ if(!isURL(url)){ isUrlFlag=false; tempUrl=url; break; }else{ urlAll+=url+","; } } if(!isUrlFlag){ return AjaxResult.error("["+tempUrl+"]不是一个有效的url!"); } if(isUrlFlag){ if(urlAll.endsWith(",")){ urlAll=urlAll.substring(0,urlAll.length()-1); spiderMission.setEntryUrls(urlAll); } } return toAjax(spiderMissionService.insertSpiderMission(spiderMission)); } private static boolean isURL(String str){ str = str.toLowerCase(); String regex = "(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"; return str.matches(regex); } /** * 修改爬虫任务 */ @GetMapping("/edit/{missionId}") public String edit(@PathVariable("missionId") Long missionId, ModelMap mmap) { SpiderMission spiderMission = spiderMissionService.selectSpiderMissionById(missionId); SpiderConfig config = spiderConfigService.selectSpiderConfigById(spiderMission.getSpiderConfigId()); if(config!=null){ mmap.put("spiderConfigName", config.getSpiderName()); } mmap.put("spiderMission", spiderMission); return prefix + "/edit"; } /** * 修改保存爬虫任务 */ @RequiresPermissions("spider:spiderMission:edit") @Log(title = "爬虫任务", businessType = BusinessType.UPDATE) @PostMapping("/edit") @ResponseBody public AjaxResult editSave(SpiderMission spiderMission) { String entryUrls=spiderMission.getEntryUrls(); String[] arr=null; if(entryUrls.contains(",")){ //前台多条传过来用逗号分隔 arr=entryUrls.split(","); }else{ //前台多条传过来用换行分隔 arr=entryUrls.split("\r\n"); } if(arr==null||arr.length==0){ return AjaxResult.error("入口URL必填!多条用逗号分隔或者换行分隔!"); } boolean isUrlFlag=true; String tempUrl=""; String urlAll=""; for(String url:arr){ if(!isURL(url)){ isUrlFlag=false; tempUrl=url; break; }else{ urlAll+=url+","; } } if(!isUrlFlag){ return AjaxResult.error("["+tempUrl+"]不是一个有效的url!"); } if(isUrlFlag){ if(urlAll.endsWith(",")){ urlAll=urlAll.substring(0,urlAll.length()-1); spiderMission.setEntryUrls(urlAll); } } return toAjax(spiderMissionService.updateSpiderMission(spiderMission)); } /** * 删除爬虫任务 */ @RequiresPermissions("spider:spiderMission:remove") @Log(title = "爬虫任务", businessType = BusinessType.DELETE) @PostMapping( "/remove") @ResponseBody public AjaxResult remove(String ids) { return toAjax(spiderMissionService.deleteSpiderMissionByIds(ids)); } /** * 启动爬虫任务 */ @PostMapping( "/runSpiderMission/{missionId}") @ResponseBody public AjaxResult runSpiderMission(@PathVariable("missionId") String missionId) { return spiderMissionService.runSpiderMission(missionId); } /** * 打开任务选择爬虫界面 * @return */ @GetMapping("/openSelectSpiderConfig") public String openSelectSpiderConfig() { return prefix + "/selectSpiderConfig"; } }