crawler.html 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. <!DOCTYPE HTML>
  2. <html>
  3. <head>
  4. <meta charset="utf-8">
  5. <meta name="renderer" content="webkit|ie-comp|ie-stand">
  6. <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  7. <meta name="viewport" content="width=device-width,initial-scale=1,minimum-scale=1.0,maximum-scale=1.0,user-scalable=no" />
  8. <meta http-equiv="Cache-Control" content="no-siteapp" />
  9. <!--[if lt IE 9]>
  10. <script type="text/javascript" src="${ctx}/assets/lib/html5.js"></script>
  11. <script type="text/javascript" src="${ctx}/assets/lib/respond.min.js"></script>
  12. <script type="text/javascript" src="${ctx}/assets/lib/PIE_IE678.js"></script>
  13. <![endif]-->
  14. <link rel="stylesheet" type="text/css" href="${ctx}/assets/static/h-ui/css/H-ui.min.css" />
  15. <link rel="stylesheet" type="text/css" href="${ctx}/assets/static/h-ui.admin/css/H-ui.admin.css" />
  16. <link rel="stylesheet" type="text/css" href="${ctx}/assets/lib/Hui-iconfont/1.0.7/iconfont.css" />
  17. <link rel="stylesheet" type="text/css" href="${ctx}/assets/lib/icheck/icheck.css" />
  18. <link rel="stylesheet" type="text/css" href="${ctx}/assets/static/h-ui.admin/skin/default/skin.css" id="skin" />
  19. <link rel="stylesheet" type="text/css" href="${ctx}/assets/static/h-ui.admin/css/style.css" />
  20. <link rel="stylesheet" type="text/css" media="screen" href="${ctx}/assets/lib/zTree/v3/css/zTreeStyle/zTreeStyle.css" >
  21. <script type="text/ecmascript" src="${ctx}/assets/lib/jquery/1.9.1/jquery.js"></script>
  22. <script type="text/javascript" src="${ctx}/assets/static/h-ui/js/H-ui.js"></script>
  23. <link rel="stylesheet" type="text/css" media="screen" href="${ctx}/assets/lib/jqueryui/jquery-ui.css" />
  24. <link rel="stylesheet" type="text/css" media="screen" href="${ctx}/assets/lib/jqgrid/css/ui.jqgrid.css" />
  25. <script type="text/ecmascript" src="${ctx}/assets/lib/jqueryui/jquery-ui.js"></script>
  26. <script type="text/ecmascript" src="${ctx}/assets/lib/jqgrid/js/i18n/grid.locale-cn.js"></script>
  27. <script type="text/ecmascript" src="${ctx}/assets/lib/jqgrid/js/jquery.jqGrid.min.js"></script>
  28. <script type="text/javascript" src="${ctx}/assets/js/base.js"></script>
  29. <script type="text/javascript" src="${ctx}/assets/lib/zTree/v3/js/jquery.ztree.all-3.5.min.js"></script>
  30. <script type="text/javascript" src="${ctx}/assets/lib/layer/3.0.3/layer.js"></script>
  31. <!--[if IE 6]>
  32. <script type="text/javascript" src="${ctx}/assets/lib/DD_belatedPNG_0.0.8a-min.js" ></script>
  33. <script>DD_belatedPNG.fix('*');</script>
  34. <![endif]-->
  35. <script type="text/javascript">
  36. $(function() {
  37. $(window).resize(function(){
  38. $("#jqGrid").setGridWidth($(window).width()-40); 
  39. });
  40. $.crawlerTypeInit=function(){
  41. var crawlerType=$("#crawlertype").val();
  42. if(crawlerType=='list'){
  43. $("#tr_infolinkselect").show();
  44. $("#tr_releasetimeselect").show();
  45. }else{
  46. $("#tr_infolinkselect").hide();
  47. $("#tr_releasetimeselect").hide();
  48. }
  49. }
  50. $("#jqGrid").jqGrid({
  51. url : '${ctx}/myconsole/crawler/getByPage',
  52. editurl:"${ctx}/myconsole/crawler/saveOrUpdate",
  53. mtype : "POST",
  54. datatype : "json",
  55. caption:'信息采集',
  56. //multiselect: true,
  57. prmNames : {
  58. id: "keyid", 
  59. oper:"oper",    
  60. addoper:"add",
  61. editoper:"edit",
  62. deloper:"del" 
  63. },
  64. colModel : [ {
  65. label : '编号',
  66. name : 'keyid',
  67. hidden : true,
  68. key : true,
  69. width : 50
  70. }, {
  71. label : '模板名称',
  72. name : 'crawlername',
  73. editable: true,
  74. editrules:{required:true},
  75. formoptions:{
  76. elmsuffix:'*'
  77. },
  78. width : 80
  79. },{
  80. label : '模板描述',
  81. name : 'crawlerdesc',
  82. editable: true,
  83. width : 80
  84. },{
  85. label : '采集引擎',
  86. name : 'crawlermode',
  87. editable: true,
  88. formatter:"select",
  89. edittype: "select",
  90. editoptions: {
  91. value: "${crawlerMode}"
  92. },
  93. stype:"select",
  94. searchoptions:{
  95. value: "${crawlerMode}"
  96. },
  97. width : 50
  98. },{
  99. label : '模板类型',
  100. name : 'crawlertype',
  101. editable: true,
  102. formatter:"select",
  103. edittype: "select",
  104. editoptions: {
  105. value: "${crawlerType}"
  106. },
  107. stype:"select",
  108. searchoptions:{
  109. value: "${crawlerType}"
  110. },
  111. width : 50
  112. },{
  113. label : '信息链接选择器',
  114. name : 'infolinkselect',
  115. editable: true,
  116. width : 140
  117. },{
  118. label : '发布时间选择器',
  119. name : 'releasetimeselect',
  120. editable: true,
  121. width : 140,
  122. formoptions:{
  123. elmsuffix:'选择器采用链接选择器相对路径;优先使用内容页发布时间,获取不到时将使用列表页发布时间'
  124. }
  125. },{
  126. label : '更新时间',
  127. name : 'updatetime',
  128. editable: false,
  129. width : 50
  130. }],
  131. rownumbers : true,//添加左侧行号
  132. viewrecords : true,//是否在浏览导航栏显示记录总数
  133. rowNum : 10,//每页显示记录数
  134. rowList : [10,15,20,30,40,50,1000],//用于改变显示行数的下拉列表框的元素数组。
  135. autowidth : true,
  136. height : 'auto',
  137. onSelectRow : function( rowid ) {
  138. if(rowid.length>10){
  139. var row = $(this).getRowData(rowid);
  140. $("#jqGrid1").setColProp('crawlerid',{editoptions:{value:rowid+":a"}});
  141. $("#jqGrid1").setCaption("内容页元素--"+row.crawlername);
  142. $("#jqGrid1").jqGrid('setGridParam',{url: "${ctx}/myconsole/crawlerfield/getByPage?crawlerID="+rowid});
  143. $("#jqGrid1").trigger("reloadGrid");
  144. $("#jqGrid2").setColProp('crawlerid',{editoptions:{value:rowid+":a"}});
  145. $("#jqGrid2").setCaption("栏目数据--"+row.crawlername);
  146. $("#jqGrid2").jqGrid('setGridParam',{url: "${ctx}/myconsole/crawlercolumn/getByPage?crawlerID="+rowid});
  147. $.ajax({
  148. async: false,
  149. type : "POST",
  150. url : "${ctx}/myconsole/crawler/getColumnOptions?crawlerID="+rowid,
  151. dataType : 'text',
  152. success : function(data) {
  153. $("#jqGrid2").setColProp('columnid', {editoptions:{value:data} });
  154. }
  155. });
  156. $.ajax({
  157. async: false,
  158. type : "POST",
  159. url : "${ctx}/myconsole/crawler/getSiteOptions?crawlerID="+rowid,
  160. dataType : 'text',
  161. success : function(data) {
  162. $("#jqGrid2").setColProp('siteid', {editoptions:{value:data} });
  163. }
  164. });
  165. $("#jqGrid2").trigger("reloadGrid");
  166. }
  167. },
  168. pager : "#jqGridPager"
  169. });
  170. $('#jqGrid').navGrid('#jqGridPager',
  171. // the buttons to appear on the toolbar of the grid
  172. {
  173. edit : true,
  174. add : true,
  175. del : true,
  176. search : true,
  177. refresh : true,
  178. view : true,
  179. position : "left",
  180. cloneToTop : true
  181. },
  182. // options for the Edit Dialog
  183. {
  184. recreateForm : true,
  185. checkOnUpdate : true,
  186. checkOnSubmit : true,
  187. closeAfterEdit : true,
  188. width:800,
  189. afterShowForm : function(formid) {
  190. $.crawlerTypeInit();
  191. $("#crawlertype").change(function(){
  192. $.crawlerTypeInit();
  193. })
  194. },
  195. serializeEditData: function(postdata) {
  196. //myconsole.log(postdata);
  197. $.each(postdata,function (index, value) {
  198. //myconsole.log(index);
  199. if(index!="jqGrid_id" && index!="oper" && index!="keyid"){
  200. modifyJosnKey(postdata,index,"crawler."+index);
  201. }
  202. if(index=="keyid"){
  203. postdata["crawler."+index]=value
  204. }
  205. })
  206. //myconsole.log(postdata);
  207. return postdata;
  208. },
  209. errorTextFormat : function(data) {
  210. return 'Error: ' + data.responseText
  211. }
  212. },
  213. // options for the Add Dialog
  214. {
  215. closeAfterAdd : true,
  216. recreateForm : true,
  217. width:800,
  218. afterShowForm : function(formid) {
  219. $.crawlerTypeInit();
  220. $("#crawlertype").change(function(){
  221. $.crawlerTypeInit();
  222. })
  223. },
  224. beforeSubmit: function(postdata, formid){
  225. $.each(postdata,function (index, value) {
  226. if(index!="jqGrid_id" && index!="oper"){
  227. modifyJosnKey(postdata,index,"crawler."+index);
  228. }
  229. })
  230. return[true,''];
  231. },
  232. errorTextFormat : function(data) {
  233. return 'Error: ' + data.responseText
  234. }
  235. },
  236. // options for the Delete Dailog
  237. {
  238. errorTextFormat : function(data) {
  239. return 'Error: ' + data.responseText
  240. }
  241. },{
  242. // search options 这里支持多条件查询
  243. multipleSearch: true,
  244. sopt:['eq','ne','cn','nc','lt','le','gt','ge'],
  245. afterSubmit: function(r, data) {
  246. var messageString = r.responseText;
  247. var mesObj = eval('(' + messageString + ')');
  248. return [mesObj.state, mesObj.message];
  249. }
  250. }
  251. );
  252. /////////////////////////////////////////////////////////////////////////////////////////////////
  253. $(window).resize(function(){
  254. $("#jqGrid1").setGridWidth($(window).width()-40); 
  255. });
  256. $("#jqGrid1").jqGrid({
  257. url : '${ctx}/myconsole/crawlerfield/getByPage',
  258. editurl:"${ctx}/myconsole/crawlerfield/saveOrUpdate",
  259. mtype : "POST",
  260. datatype : "json",
  261. caption:'内容页元素',
  262. prmNames : {
  263. id: "keyid", 
  264. oper:"oper",    
  265. addoper:"add",
  266. editoper:"edit",
  267. deloper:"del" 
  268. },
  269. colModel : [ {
  270. label : '编号',
  271. name : 'keyid',
  272. hidden : true,
  273. key : true,
  274. width : 50
  275. }, {
  276. label : '采集任务编号',
  277. name : 'crawlerid',
  278. hidden : true,
  279. editable : true,
  280. edittype :"select",
  281. editoptions:{value:"a:a"}
  282. },{
  283. label : '信息属性名称',
  284. name : 'crawlerfieldname',
  285. editable: true,
  286. formatter:"select",
  287. edittype: "select",
  288. editoptions: {
  289. value: "${crawlerFieldName}"
  290. },
  291. stype:"select",
  292. searchoptions:{
  293. value: "${crawlerFieldName}"
  294. },
  295. width : 50
  296. }, {
  297. label : '信息属性选择器',
  298. name : 'crawlerfieldselect',
  299. editable: true,
  300. width : 180
  301. }, {
  302. label : '更新时间',
  303. name : 'updatetime',
  304. editable: false,
  305. width : 80
  306. }],
  307. rownumbers : true,//添加左侧行号
  308. viewrecords : true,//是否在浏览导航栏显示记录总数
  309. rowNum : 10,//每页显示记录数
  310. rowList : [10,15,20,30,40,50 ],//用于改变显示行数的下拉列表框的元素数组。
  311. autowidth : true,
  312. height : 'auto',
  313. pager : "#jqGridPager1"
  314. });
  315. $('#jqGrid1').navGrid('#jqGridPager1',
  316. // the buttons to appear on the toolbar of the grid
  317. {
  318. edit : true,
  319. add : true,
  320. del : true,
  321. search : true,
  322. refresh : true,
  323. view : true,
  324. position : "left",
  325. cloneToTop : true
  326. },
  327. // options for the Edit Dialog
  328. {
  329. recreateForm : true,
  330. checkOnUpdate : true,
  331. checkOnSubmit : true,
  332. closeAfterEdit : true,
  333. onInitializeForm:function(formid) {
  334. },
  335. serializeEditData: function(postdata) {
  336. //myconsole.log(postdata);
  337. $.each(postdata,function (index, value) {
  338. //myconsole.log(index);
  339. if(index!="jqGrid_id" && index!="oper" && index!="keyid"){
  340. modifyJosnKey(postdata,index,"crawlerField."+index);
  341. }
  342. if(index=="keyid"){
  343. postdata["crawlerField."+index]=value
  344. }
  345. })
  346. //myconsole.log(postdata);
  347. return postdata;
  348. },
  349. errorTextFormat : function(data) {
  350. return 'Error: ' + data.responseText
  351. }
  352. },
  353. // options for the Add Dialog
  354. {
  355. closeAfterAdd : true,
  356. recreateForm : true,
  357. beforeSubmit: function(postdata, formid){
  358. $.each(postdata,function (index, value) {
  359. if(index!="jqGrid_id" && index!="oper"){
  360. modifyJosnKey(postdata,index,"crawlerField."+index);
  361. }
  362. })
  363. return[true,''];
  364. },
  365. errorTextFormat : function(data) {
  366. return 'Error: ' + data.responseText
  367. }
  368. },
  369. // options for the Delete Dailog
  370. {
  371. errorTextFormat : function(data) {
  372. return 'Error: ' + data.responseText
  373. }
  374. },{
  375. // search options 这里支持多条件查询
  376. multipleSearch: true,
  377. sopt:['eq','ne','cn','nc','lt','le','gt','ge'],
  378. afterSubmit: function(r, data) {
  379. var messageString = r.responseText;
  380. var mesObj = eval('(' + messageString + ')');
  381. return [mesObj.state, mesObj.message];
  382. }
  383. }
  384. );
  385. /////////////////////////////////////////////////////////////////////////////////////////
  386. $(window).resize(function(){
  387. $("#jqGrid2").setGridWidth($(window).width()-40); 
  388. });
  389. $("#jqGrid2").jqGrid({
  390. url : '${ctx}/myconsole/crawlercolumn/getByPage',
  391. editurl:"${ctx}/myconsole/crawlercolumn/saveOrUpdate",
  392. mtype : "POST",
  393. datatype : "json",
  394. caption:'栏目数据',
  395. multiselect: true,
  396. prmNames : {
  397. id: "keyid", 
  398. oper:"oper",    
  399. addoper:"add",
  400. editoper:"edit",
  401. deloper:"del" 
  402. },
  403. colModel : [ {
  404. label : '编号',
  405. name : 'keyid',
  406. hidden : true,
  407. key : true,
  408. width : 50
  409. }, {
  410. label : '采集任务编号',
  411. name : 'crawlerid',
  412. hidden : true,
  413. editable : true,
  414. edittype :"select",
  415. editoptions:{value:"a:a"}
  416. },{
  417. label :'所属站点',
  418. name :'siteid',
  419. editable: true,
  420. width : 50,
  421. editrules:{required:true},
  422. edittype :"select",
  423. formatter:"select",
  424. editoptions:{
  425. dataInit: function(elem){
  426. $(elem).width(225);
  427. },
  428. value:""},
  429. formoptions:{
  430. elmsuffix:'*'
  431. }
  432. },{
  433. label : '选择栏目',
  434. name : 'columnid',
  435. editable: true,
  436. width : 50,
  437. editrules:{required:true},
  438. edittype :"select",
  439. formatter:"select",
  440. editoptions: {
  441. dataInit: function(elem){
  442. $(elem).width(225);
  443. },
  444. value:"",
  445. maxlength: 10,
  446. },
  447. formoptions:{
  448. elmsuffix:'*'
  449. }
  450. },{
  451. label : '入口地址',
  452. name : 'accessurl',
  453. editable: true,
  454. width : 80,
  455. hidden : true,
  456. editrules:{
  457. required:true,
  458. edithidden:true
  459. },
  460. formoptions:{
  461. elmsuffix:'*'
  462. }
  463. },{
  464. label : '采集页数',
  465. name : 'pagenum',
  466. editable: true,
  467. width : 20,
  468. editrules:{required:true,integer:true,minValue:1},
  469. formoptions:{
  470. elmsuffix:'*'
  471. }
  472. }, {
  473. label : '更新时间',
  474. name : 'updatetime',
  475. editable: false,
  476. width : 20
  477. },{
  478. label : '操作',
  479. name:'operMod',
  480. index:'operMod',
  481. align:"center",
  482. width:20,
  483. formatter: function (cellvalue, options, rowdata) {
  484. var html = "<button class='btn btn-success radius size-S' onclick=crawlerOne('"+rowdata.keyid+"')>执行</button>";
  485. return html;
  486. }
  487. }],
  488. rownumbers : true,//添加左侧行号
  489. viewrecords : true,//是否在浏览导航栏显示记录总数
  490. rowNum : 50,//每页显示记录数
  491. rowList : [10,15,20,30,40,50,100 ],//用于改变显示行数的下拉列表框的元素数组。
  492. autowidth : true,
  493. height : 'auto',
  494. pager : "#jqGridPager2"
  495. });
  496. $('#jqGrid2').navGrid('#jqGridPager2',
  497. // the buttons to appear on the toolbar of the grid
  498. {
  499. edit : true,
  500. add : true,
  501. del : true,
  502. search : true,
  503. refresh : true,
  504. view : true,
  505. position : "left",
  506. cloneToTop : true
  507. },
  508. // options for the Edit Dialog
  509. {
  510. recreateForm : true,
  511. checkOnUpdate : true,
  512. checkOnSubmit : true,
  513. closeAfterEdit : true,
  514. afterShowForm : function(formid) {
  515. $("#siteid").change(function(){
  516. /* var siteid=$('#siteid').val; */
  517. $("#columnid").empty().append("<option>请选择</option>");
  518. var siteid=this.value;
  519. $.ajax({
  520. async: false,
  521. type : "POST",
  522. url : "${ctx}/myconsole/crawler/getColumnOptionsBySite?siteid="+siteid,
  523. dataType : 'text',
  524. success : function(data) {
  525. $("#columnid").append(data);
  526. }
  527. });
  528. });
  529. },
  530. serializeEditData: function(postdata) {
  531. //myconsole.log(postdata);
  532. $.each(postdata,function (index, value) {
  533. //myconsole.log(index);
  534. if(index!="jqGrid_id" && index!="oper" && index!="keyid"){
  535. modifyJosnKey(postdata,index,"crawlerColumn."+index);
  536. }
  537. if(index=="keyid"){
  538. postdata["crawlerColumn."+index]=value
  539. }
  540. })
  541. //myconsole.log(postdata);
  542. return postdata;
  543. },
  544. errorTextFormat : function(data) {
  545. return 'Error: ' + data.responseText
  546. }
  547. },
  548. // options for the Add Dialog
  549. {
  550. closeAfterAdd : true,
  551. recreateForm : true,
  552. afterShowForm : function(formid) {
  553. $("#columnid").empty().append("<option>请选择</option>");
  554. $("#siteid").change(function(){
  555. /* var siteid=$('#siteid').val; */
  556. $("#columnid").empty().append("<option>请选择</option>");
  557. var siteid=this.value;
  558. $.ajax({
  559. async: false,
  560. type : "POST",
  561. url : "${ctx}/myconsole/crawler/getColumnOptionsBySite?siteid="+siteid,
  562. dataType : 'text',
  563. success : function(data) {
  564. $("#columnid").append(data);
  565. }
  566. });
  567. });
  568. },
  569. beforeSubmit: function(postdata, formid){
  570. $.each(postdata,function (index, value) {
  571. if(index!="jqGrid_id" && index!="oper"){
  572. modifyJosnKey(postdata,index,"crawlerColumn."+index);
  573. }
  574. })
  575. return[true,''];
  576. },
  577. errorTextFormat : function(data) {
  578. return 'Error: ' + data.responseText
  579. }
  580. },
  581. // options for the Delete Dailog
  582. {
  583. errorTextFormat : function(data) {
  584. return 'Error: ' + data.responseText
  585. }
  586. },{
  587. // search options 这里支持多条件查询
  588. multipleSearch: true,
  589. sopt:['eq','ne','cn','nc','lt','le','gt','ge'],
  590. afterSubmit: function(r, data) {
  591. var messageString = r.responseText;
  592. var mesObj = eval('(' + messageString + ')');
  593. return [mesObj.state, mesObj.message];
  594. }
  595. }
  596. );
  597. $("#crawlerAll").click(function(){
  598. var columnKeyIDs = $('#jqGrid2').jqGrid('getGridParam','selarrrow');
  599. $.post("${ctx}/myconsole/crawler/all",{"columnKeyIDs":columnKeyIDs},function(data){
  600. layer.alert(data);
  601. })
  602. })
  603. })
  604. function crawlerOne(keyid){
  605. $.get("${ctx}/myconsole/crawler/run",{"columnKeyID":keyid},function(data){
  606. layer.alert(data);
  607. })
  608. }
  609. </script>
  610. <title>我的桌面</title>
  611. </head>
  612. <body>
  613. <div class="page-container">
  614. <table id="jqGrid"></table>
  615. <div id="jqGridPager"></div>
  616. <div>&nbsp;</div>
  617. <table id="jqGrid1"></table>
  618. <div id="jqGridPager1"></div>
  619. <div>&nbsp;</div>
  620. <div style="text-align:right">
  621. <button id="crawlerAll" class="btn btn-primary radius" type="button"><i class="Hui-iconfont">&#xe632;</i>批量采集</button>
  622. </div>
  623. <table id="jqGrid2"></table>
  624. <div id="jqGridPager2"></div>
  625. </div>
  626. <footer class="footer mt-20">
  627. <div class="container">
  628. <p></p>
  629. </div>
  630. </footer>
  631. </body>
  632. </html>