Java使用httpRequest+Jsoup爬取红蓝球号码

<tr>

   <td align= "center" > 2021 - 06 - 13 </td>

   <td align= "center" > 2021065 </td>

   <td align= "center" style= "padding-left:10px;" >         

   <em class = "rr" > 01 </em>         

   <em class = "rr" > 04 </em>         

   <em class = "rr" > 08 </em>         

   <em class = "rr" > 19 </em>         

   <em class = "rr" > 29 </em>         

   <em class = "rr" > 33 </em>                   

   <em> 16 </em></td>

   <td><strong> 362 , 527 , 724 </strong></td>

   <td align= "left" style= "color:#999;" ><strong> 3 </strong></td>

   <td align= "center" ><strong class = "rc" > 82 </strong></td>

   <td align= "center" >

       <a href= "http://HdhCmsTestzhcw测试数据/ssq/kjgg/" rel= "external nofollow"   target= "_blank" ><img src= "http://images.zhcw测试数据/zhcw2010/kaijiang/zhcw/ssqpd_42.jpg" width= "16" height= "16" align= "absmiddle" title= "详细信息" ></a>

       <a href= "http://HdhCmsTestzhcw测试数据/video/kaijiangshipin/" rel= "external nofollow"   target= "_blank" ><img src= "http://images.zhcw测试数据/zhcw2010/kaijiang/zhcw/ssqpd_43.jpg" width= "16" height= "16" align= "absmiddle" title= "开奖视频" ></a>

   </td>

</tr>

<tr>

     <td colspan= "7" align= "center" style= "background:#fdf2e3;" >

         <p class = "zhu" ></p>

         <p class = "pg" >

             共<strong> 136 </strong> 页 /<strong> 2709 </strong>条记录

             <strong><a href= "/zhcw/inc/ssq/ssq_wqhg.jsp" rel= "external nofollow" >首页</a></strong>

             <strong><a href= "/zhcw/inc/ssq/ssq_wqhg.jsp?pagenum=1" rel= "external nofollow" >上一页</a></strong>

             <strong><a href= "/zhcw/inc/ssq/ssq_wqhg.jsp?pagenum=2" rel= "external nofollow" >下一页</a></strong>

             <strong><a href= "/zhcw/inc/ssq/ssq_wqhg.jsp?pagenum=136" rel= "external nofollow" >末页</a></strong>

             当前第<strong> 1 </strong>页</p>

      </td>

  </tr>

<dependency>

     <groupid>org.jsoup</groupid>

     <artifactid>jsoup</artifactid>

     <version> 1.13 . 1 </version>

</dependency>

public void execute(jobexecutioncontext arg0) throws jobexecutionexception {

  integer maxpage=getmaxpage(); //获取总页数

  for ( int pageno= 0 ;pageno<maxpage;pageno++){ //循环处理每一页的数据

   string url = "http://kaijiang.zhcw测试数据/zhcw/html/ssq/list_" +(pageno+ 1 )+ ".html" ;//拼访问地址

      string strdateformat = "yyyy-mm-dd" ;

         simpledateformat sdf = new simpledateformat(strdateformat);

   string html = jwthttputil.httprequest(url, "get" , null ); //访问对应的页面

   document doc =  jsoup.parse(html); //用jsoup解析页面内容并且解析为document

   document doc1= jsoup.parse(doc.getelementsbyclass( "wqhgt" ).tostring()); //wqhgt是table标签的class，这里获取到开奖号码的table的内容并且解析为document

   elements trs= doc1.getelementsbytag( "tr" ); //根据tag获取到所有的tr标签，这里获取到的是一个element数组

   logger.info( "当前页码---" +(pageno+ 1 )+ "----" +trs.size());

   for ( int i= 2 ;i<trs.size()- 1 ;i++){ //这里是根据页面内容确定开奖内容是从第三个tr开始到倒数第二个tr结束，最后一个tr是翻页按钮

    try {

     elements tds=trs.get(i).getelementsbytag( "td" );

     string kjrq=tds.get( 0 ).text();

     string kjqh=tds.get( 1 ).text();

     //这里是为了防止重复拉取，所以做了判断，如果已存在当前期的奖号，就直接跳过

     long count=lotteryssqkjjlservice.getcountforjdbc( "select count(*) from lottery_ssq_kjjl where ssq_kjqh='" + kjqh + "'" );

     //下面的内容很容易理解，就是可以对照页面分析部分内容看，就不赘述了

     if (count> 0 ) break ;

     lotteryssqkjjlentity kjhmentiry= new lotteryssqkjjlentity();

        elements kjhm=tds.get( 2 ).getelementsbytag( "em" );

     kjhmentiry.setssqkjrq(sdf.parse(kjrq));

     kjhmentiry.setssqkjqh(kjqh);

        kjhmentiry.setssqr1(integer.parseint(kjhm.get( 0 ).text()));

        kjhmentiry.setssqr2(integer.parseint(kjhm.get( 1 ).text()));

        kjhmentiry.setssqr3(integer.parseint(kjhm.get( 2 ).text()));

        kjhmentiry.setssqr4(integer.parseint(kjhm.get( 3 ).text()));

        kjhmentiry.setssqr5(integer.parseint(kjhm.get( 4 ).text()));

        kjhmentiry.setssqr6(integer.parseint(kjhm.get( 5 ).text()));

        kjhmentiry.setssqb1(integer.parseint(kjhm.get( 6 ).text()));

        lotteryssqkjjlservice.save(kjhmentiry);

    } catch (parseexception e) {

     // todo auto-generated catch block

     e.printstacktrace();

    } catch (exception e) {

     // todo auto-generated catch block

     e.printstacktrace();

    }

   }

  }

}

private integer getmaxpage(){

  string url = "http://kaijiang.zhcw测试数据/zhcw/html/ssq/list.html" ;

  string html = jwthttputil.httprequest(url, "get" , null );

  document doc =  jsoup.parse(html);

  document docpagebar= jsoup.parse(doc.getelementsbyclass( "pg" ).tostring());

  elements elepagebar = docpagebar.getelementsbytag( "strong" );

  integer maxpageno=integer.parseint(elepagebar.get( 0 ).text());

  return maxpageno;

}

public void run(){

  list<lotteryssqkjjlentity> list= lotteryssqkjjlservice.findbyquerystring( "from lotteryssqkjjlentity t" );

  for ( int i= 0 ;i<list.size();i++){

   lotteryssqkjjlentity kjjg=list.get(i);

   logger.info( "正在处理:" +kjjg.getssqkjqh());

   kjjg.setssqmin(kjjg.getssqr1()); //最小号

   kjjg.setssqmax(kjjg.getssqr6()); //最大号

   kjjg.setssqkd(kjjg.getssqr6()-kjjg.getssqr1()); //号码跨度

   kjjg.setssqjscnt(ssqkjjgutils(kjjg, "jscnt" )); //奇数个数

   kjjg.setssqoscnt(ssqkjjgutils(kjjg, "oscnt" )); //偶数个数

   kjjg.setssqsum(ssqkjjgutils(kjjg, "sum" )); //号码和

   lotteryssqkjjlservice.updateentitie(kjjg);

  }

  logger.info( "结果数据条数" +list.size());

}

/**

  *

  * @param kjjg

  * @param optype jsgs=奇数个数

  * @return

  */

private integer ssqkjjgutils(lotteryssqkjjlentity kjjg,string optype){

  list<integer> t = new arraylist<integer>();

  t.add(kjjg.getssqr1());

  t.add(kjjg.getssqr2());

  t.add(kjjg.getssqr3());

  t.add(kjjg.getssqr4());

  t.add(kjjg.getssqr5());

  t.add(kjjg.getssqr6());

  integer result= 0 ;

  switch (optype) {

  case "jscnt" :

  case "oscnt" :

   int jsgs= 0 ;

   int osgs= 0 ;

   for ( int i= 0 ;i<t.size();i++){

    if (t.get(i)% 2 == 0 ){

     osgs++;

    } else {

     jsgs++;

    }

   }

   if ( "jscnt" .equals(optype)){

    result=jsgs;

   } else {

    result=osgs;

   }

   break ;

  case "sum" :

   int sum= 0 ;

   for ( int i= 0 ;i<t.size();i++){

    sum+=t.get(i);

   }

   result=sum;

   break ;

  default :

   break ;

  }

  return result;

}