标签：info htmlunit contains Question date Answer 源代码 class webmagic
由于首都之窗网站第二页和第二页网址不变，已经和林子雨老师教程相差甚远，所以现在选择htmlunit模拟点击，（跳转摁钮显示网页仍是第一页），所以本代码用的一直是点击下一页摁钮。
爬取网址：http://www.beijing.gov.cn/hudong/hdjl/com.web.search.mailList.flow
获取代码：
1 package util;
2
3 import java.io.IOException;
4 import java.util.ArrayList;
5 import java.util.LinkedList;
6 import java.util.List;
7
8 import com.gargoylesoftware.htmlunit.BrowserVersion;
9 import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
10 import com.gargoylesoftware.htmlunit.ImmediateRefreshHandler;
11 import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
12 import com.gargoylesoftware.htmlunit.WebClient;
13 import com.gargoylesoftware.htmlunit.html.HtmlElement;
14 import com.gargoylesoftware.htmlunit.html.HtmlPage;
15
16 public class 首都之窗 {
17 static List<String> lines_zi=new LinkedList<String>();
18 static List<String> lines_jian=new LinkedList<String>();
19 static List<String> lines_tou=new LinkedList<String>();
20
21 static String line;
22 public static void Value_start()
23 {
24 // TODO 自动生成的方法存根
25 WebClient webClient=new WebClient(BrowserVersion.CHROME); // 实例化Web客户端
26
27 System.out.println("AAAAAA");
28 try {
29 webClient.getOptions().setActiveXNative(false);
30 //webClient.getOptions().setCssEnabled(false);
31 //webClient.getOptions().setRedirectEnabled(true);
32 webClient.getOptions().setJavaScriptEnabled(true);
33 webClient.getOptions().setDoNotTrackEnabled(true);
34 webClient.getOptions().setThrowExceptionOnScriptError(false);
35 webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
36 webClient.getCache().setMaxSize(100);
37 webClient.getOptions().setJavaScriptEnabled(true);//运行js脚本执行
38 webClient.setAjaxController(new NicelyResynchronizingAjaxController());//设置支持AJAX
39 webClient.getOptions().setCssEnabled(false);//忽略css
40 webClient.getOptions().setUseInsecureSSL(true);//ssl安全访问
41 webClient.getOptions().setThrowExceptionOnScriptError(false); //解析js出错时不抛异常
42 //webClient.getOptions().setTimeout(50000); //超时时间 ms
43 webClient.getCookieManager().setCookiesEnabled(true);
44 webClient.getCache().clear();
45 webClient.setRefreshHandler(new ImmediateRefreshHandler());
46 webClient.getOptions().setTimeout(2*1000); //网页多少ms超时响应
47 webClient.setJavaScriptTimeout(600*1000); //javaScript多少ms超时
48 webClient.setAjaxController(new NicelyResynchronizingAjaxController());
49 //webClient.setJavaScriptTimeout(600*1000);
50 //webClient.getOptions().setRedirectEnabled(true);
51 webClient.waitForBackgroundJavaScript(60*1000);
52
53 HtmlPage page=webClient.getPage("http://www.beijing.gov.cn/hudong/hdjl/com.web.search.mailList.flow"); // 解析获取页面
54 HtmlElement a=page.getElementByName("nextPage");
55 int j=1,lastj=0;
56 FileHandle fh=new FileHandle();
57 StringHandle sh=new StringHandle();
58 List<String> lastInfo_zi=new ArrayList<String>();
59 List<String> lastInfo_jian=new ArrayList<String>();
60 List<String> lastInfo_tou=new ArrayList<String>();
61 System.out.println("asdfsdaf");
62 fh.outFile(""+"\r\n", "E:\\578095023\\FileRecv\\寒假作业\\大三寒假作业\\北京市政百姓信件分析实战\\list.txt", false);
63
64 while(j!=600)
65 {
66
67 String nowInfo=page.asXml();
68
69 List<String> infoList_zi=sh.getExpString("letterdetail\\('.*?','.*?'\\)", nowInfo);
70 int g_size_zi=infoList_zi.size();
71 if(sh.StringListSameOutStringList(infoList_zi, lastInfo_zi).size()!=g_size_zi&&g_size_zi==7)
72 {
73 //System.out.println(g_size);
74 for(int i=0;i<g_size_zi;i++)
75 {
76 String theWeb=infoList_zi.get(i).replaceAll("letterdetail\\('.*?','", "").replace("')", "");
77 System.out.println(theWeb);
78 lines_zi.add(theWeb);
79 fh.outFile(theWeb+"\r\n", "E:\\578095023\\FileRecv\\寒假作业\\大三寒假作业\\北京市政百姓信件分析实战\\list.txt", true);
80
81 if(i==g_size_zi-1)
82 {
83 lastInfo_zi=infoList_zi;
84 System.out.println(j);
85 j++;
86 break;
87 }
88
89 }
90 page=a.click();
91 }
92 //page=a.click();
93 }
94
95
96 }catch (FailingHttpStatusCodeException | IOException e) {
97 // TODO Auto-generated catch block
98 e.printStackTrace();
99 } finally{
100 webClient.close(); // 关闭客户端，释放内存
101 }
102
103 }
104 public static void main(String[] args) {
105 Value_start();
106 }
107
108 }
getPass
爬取详细数据：
1 package util;
2 import java.io.*;
3 import java.util.List;
4
5 import org.jsoup.Connection;
6 import org.jsoup.Jsoup;
7 import org.jsoup.nodes.Document;
8 import org.jsoup.nodes.Element;
9 import org.jsoup.select.Elements;
10
11 import java.io.IOException;
12 import java.util.ArrayList;
13 import java.util.List;
14
15 import org.jsoup.Connection;
16 import org.jsoup.Jsoup;
17 import org.jsoup.nodes.Document;
18 import org.jsoup.nodes.Element;
19 import org.jsoup.select.Elements;
20
21 import util.SslUtils;
22
23 import us.codecraft.webmagic.Page;
24 import us.codecraft.webmagic.Site;
25 import us.codecraft.webmagic.Spider;
26 import us.codecraft.webmagic.processor.PageProcessor;
27
28 import com.bean.InfoBean;
29 import com.dao.InfoDao;
30 public class pa2 implements PageProcessor {
31 static int num=0;
32 static String Id;
33 static String Question;
34 static String Question_user;
35 static String Question_date;
36 static String Question_info;
37 static String Answer;
38 static String Answer_user;
39 static String Answer_date;
40 static String Answer_info;
41 static String Url;
42 //static String regEx="[\n`~!@#$%^&()+=|{}':;',\\[\\].<>/?~！@#￥%……&*（）——+|{}【】‘；：”“’。， 、？? ]";
43 static String aa = "";//这里是将特殊字符换为aa字符串," "代表直接去掉
44
45 // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
46 private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
47 private static int count =0;
48
49 @Override
50 public Site getSite() {
51 return site;
52 }
53 //主页面
54 public void parent(Page page)
55 {
56
57 System.out.println("抓取的内容\n"+
58 page.getHtml().xpath("//span[@name='cutStr' and @dispLength='68']//text()").get()
59 );
60 }
61 //子页面
62 public void child(Page page) throws IOException {
63
64 System.out.println("RRRRRRRRR");
65 BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File("E:\\578095023\\FileRecv\\寒假作业\\大三寒假作业\\list.txt")),
66 "UTF-8"));
67 String line=null;
68 System.out.println("SSSSSSSS");
69 while((line=br.readLine())!=null)
70 {
71
72 String url= "http://www.beijing.gov.cn/hudong/hdjl/com.web.consult.";
73 String type="";//声明类型码
74 type="consultDetail.flow?originalId=";
75 url+=type;
76 url+=line;
77 System.out.println(url);
78 page.addTargetRequest(url);
79
80 url= "http://www.beijing.gov.cn/hudong/hdjl/com.web.consult.";
81 type="";//声明类型码
82 type="suggesDetail.flow?originalId=";
83 url+=type;
84 url+=line;
85 System.out.println(url);
86 page.addTargetRequest(url);
87
88 url= "http://www.beijing.gov.cn/hudong/hdjl/com.web.consult.";
89 type="";//声明类型码
90 type="complainDetail.flow?originalId=";
91 url+=type;
92 url+=line;
93 System.out.println(url);
94 page.addTargetRequest(url);
95 }
96
97 if(page.getUrl().regex("http://www.beijing.gov.cn/hudong/hdjl/com.web.search.mailList.flow").match())
98 {
99
100 parent(page);
101 }
102 else
103 {
104 Question=page.getHtml().xpath("//div[contains(@class, 'col-xs-10')]/strong//text()").get().trim();
105 // Question=Question.replaceAll(regEx, aa);
106
107 Question_user=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'my-3')]/div[contains(@class, 'col-xs-10') and contains(@class, 'text-muted')]//text()").get().trim();
108 //Question_user=Question_user.replaceAll(regEx, aa);
109 Question_user=Question_user.replaceAll("来信人", aa).trim();
110 Question_user=Question_user.replaceAll("：", aa).trim();
111 Question_date=page.getHtml().xpath("//div[contains(@class, 'col-xs-12')]/div[contains(@class, 'col-xs-5')]//text()").get();
112 // Question=Question.replaceAll(regEx, aa);
113 Question_date=Question_date.replaceAll("时间", aa).trim();
114 Question_date=Question_date.replaceAll("：", aa).trim();
115
116 Question_info=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'mx-2') ]//text()").get();
117 //Question_info=Question_info.replaceAll(regEx, aa);
118
119 Answer=page.getHtml().xpath("//div[contains(@class, 'col-xs-9') and contains(@class, 'my-2')]//text()").get();
120 //Answer=Answer.replaceAll(regEx, aa);
121
122 Answer_user=page.getHtml().xpath("//div[contains(@class, 'col-xs-9') and contains(@class, 'my-2')]//text()").get();
123 // Answer_user=Answer_user.replaceAll(regEx, aa);
124
125 Answer_date=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'col-sm-3')and contains(@class, 'col-md-3') and contains(@class, 'my-2')]//text()").get();
126 // Answer_date=Answer_date.replaceAll(regEx, aa);
127 Answer_date=Answer_date.replaceAll("答复时间", aa).trim();
128 Answer_date=Answer_date.replaceAll("：", aa).trim();
129
130
131 List<String> values=new ArrayList<String>();
132 values=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'my-3')and contains(@class, 'p-4')]//*//text()").all();
133 Answer_info=null;
134 for(String value:values)
135 {
136 Answer_info+=value;
137 }
138 if(Answer_info==null)
139 {
140 Answer_info=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'my-3')and contains(@class, 'p-4')]//text()").get();
141 }
142 Answer_info=Answer_info.replaceAll("？", aa).trim();
143 Answer_info=Answer_info.replaceAll("null", aa).trim();
144
145 Url=page.getUrl().get();
146 System.out.println("抓取的内容\n"+
147 page.getHtml().xpath("//div[contains(@class, 'col-xs-10')]/strong//text()").get()
148 );
149
150 System.out.println("Id:" + Id+
151 "\n Question:" + Question+
152 "\\n Question_user:" + Question_user+
153 "\n Question_date:" + Question_date+
154 "\n Question_info:" + Question_info+
155 "\n Answer:" + Answer+
156 "\n Answer_user:" + Answer_user+
157 "\n Answer_date:" + Answer_date+
158 "\n Answer_info:"+Answer_info+
159 "\n Url:"+Url);
160 InfoDao.add(Question, Question_user, Question_date, Question_info, Answer, Answer_user, Answer_date, Answer_info, Url);
161 }
162 count ++;
163 }
164 @Override
165 public void process(Page page) {
166 num=num+1;
167 if(num==1)
168 {
169 try {
170 child(page);
171 } catch (IOException e) {
172 e.printStackTrace();
173 }
174 }
175 else
176 {
177 Question=page.getHtml().xpath("//div[contains(@class, 'col-xs-10')]/strong//text()").get().trim();
178 // Question=Question.replaceAll(regEx, aa);
179
180 Question_user=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'my-3')]/div[contains(@class, 'col-xs-10') and contains(@class, 'text-muted')]//text()").get().trim();
181 //Question_user=Question_user.replaceAll(regEx, aa);
182 Question_user=Question_user.replaceAll("来信人", aa).trim();
183 Question_user=Question_user.replaceAll("：", aa).trim();
184 Question_date=page.getHtml().xpath("//div[contains(@class, 'col-xs-12')]/div[contains(@class, 'col-xs-5')]//text()").get();
185 // Question=Question.replaceAll(regEx, aa);
186 Question_date=Question_date.replaceAll("时间", aa).trim();
187 Question_date=Question_date.replaceAll("：", aa).trim();
188
189 Question_info=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'mx-2') ]//text()").get();
190 //Question_info=Question_info.replaceAll(regEx, aa);
191
192 Answer=page.getHtml().xpath("//div[contains(@class, 'col-xs-9') and contains(@class, 'my-2')]//text()").get();
193 //Answer=Answer.replaceAll(regEx, aa);
194
195 Answer_user=page.getHtml().xpath("//div[contains(@class, 'col-xs-9') and contains(@class, 'my-2')]//text()").get();
196 // Answer_user=Answer_user.replaceAll(regEx, aa);
197
198 Answer_date=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'col-sm-3')and contains(@class, 'col-md-3') and contains(@class, 'my-2')]//text()").get();
199 // Answer_date=Answer_date.replaceAll(regEx, aa);
200 Answer_date=Answer_date.replaceAll("答复时间", aa).trim();
201 Answer_date=Answer_date.replaceAll("：", aa).trim();
202
203 List<String> values=new ArrayList<String>();
204 values=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'my-3')and contains(@class, 'p-4')]//*//text()").all();
205 Answer_info=null;
206 for(String value:values)
207 {
208 Answer_info+=value;
209 }
210 if(Answer_info==null)
211 {
212 Answer_info=page.getHtml().xpath("//div[contains(@class, 'col-xs-12') and contains(@class, 'my-3')and contains(@class, 'p-4')]//text()").get();
213 }
214 Answer_info=Answer_info.replaceAll("？", aa).trim();
215 Answer_info=Answer_info.replaceAll("null", aa).trim();
216
217 Url=page.getUrl().get();
218 System.out.println("抓取的内容\n"+
219 page.getHtml().xpath("//div[contains(@class, 'col-xs-10')]/strong//text()").get()
220 );
221
222 System.out.println("Id:" + Id+
223 "\n Question:" + Question+
224 "\\n Question_user:" + Question_user+
225 "\n Question_date:" + Question_date+
226 "\n Question_info:" + Question_info+
227 "\n Answer:" + Answer+
228 "\n Answer_user:" + Answer_user+
229 "\n Answer_date:" + Answer_date+
230 "\n Answer_info:"+Answer_info+
231 "\n Url:"+Url);
232 InfoDao.add(Question, Question_user, Question_date, Question_info, Answer, Answer_user, Answer_date, Answer_info, Url);
233 }
234
235 }
236
237 public static void main(String[] args) {
238 try {
239 SslUtils.ignoreSsl();
240 } catch (Exception e) {
241 e.printStackTrace();
242 }
243 // jsoup("http://www.beijing.gov.cn/hudong/hdjl/com.web.search.mailList.flow");
244 long startTime, endTime;
245 System.out.println("开始爬取...");
246 InfoDao.delete();
247 startTime = System.currentTimeMillis();
248 Spider.create(new pa2()).addUrl("http://www.beijing.gov.cn/hudong/hdjl/com.web.search.mailList.flow").thread(5).run();
249 endTime = System.currentTimeMillis();
250 System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒，抓取了"+count+"条记录");
251 }
252
253
254 }
getInfo
下载地址：
标签：info,htmlunit,contains,Question,date,Answer,源代码,class,webmagic 来源： https://www.cnblogs.com/smartisn/p/12237534.html
本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享； 2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关； 3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关； 4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除； 5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。