标签：Console string c# html req1 爬取 源码 WriteLine using
c#使用控制台爬取笔趣阁小说，以下为效果图
以下为完整代码
using System;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
namespace ConsoleApp3
{
class Program
{
static void Main(string[] args)
{
string searchbook = "https://www.biqugeu.net/searchbook.php?keyword=<<bookname>>";
string searchurl = null;
string searchcontent = null;
string baseurl = "https://www.biqugeu.net/";
string nextChapter = null;
string html = null;
string bookname = null;
string bookTitle = null;
string ChapterContent;
string regex1 = "<h1>(?<bookname>.*?)</h1>";
string regex2 = "<a href=\"/.*?\" target=\"_top\" class=\"pre\">上一章</a> ← <a href=\"/.*?/\" target=\"_top\" title=\"\" class=\"back\">章节列表</a> → <a href=\"(?<nextChapter>.*?)\" target=\"_top\" class=\"next\"";
string regex3 = "booktitle = \"(?<booktitle>.*?)\";";
string regex4 = "(?<data>.*?)<br/><br/>";
string regex5 = "<div class=\"image\">\\s*<a href=\"/(?<bookurl>.*?)\"";
string regex6 = "<dt>.*?</dt><dd><ahref=\"/(?<bookfirst>.*?)\">.*?</a></dd>";
Console.WriteLine("请输入需要爬取的小说！");
string novelName = Console.ReadLine();
try
{
searchurl = searchbook.Replace("<<bookname>>", novelName);
HttpWebRequest req1 = (HttpWebRequest)WebRequest.Create(searchurl);
req1.Method = "GET";
req1.Accept = "text/html";
req1.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse();
using (StreamReader reader = new StreamReader(res1.GetResponseStream()))
{
html = reader.ReadToEnd();
if (!string.IsNullOrEmpty(html))
{
//Console.WriteLine(html);
html = html.Replace("\n", "").Replace("\t", "").Replace("\r", "");
searchcontent = Regex.Match(html, regex5).Groups["bookurl"].ToString();
if (searchcontent == "")
{
Console.WriteLine("没有找到该小说！");
}
searchurl = baseurl + searchcontent;
}
}
}
catch (WebException we)
{
Console.WriteLine(we.Message);
}
try
{
HttpWebRequest req1 = (HttpWebRequest)WebRequest.Create(searchurl);
req1.Method = "GET";
req1.Accept = "text/html";
req1.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
HttpWebResponse res1 = (HttpWebResponse)req1.GetResponse();
using (StreamReader reader = new StreamReader(res1.GetResponseStream()))
{
html = reader.ReadToEnd();
if (!string.IsNullOrEmpty(html))
{
//Console.WriteLine(html);
html = html.Replace("\n", "").Replace("\t", "").Replace("\r", "").Replace(" ","");
searchcontent = Regex.Matches(html, regex6)[1].Groups["bookfirst"].ToString();
searchurl = baseurl + searchcontent;
}
}
}
catch (Exception)
{
throw;
}
do
{
restart: try
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(searchurl);
req.Method = "GET";
req.Accept = "text/html";
req.AllowAutoRedirect = true;
req.Headers.Add("Encoding", Encoding.UTF8.ToString());
req.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)";
HttpWebResponse res = (HttpWebResponse)req.GetResponse();
using (StreamReader reader = new StreamReader(res.GetResponseStream()))
{
html = reader.ReadToEnd();
if (!string.IsNullOrEmpty(html))
{
ChapterContent = "";
//获取下一章
nextChapter = Regex.Match(html, regex2).Groups["nextChapter"].ToString();
searchurl = baseurl + nextChapter;
//获取章节名
bookname = Regex.Match(html, regex1).Groups["bookname"].ToString();
ChapterContent += "\r\n";
ChapterContent += bookname;
ChapterContent += "\r\n";
//获取书名
bookTitle = Regex.Match(html, regex3).Groups["booktitle"].ToString();
//获取内容
MatchCollection match = Regex.Matches(html, regex4);
foreach (Match item in match)
{
string book = Regex.Match(item.Value, regex4).Groups["data"].ToString().Trim();
ChapterContent += book;
}
Console.WriteLine(bookname + "-------下载完毕！");
AddBookToTXT(ChapterContent, bookTitle);
}
}
}
catch (WebException we)
{
//Console.WriteLine(we.Message);
Console.WriteLine("远程主机强迫关闭了一个现有的连接,重新爬取当前章节。。。");
goto restart;
}
} while (nextChapter.Contains("html"));//当下一章链接没有跳转时结束
}
/// <summary>
/// 将内容保存到txt文件
/// </summary>
/// <param name="logstring">内容</param>
/// <param name="pathName">书名</param>
public static void AddBookToTXT(string logstring, string pathName)
{
string path = AppDomain.CurrentDomain.BaseDirectory + pathName + ".txt";
if (!System.IO.File.Exists(path))
{
FileStream stream = System.IO.File.Create(path);
stream.Close();
stream.Dispose();
}
using (StreamWriter writer = new StreamWriter(path, true))
{
writer.WriteLine(logstring);
}
}
}
}
标签：Console,string,c#,html,req1,爬取,源码,WriteLine,using 来源： https://www.cnblogs.com/liuyang-ux/p/15654756.html
本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享； 2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关； 3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关； 4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除； 5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。