借助java的html解析框架Jsoup，可以方便爬取网页，该框架用法见Java html解析器之Jsoup。

代码

该爬虫只有一个文件
WebBookCrawler.java

package top.sidian123;import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.TextNode;import javax.print.Doc;
import java.io.*;
import java.nio.charset.Charset;
import java.util.List;/*** 爬取网路小说的爬虫* @author sidian* @date 2019/5/29 15:21*/
public class WebBookCrawler {/*** 起始页*/private String url;/*** 每一章标题的选择器*/private String titleSelector;/*** 每一章正文的选择器*/private String contentSelector;/*** 下一章链接的css选择器*/private String nextChapterSelector;/*** 爬取后文件保存位置和命名，默认当前位置*/private String filename="./小说.txt";/*** 是否以添加的方式写入文件，默认false*/private boolean append=false;/*** 用于存储到文件中*/private BufferedWriter write;public WebBookCrawler(String url,String titleSelector,String contentSelector,String nextChapterSelector){this.url=url;this.titleSelector=titleSelector;this.contentSelector=contentSelector;this.nextChapterSelector=nextChapterSelector;}public WebBookCrawler(String url,String titleSelector,String contentSelector,String nextChapterSelector,String filename){this.url=url;this.titleSelector=titleSelector;this.contentSelector=contentSelector;this.nextChapterSelector=nextChapterSelector;this.filename=filename;}/*** 爬取数据，默认延迟200ms* @throws IOException* @throws InterruptedException*/public void crawl() throws IOException, InterruptedException {crawl(200);}/*** 爬取数据，异常时重试10次* @param delay* @throws IOException* @throws InterruptedException*/public void crawl(int delay) throws IOException, InterruptedException {int count=10;//重试次数boolean flag=true;while (flag){try{_crawl(delay);flag=false;}catch (Exception e){if(--count!=0){System.out.println("莫名错误，原因："+e.getMessage());System.out.println("开始第"+(10-count)+"次重试");flag=true;Thread.sleep(1000);//设置为添加模式，防止文件被覆盖this.append=true;}else{throw e;}}}}/*** 设置append属性* @param append*/public void setAppend(Boolean append){this.append=append;}/*** 开始爬出数据* @param delay 每次读取网页的延迟时间,单位ms，用于反爬虫* @throws IOException 获取网页和保存文件时错误，抛出该异常*/private void _crawl(int delay) throws IOException, InterruptedException {try{//初始化write=new BufferedWriter(new FileWriter(filename, Charset.forName("utf-8"),append));int count=10;//文件写入的刷新间隔//遍历所有网页while(true){//获取文档Document document=getDocument(url);//获取小说内容getChapter(document);//判断html是否有“下一页”的链接Element link=document.selectFirst(nextChapterSelector);if(link==null){//没有“下一页”//爬取完毕，跳出循环break;}//获取下一页链接url=link.attr("abs:href");//每十章刷新一次if(--count==0){write.flush();count=10;}//延迟if(delay>0){Thread.sleep(delay);}}}finally {write.close();}}/*** 获取该章节的标题、正文，并保存在文件中* @param document* @throws IOException*/private void getChapter(Document document) throws IOException {//获取标题Element titleEle=document.selectFirst(titleSelector);String title=getContent(titleEle).trim();write.write(title+"\n\n");System.out.println("已获取章节："+title);//获取正文Element contentEle=document.selectFirst(contentSelector);String content=getContent(contentEle).trim();write.write(content+"\n\n");}/*** 获取该元素中的文本内容。* 会遍历子元素* @param element* @return*/private String getContent(Element element) {List<TextNode> textNodes=element.textNodes();StringBuilder stringBuilder=new StringBuilder();textNodes.forEach((node)->{if(!node.text().trim().isEmpty()){stringBuilder.append(node.text().trim()+"\n\n");}});return stringBuilder.toString();}/*** 从url上获取文档，为了防止反爬虫，这是一些头字段* 如果失败，会重试10次* @param url* @return*/private Document getDocument(String url) throws IOException {int count=10;//重试次数boolean flag=true;Document document=null;while (flag){try{document = Jsoup.connect(url).userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36").get();flag=false;}catch (IOException e){if(count--!=0){System.out.println("网页获取失败，原因："+e.getMessage());System.out.println("开始第"+(10-count)+"次重试");}else{throw e;}}}return document;}}

用法很简单，传入小说url、标题css选择器、内容css选择器、下一章链接css选择器，即可。下面给出一个helper类：
WebBookHelper.java

package top.sidian123;import java.io.IOException;/*** @author sidian* @date 2019/5/29 16:49*/
public class WebBookHelper {public static void 斗破苍穹() throws IOException, InterruptedException {WebBookCrawler crawler=new WebBookCrawler("https://www.biqukan.com/3_3037/1349252.html","#wrapper > div.book.reader > div.content > h1","#content","#wrapper > div.book.reader > div.content > div.page_chapter > ul > li:nth-child(3) > a","斗破苍穹.txt");crawler.crawl();}public static void 姐妹花的最强兵王() throws IOException, InterruptedException {WebBookCrawler crawler=new WebBookCrawler("https://www.81xzw.com/book/148528/0.html","body > div.novel > h1","#content","body > div.novel > div:nth-child(5) > a:nth-child(3)","姐妹花的最强兵王.txt");crawler.crawl();}
}

下面爬取一本小说：
App.java

package top.sidian123;import java.io.IOException;/*** Hello world!**/
public class App
{public static void main( String[] args ) throws IOException, InterruptedException {WebBookHelper.姐妹花的最强兵王();//WebBookHelper.斗破苍穹();}}

css选择器不会写？？？没关系！！借助浏览器一步到位！！！

爬取结果