三更灯火五更鸡,正是男儿读书时。
本次爬取百度贴吧的图片。
本次爬虫网址为 https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search
本次爬虫用到的工具: IJ URL类,IO流,UUID(随机命名),正则表达式。
首先得到 https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search 的html。
HttpURLConnection connection =(HttpURLConnection)
new URL("https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search").openConnection();
返回 一个,字符串形式的html便于后面操作。
接下来就要写正则来匹配图片的地址。首先要分析我们要的图片是那个地址。
我们要的是中间的风景图片,而不是其他的二维码图片或者头像图片,这个时候就打开开发者工具F12
那么将有用的代码拿出来,供我们写正则
data-original="https://imgsa.baidu.com/forum/wh%3D200%2C90%3B/sign=561eeed4c4ea15ce41bbe80b863016ca/57335443fbf2b211c636051fc78065380cd78e56.jpg"
bpic="https://imgsa.baidu.com/forum/w%3D580%3B/sign=8436c6918f025aafd3327ec3cbd6aa64/a08b87d6277f9e2f522d9d591230e924b899f343.jpg"
经过验证,上面的是我们需要的风景图片的地址,就是date-original后面的。
注意:正则写的时候,需要注意我们把后面的bpic带上来精确正则表达式。于是就开始匹配图片地址,放再一个List中。
private static ArrayList getPictureUrl(String html) {
Pattern patternRegex = Pattern.compile("bpic=\"(.*?)\"");//匹配图片地址的正则
Matcher matcher = patternRegex.matcher(html);
String htmlUrl=null;
ArrayList<String> listUrl = new ArrayList<>();
while (matcher.find()){
htmlUrl = matcher.group();
String[] split = htmlUrl.split("=\"");
String[] split1 = split[1].split("\"");//切出来了标准的URL
listUrl.add(split1[0]);//将图片的url存入数组中
}
return listUrl;
}
返回一个listURl,那么接下来就继续再listURL中去下载图片,用到IO流。
private static void getPicture(ArrayList pictureUrl) throws IOException {
for (Object p : pictureUrl) {
HttpURLConnection conn = (HttpURLConnection) new URL((String)p).openConnection();
InputStream in = conn.getInputStream();
UUID uuid = UUID.randomUUID();//形成随机名字
FileOutputStream out = new FileOutputStream("C:\\Users\\Administrator\\IdeaProjects\\homework\\picture\\"+uuid+".png");
while (true){
byte[] bytes = new byte[1024];
int len = in.read(bytes);
if(len == -1){
break;
}
out.write(bytes,0,len);
}
out.close();
}
}
完整代码
package com.westos.danli;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Spider1 {
public static void main(String[] args) throws IOException {
HttpURLConnection connection =(HttpURLConnection)
new URL("https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search").openConnection();
// InputStream in = connection.getInputStream();
// BufferedReader reader;
// reader = new BufferedReader(new FileReader(String.valueOf(in)));
String html = Spider1.getHtml(connection);//返回一个字符串的html对象
ArrayList pictureUrl = getPictureUrl(html);
getPicture(pictureUrl);//直接存在C:\Users\Administrator\IdeaProjects\homework\picture中
return;
}
private static void getPicture(ArrayList pictureUrl) throws IOException {
for (Object p : pictureUrl) {
HttpURLConnection conn = (HttpURLConnection) new URL((String)p).openConnection();
InputStream in = conn.getInputStream();
UUID uuid = UUID.randomUUID();//形成一个随机命名。
FileOutputStream out = new FileOutputStream("C:\\Users\\Administrator\\IdeaProjects\\homework\\picture\\"+uuid+".png");//直接用网址命名
while (true){
byte[] bytes = new byte[1024];
int len = in.read(bytes);
if(len == -1){
break;
}
out.write(bytes,0,len);
}
out.close();
}
}
private static ArrayList getPictureUrl(String html) {
Pattern patternRegex = Pattern.compile("bpic=\"(.*?)\"");//匹配图片地址的正则
Matcher matcher = patternRegex.matcher(html);
String htmlUrl=null;
ArrayList<String> listUrl = new ArrayList<>();
while (matcher.find()){
htmlUrl = matcher.group();
String[] split = htmlUrl.split("=\"");
String[] split1 = split[1].split("\"");//切出来了标准的URL
listUrl.add(split1[0]);//将图片的url存入数组中
}
return listUrl;
}
/*
获得html页面,写入本地。
*/
private static String getHtml(HttpURLConnection connection) throws IOException {
InputStream in = connection.getInputStream();//拿到html字节流
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream("123.html"));
StringBuffer stringBuffer1 = new StringBuffer();//定义一个stringbuffer,将html转换为字符串对象
while (true) {
String s = reader.readLine();//读取html
if (s == null) {
break;
}
bufferedOutputStream.write(s.getBytes());
stringBuffer1.append(s).append("\n");//存为字符串对象
}
return stringBuffer1.toString();
}
}