分享一个java爬虫抓取并下载不重复图片程序

今天分享一个因自己需要，而写的一个java小程序，可以自定义抓取并下载一个网站的图片资源，当然一定得是返回一个图片地址。

使用时，新建包名为saveimg，新建类名DownloadImg ，相信这点还是会的！此代码有一定的风险性，仅作为技术交流

package saveimg;
import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
/**
 *
 * @author 1059767677@qq.com
 * @version 2019年3月21日下午1:14:27
 */
public class DownloadImg {
    static List imgUrlList = new ArrayList();
    public static void main(String[] args) throws Exception {
        DownloadImg dmg = new DownloadImg();
        String location = dmg.getImgUrl();
        //设置下载的次数
        for (int i = 0; i < 20; i++) {
            if (checkImgUrl(location, imgUrlList)) {
                imgUrlList.add(location);
                System.out.println("第" + i + "张：" + location);
                // 保存到指定地址,命名规则为数字递增
                dmg.saveImg(location, "E:\\img\\" + i + ".jpg");
            } else {
                continue;
            }
            location = dmg.getImgUrl();
        }
        //System.out.println("imgUrlList.num====" + imgUrlList.size());
    }
    private String getImgUrl() throws Exception {
        // 设置要访问的图片地址,这个地址是类似于接口,如果一个地址只有一张图片,那么也不用这么麻烦了对不对
        String url = "http://api.laoguoba.com/random/api.php?id=5";
        URL serverUrl = new URL(url);
        HttpURLConnection conn = (HttpURLConnection) serverUrl.openConnection();
        conn.setRequestMethod("GET");
        // 必须设置false，否则会自动redirect到Location的地址
        conn.setInstanceFollowRedirects(false);
        conn.addRequestProperty("Accept-Charset", "UTF-8;");
        conn.connect();
        String location = conn.getHeaderField("Location");
        return location;
    }
    /**
     *
     * @param ImgUrl       图片路径
     * @param imgUrlList_t 集合
     * @return
     */
    private static Boolean checkImgUrl(String ImgUrl, List<String> imgUrlList_t) {
        if (imgUrlList_t.isEmpty()) {
            return true;
        } else {
            for (String temp : imgUrlList_t) {
                return !(ImgUrl.equals(temp));
            }
        }
        return false;
    }
    /**
     * @说明 用于下载图片的方法
     * @param destUrl  下载地址
     * @param filePath 下载后保存的路径
     */
    public void saveImg(String destUrl, String filePath) {
        FileOutputStream fos = null;
        BufferedInputStream bis = null;
        HttpURLConnection httpUrl = null;
        URL url = null;
        int BUFFER_SIZE = 1024;
        byte[] buf = new byte[BUFFER_SIZE];
        int size = 0;
        try {
            url = new URL(destUrl);
            httpUrl = (HttpURLConnection) url.openConnection();
            httpUrl.connect();
            bis = new BufferedInputStream(httpUrl.getInputStream());
            fos = new FileOutputStream(filePath);
            while ((size = bis.read(buf)) != -1) {
                fos.write(buf, 0, size);
            }
            fos.flush();
        } catch (IOException e) {
        } catch (ClassCastException e) {
        } finally {
            try {
                fos.close();
                bis.close();
                httpUrl.disconnect();
            } catch (IOException e) {
            } catch (NullPointerException e) {
            }
        }

    }
}