运行环境

oracle 圣何塞 debian arm 4C 24G

JDK 21, 不能在本地上复现代理程序在服务器上

htop 观察到是某一个核心突然满载然后就卡住了

代码

public class Main {

    static Log log = Log.get();
    static final Gson gson = new Gson();
    static List<Image> images = null;
    static int maxThreads = 100; // 控制固定线程池的大小
    static ExecutorService virtualThreadPool = Executors.newFixedThreadPool(maxThreads, Thread.ofVirtual().factory());
    static String filePath = "/root/java_work/imglist/";
    // 失败 list
    static List<String> failList = Lists.newArrayList();

    public static void main(String[] args) {
        OkHttpUtils.setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 6789)));
        List<String> types = List.of(".jpg", ".png");
        try {
            images = gson.fromJson(new FileReader("output-2024-4-16.json"), new TypeToken<List<Image>>(){}.getType());
        } catch (IOException e) {
            log.info("读取文件失败: {}", e.getMessage());
            return;
        }

        log.info("加载到的图片数量: {}", images.size());

        images.forEach(image -> virtualThreadPool.submit(() -> {
            for (String type : types) {
                String url = convertPreviewToImageUrl(image.getHref(), type);
                if (attemptToDownloadImage(url,0)) {
                    image.setSourceUrl(url);  // 更新 Image 对象
                    break;
                }
            }
        }));
        virtualThreadPool.shutdown();
        while (!virtualThreadPool.isTerminated()) {
            Thread.onSpinWait();
        }
        // 将更新后的 images 列表写回到 JSON 文件
        writeImagesToJson(images, "output-2024-4-16-ok.json");
        writeImagesToJson(failList, "output-2024-4-16-fail.json");
    }

    private static boolean attemptToDownloadImage(String url, int retryCount) {
        if (retryCount >= 3) {
            log.info("重试次数过多，放弃下载: {}", url);
            failList.add(url);
            return false;
        }
        try (Response response = OkHttpUtils.get(url, Headers.of("Connection", "close"))) {
            switch (response.code()) {
                case 200:
                    log.info("成功下载图片: {}", url);
                    byte[] bytes = Objects.requireNonNull(response.body()).bytes();
                    writeImageToFile(bytes, url.substring(url.lastIndexOf('/') + 1));
                    return true;
                case 404:
                    log.info("图片不存在: {}", url);
                    return false;
                case 429:
                    log.info("请求过于频繁，需要稍后重试: {}", url);
                    handleRateLimiting();
                    return attemptToDownloadImage(url, retryCount + 1);
                default:
                    log.info("其他 HTTP 响应: {}", response.code());
                    return false;
            }
        } catch (Exception e) {
            log.error("请求图片时出错: {}", e.fillInStackTrace());
            handleRateLimiting();
            return attemptToDownloadImage(url, retryCount + 1);
        }
    }

    private static void handleRateLimiting() {
        try {
            log.info("等待 5 秒后重试");
            Thread.sleep(5000); // 延迟 5 秒后重试
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }

    public static String convertPreviewToImageUrl(String previewUrl, String type) {
        String id = previewUrl.substring(previewUrl.lastIndexOf('/') + 1);
        return String.format("https://w.wallhaven.cc/full/%s/wallhaven-%s%s", id.substring(0, 2), id, type);
    }

    public static void writeImagesToJson(List<?> objects,String fileName) {
        try (FileWriter writer = new FileWriter(fileName)) {
            gson.toJson(objects, writer);
        } catch (IOException e) {
            log.error("写入文件时出错: {}", e.getMessage());
        }
    }

    public static void writeImageToFile(byte[] bytes, String fileName) {
        try (FileOutputStream fos = new FileOutputStream(filePath+fileName)) {
            fos.write(bytes);
            log.info("成功写入文件: {}", fileName);
        } catch (IOException e) {
            log.error("写入文件时出错: {}", e.getMessage());
        }
    }
}

https://gist.github.com/dnslin/fe657f9df08f4286c197c5e9e5fd6a51

6 条回复 • 2024-04-16 23:24:01 +08:00