From 83bb9e580523e13a1c15cf9c174baf3fd9f7928c Mon Sep 17 00:00:00 2001 From: Nekojimi Date: Sat, 20 Nov 2021 00:40:52 +0000 Subject: [PATCH] Optimise downloads by selecting download format manually. --- .../java/moe/nekojimi/chords/Downloader.java | 121 +++++++++++++++++- src/main/java/moe/nekojimi/chords/Format.java | 104 +++++++++++++++ 2 files changed, 218 insertions(+), 7 deletions(-) create mode 100644 src/main/java/moe/nekojimi/chords/Format.java diff --git a/src/main/java/moe/nekojimi/chords/Downloader.java b/src/main/java/moe/nekojimi/chords/Downloader.java index f030efd..601dbad 100644 --- a/src/main/java/moe/nekojimi/chords/Downloader.java +++ b/src/main/java/moe/nekojimi/chords/Downloader.java @@ -10,11 +10,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; import java.nio.file.Files; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; -import java.util.Queue; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -24,6 +22,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.json.Json; import javax.json.JsonObject; import javax.json.JsonReader; @@ -34,6 +33,10 @@ import javax.json.JsonReader; */ public class Downloader implements Consumer { + + private static final int BITRATE_TARGET = 64_000; + private static final Pattern FORMAT_PATTERN = Pattern.compile("^([\\w]+)\\s+([\\w]+)\\s+(\\w+ ?\\w*)\\s+(.*)$"); + private final List downloadQueue = new LinkedList<>(); private final LinkedBlockingDeque workQueue = new LinkedBlockingDeque<>(); private final ThreadPoolExecutor exec = new ThreadPoolExecutor(2, 4, 30, TimeUnit.SECONDS, workQueue); @@ -57,7 +60,15 @@ public class Downloader implements Consumer @Override public void run() { - download(song); + try + { + List formats = getFormats(song); + Format chosenFormat = chooseFormat(formats); + download(song, chosenFormat); + } catch (Exception ex) + { + ex.printStackTrace(); + } } }); } @@ -67,6 +78,95 @@ public class Downloader implements Consumer this.next = next; } + private Format chooseFormat(List formats) + { + if (formats.isEmpty()) + return null; + System.out.println("Choosing from " + formats.size() + " formats:"); + System.out.println(formats); + formats.sort((Format a, Format b) -> + { + // audio only preferred to video + System.out.println("sort entered; a=" + a.toString() + " b=" + b.toString()); + int comp = 0; + comp = Boolean.compare(a.isAudioOnly(), b.isAudioOnly()); + System.out.println("\tCompared on audio only: " + comp); + if (comp == 0) + { + // known preferred to unknown + if (a.getBitrate() == b.getBitrate()) + comp = 0; + else if (a.getBitrate() <= 0) + comp = 1; + else if (b.getBitrate() <= 0) + comp = -1; + else // closer to the target bitrate is best + { + int aDist = Math.abs(a.getBitrate() - BITRATE_TARGET); + int bDist = Math.abs(b.getBitrate() - BITRATE_TARGET); + comp = Integer.compare(bDist, aDist); + System.out.println("\tCompared on bitrate distance: " + comp); + } + } + if (comp == 0) + { + // known preferred to unknown + if (a.getSize() == b.getSize()) + comp = 0; + if (a.getSize() <= 0) + comp = 1; + else if (b.getSize() <= 0) + comp = -1; + else // smaller is better + { + comp = Long.compare(b.getSize(), a.getSize()); + System.out.println("\tCompared on filesize: " + comp); + } + } + System.out.println("\tOverall: " + comp); + return comp; + }); + System.out.println("Sorting done! Formats:" + formats); + return formats.get(formats.size() - 1); + } + + private List getFormats(Song song) + { + // + try + { + String cmd = "/usr/bin/youtube-dl --skip-download -F " + song.getUrl().toString(); + Process exec = runCommand(cmd, 5); + InputStream input = exec.getInputStream(); + String output = new String(input.readAllBytes(), Charset.defaultCharset()); + + List formats = new ArrayList<>(); + + List list = output.lines().collect(Collectors.toList()); + int i = 0; + while (!list.get(i).contains("Available formats")) + i++; + i++; + for (; i < list.size(); i++) + { + String line = list.get(i); + String[] split = line.split("\\s\\s+", 4); + if (split.length < 4) + continue; + formats.add(new Format(split[0], split[1], split[2], split[3])); + } +// Matcher matcher = FORMAT_PATTERN.matcher(output); +// while (matcher.find()) +// formats.add(new Format(matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(4))); + return formats; + + } catch (Exception ex) + { + Logger.getLogger(Downloader.class.getName()).log(Level.SEVERE, null, ex); + return List.of(); + } + } + private void getInfo(Song song) { try @@ -93,13 +193,20 @@ public class Downloader implements Consumer return downloadDir; } - private void download(Song song) + private void download(Song song, Format format) { + final String formatCodes = format != null ? format.getCode() + "/" : ""; + try { - messageHandler.accept(song, null); - String cmd = "/usr/bin/youtube-dl -x -f=worstaudio/worst --audio-format=wav --no-playlist -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s " + song.getUrl().toString(); + String cmd = "/usr/bin/youtube-dl -x" + + " -f " + formatCodes + "worstaudio/bestaudio/worst/best " + + " --audio-format=wav" + + " --no-playlist" + + " -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s " + + song.getUrl().toString(); + Process exec = runCommand(cmd, 300); InputStream in = exec.getInputStream(); String output = new String(in.readAllBytes(), Charset.defaultCharset()); diff --git a/src/main/java/moe/nekojimi/chords/Format.java b/src/main/java/moe/nekojimi/chords/Format.java new file mode 100644 index 0000000..81a3808 --- /dev/null +++ b/src/main/java/moe/nekojimi/chords/Format.java @@ -0,0 +1,104 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package moe.nekojimi.chords; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * + * @author jimj316 + */ +class Format +{ + + private static final Pattern SIZE_PATTERN = Pattern.compile("\\b([0-9]+\\.?[0-9]*)([kkMmGg])i?[bB]\\b"); + private static final Pattern BITRATE_PATTERN = Pattern.compile("\\b([0-9]+)k(b(ps?))?\\b"); + + private final String code; + private final String extension; + private final String resolution; + private final String note; + + public Format(String code, String extension, String resolution, String note) + { + this.code = code; + this.extension = extension; + this.resolution = resolution; + this.note = note; + } + + public boolean isAudioOnly() + { + return resolution.trim().toLowerCase().contains("audio only"); + } + + public long getSize() + { + // try to find eg. "1.32MiB" inside note + Matcher matcher = SIZE_PATTERN.matcher(note); + if (matcher.find()) + { + double value = Double.parseDouble(matcher.group(1)); + String mag = matcher.group(2).toUpperCase(); + long mult = 1; + switch (mag) + { + case "K": + mult = 1024; + break; + case "M": + mult = 1024 * 1024; + break; + case "G": + mult = 1024 * 1024 * 1024; + break; + } + value *= mult; + return (long) value; + + } + return -1; + } + + public int getBitrate() + { + // try to find eg. "51k" inside note + Matcher matcher = BITRATE_PATTERN.matcher(note); + if (matcher.find()) + { + return Integer.parseInt(matcher.group(1)) * 1000; + } + return -1; + } + + public String getCode() + { + return code; + } + + public String getExtension() + { + return extension; + } + + public String getResolution() + { + return resolution; + } + + public String getNote() + { + return note; + } + + @Override + public String toString() + { + return "Format{" + "code=" + code + ", extension=" + extension + ", resolution=" + resolution + ", note=" + note + '}'; + } + +}