From 4e654d2d184580075cf4d098f79d07e5554934b4 Mon Sep 17 00:00:00 2001 From: Nekojimi Date: Thu, 23 Jun 2022 17:57:53 +0100 Subject: [PATCH] Read format info from Youtube JSON response. --- .../java/moe/nekojimi/chords/Downloader.java | 57 ++++++++++--------- src/main/java/moe/nekojimi/chords/Format.java | 57 +++++++++++++++++-- src/main/java/moe/nekojimi/chords/Main.java | 2 +- 3 files changed, 85 insertions(+), 31 deletions(-) diff --git a/src/main/java/moe/nekojimi/chords/Downloader.java b/src/main/java/moe/nekojimi/chords/Downloader.java index 1152a9b..0abb3ad 100644 --- a/src/main/java/moe/nekojimi/chords/Downloader.java +++ b/src/main/java/moe/nekojimi/chords/Downloader.java @@ -24,6 +24,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.json.Json; +import javax.json.JsonArray; import javax.json.JsonObject; import javax.json.JsonReader; import moe.nekojimi.chords.Downloader.DownloadTask; @@ -35,13 +36,17 @@ import moe.nekojimi.chords.Downloader.DownloadTask; public class Downloader implements Consumer { + private static final int DOWNLOAD_TIMEOUT = 300; + private static final int INFO_TIMEOUT = 10; + private static final int FORMAT_TIMEOUT = 5; + private static final int BITRATE_TARGET = 64_000; private static final Pattern FORMAT_PATTERN = Pattern.compile("^([\\w]+)\\s+([\\w]+)\\s+(\\w+ ?\\w*)\\s+(.*)$"); public static final Pattern DESTINATION_PATTERN = Pattern.compile("Destination: (.*\\.wav)"); private final List downloadQueue = new LinkedList<>(); private final LinkedBlockingDeque workQueue = new LinkedBlockingDeque<>(); - private final ThreadPoolExecutor exec = new ThreadPoolExecutor(2, 4, 30, TimeUnit.SECONDS, workQueue); + private final ThreadPoolExecutor executor = new ThreadPoolExecutor(2, 4, 30, TimeUnit.SECONDS, workQueue); // private Consumer next; private BiConsumer messageHandler; @@ -65,7 +70,7 @@ public class Downloader implements Consumer downloadQueue.add(task); getInfo(song); - exec.submit(() -> + executor.submit(() -> { try { @@ -83,30 +88,25 @@ public class Downloader implements Consumer List formats = song.getFormats(); if (formats.isEmpty()) return; -// System.out.println("Choosing from " + formats.size() + " formats:"); -// System.out.println(formats); formats.sort((Format a, Format b) -> { // audio only preferred to video -// System.out.println("sort entered; a=" + a.toString() + " b=" + b.toString()); int comp = 0; comp = Boolean.compare(a.isAudioOnly(), b.isAudioOnly()); -// System.out.println("\tCompared on audio only: " + comp); if (comp == 0) { // known preferred to unknown - if (a.getBitrate() == b.getBitrate()) + if (a.getSampleRate() == b.getSampleRate()) comp = 0; - else if (a.getBitrate() <= 0) + else if (a.getSampleRate() <= 0) comp = 1; - else if (b.getBitrate() <= 0) + else if (b.getSampleRate() <= 0) comp = -1; else // closer to the target bitrate is best { - int aDist = Math.abs(a.getBitrate() - BITRATE_TARGET); - int bDist = Math.abs(b.getBitrate() - BITRATE_TARGET); + int aDist = Math.abs(a.getSampleRate() - BITRATE_TARGET); + int bDist = Math.abs(b.getSampleRate() - BITRATE_TARGET); comp = Integer.compare(bDist, aDist); -// System.out.println("\tCompared on bitrate distance: " + comp); } } if (comp == 0) @@ -114,30 +114,26 @@ public class Downloader implements Consumer // known preferred to unknown if (a.getSize() == b.getSize()) comp = 0; - if (a.getSize() <= 0) + else if (a.getSize() <= 0) comp = 1; else if (b.getSize() <= 0) comp = -1; else // smaller is better - { comp = Long.compare(b.getSize(), a.getSize()); -// System.out.println("\tCompared on filesize: " + comp); - } } -// System.out.println("\tOverall: " + comp); return -comp; }); song.setFormats(formats); -// System.out.println("Sorting done! Formats:" + formats); } private void getFormats(Song song) { - // + if (!song.getFormats().isEmpty()) + return; try { String cmd = "/usr/bin/youtube-dl --skip-download -F " + song.getUrl().toString(); - Process exec = runCommand(cmd, 5); + Process exec = runCommand(cmd, FORMAT_TIMEOUT); InputStream input = exec.getInputStream(); String output = new String(input.readAllBytes(), Charset.defaultCharset()); @@ -157,10 +153,6 @@ public class Downloader implements Consumer formats.add(new Format(split[0], split[1], split[2], split[3])); } song.setFormats(formats); -// Matcher matcher = FORMAT_PATTERN.matcher(output); -// while (matcher.find()) -// formats.add(new Format(matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(4))); -// return formats; } catch (Exception ex) { @@ -174,7 +166,7 @@ public class Downloader implements Consumer try { String cmd = "/usr/bin/youtube-dl --skip-download --print-json " + song.getUrl().toString(); - Process exec = runCommand(cmd, 5); + Process exec = runCommand(cmd, INFO_TIMEOUT); InputStream input = exec.getInputStream(); JsonReader reader = Json.createReader(input); JsonObject object = reader.readObject(); @@ -182,6 +174,19 @@ public class Downloader implements Consumer song.setTitle(object.getString("title", null)); if (song.getArtist() == null) song.setArtist(object.getString("uploader", null)); + + JsonArray formatsJSON = object.getJsonArray("formats"); + if (formatsJSON != null) + { + List formats = new ArrayList<>(); + for (JsonObject formatJson : formatsJSON.getValuesAs(JsonObject.class)) + { + Format format = Format.fromJSON(formatJson); + if (format != null) + formats.add(format); + } + song.setFormats(formats); + } } catch (Exception ex) { Logger.getLogger(Downloader.class.getName()).log(Level.SEVERE, null, ex); @@ -214,7 +219,7 @@ public class Downloader implements Consumer + " -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s " + song.getUrl().toString(); - Process exec = runCommand(cmd, 300); + Process exec = runCommand(cmd, DOWNLOAD_TIMEOUT); InputStream in = exec.getInputStream(); String output = new String(in.readAllBytes(), Charset.defaultCharset()); String error = new String(exec.getErrorStream().readAllBytes(), Charset.defaultCharset()); diff --git a/src/main/java/moe/nekojimi/chords/Format.java b/src/main/java/moe/nekojimi/chords/Format.java index 47ec8d7..5df439a 100644 --- a/src/main/java/moe/nekojimi/chords/Format.java +++ b/src/main/java/moe/nekojimi/chords/Format.java @@ -9,6 +9,7 @@ import com.amihaiemil.eoyaml.Yaml; import com.amihaiemil.eoyaml.YamlMapping; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.json.JsonObject; /** * @@ -18,12 +19,15 @@ class Format { private static final Pattern SIZE_PATTERN = Pattern.compile("\\b([0-9]+\\.?[0-9]*)([kkMmGg])i?[bB]\\b"); - private static final Pattern BITRATE_PATTERN = Pattern.compile("\\b([0-9]+)k(b(ps?))?\\b"); + private static final Pattern SAMPLE_RATE_PATTERN = Pattern.compile("\\b([0-9]+)k(b(ps?))?\\b"); private final String code; private final String extension; private final String resolution; private final String note; + private long size = -1; + private int samplerate = -1; + private boolean audioOnly = false; public Format(String code, String extension, String resolution, String note) { @@ -33,6 +37,28 @@ class Format this.note = note; } + /** + * Read format info from a youtube-dl JSON response. + * + * @param object the JSON object from youtube-dl. + * @return a new Format object if the JSON is valid, or null if not. + */ + public static Format fromJSON(JsonObject object) + { + String code = object.getString("format_id"); + String ext = object.getString("ext"); +// String res = object.getString() + String res = object.getString("format"); + String note = ""; + Format ret = new Format(code, ext, res, note); + int asr = object.getInt("asr", -1); + int size = object.getInt("filesize", -1); + ret.setSampleRate(asr); + ret.setSize(size); + return ret; + + } + public static Format fromYaml(YamlMapping yaml) { Format format = new Format( @@ -55,11 +81,21 @@ class Format public boolean isAudioOnly() { - return resolution.trim().toLowerCase().contains("audio only"); + if (audioOnly) + return true; + return resolution.trim().toLowerCase().contains("audio only") || note.trim().toLowerCase().contains("tiny"); + } + + public void setAudioOnly(boolean audioOnly) + { + this.audioOnly = audioOnly; } public long getSize() { + if (size != -1) + return size; + // try to find eg. "1.32MiB" inside note Matcher matcher = SIZE_PATTERN.matcher(note); if (matcher.find()) @@ -86,10 +122,13 @@ class Format return -1; } - public int getBitrate() + public int getSampleRate() { + if (samplerate != -1) + return samplerate; + // try to find eg. "51k" inside note - Matcher matcher = BITRATE_PATTERN.matcher(note); + Matcher matcher = SAMPLE_RATE_PATTERN.matcher(note); if (matcher.find()) { return Integer.parseInt(matcher.group(1)) * 1000; @@ -97,6 +136,16 @@ class Format return -1; } + public void setSampleRate(int samplerate) + { + this.samplerate = samplerate; + } + + public void setSize(long size) + { + this.size = size; + } + public String getCode() { return code; diff --git a/src/main/java/moe/nekojimi/chords/Main.java b/src/main/java/moe/nekojimi/chords/Main.java index 6984e8c..c4aa11b 100644 --- a/src/main/java/moe/nekojimi/chords/Main.java +++ b/src/main/java/moe/nekojimi/chords/Main.java @@ -118,7 +118,7 @@ public final class Main extends ListenerAdapter String formatDetails = ""; if (format != null) { - final int bitrate = format.getBitrate() / 1000; + final int bitrate = format.getSampleRate() / 1000; final long size = format.getSize(); String sizeFmt = (size <= 0 ? "?.??" : String.format("%.2f", size / (1024.0 * 1024.0))) + "MiB"; String bitFmt = (bitrate <= 0 ? "??" : bitrate) + "k";