From 83bb9e580523e13a1c15cf9c174baf3fd9f7928c Mon Sep 17 00:00:00 2001
From: Nekojimi <Jim@nekojimi.moe>
Date: Sat, 20 Nov 2021 00:40:52 +0000
Subject: [PATCH] Optimise downloads by selecting download format manually.
---
.../java/moe/nekojimi/chords/Downloader.java | 121 +++++++++++++++++-
src/main/java/moe/nekojimi/chords/Format.java | 104 +++++++++++++++
2 files changed, 218 insertions(+), 7 deletions(-)
create mode 100644 src/main/java/moe/nekojimi/chords/Format.java
diff --git a/src/main/java/moe/nekojimi/chords/Downloader.java b/src/main/java/moe/nekojimi/chords/Downloader.java
index f030efd..601dbad 100644
--- a/src/main/java/moe/nekojimi/chords/Downloader.java
+++ b/src/main/java/moe/nekojimi/chords/Downloader.java
@@ -10,11 +10,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
-import java.nio.file.attribute.FileAttribute;
-import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
-import java.util.Queue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -24,6 +22,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
@@ -34,6 +33,10 @@ import javax.json.JsonReader;
*/
public class Downloader implements Consumer<Song>
{
+
+ private static final int BITRATE_TARGET = 64_000;
+ private static final Pattern FORMAT_PATTERN = Pattern.compile("^([\\w]+)\\s+([\\w]+)\\s+(\\w+ ?\\w*)\\s+(.*)$");
+
private final List<Song> downloadQueue = new LinkedList<>();
private final LinkedBlockingDeque<Runnable> workQueue = new LinkedBlockingDeque<>();
private final ThreadPoolExecutor exec = new ThreadPoolExecutor(2, 4, 30, TimeUnit.SECONDS, workQueue);
@@ -57,7 +60,15 @@ public class Downloader implements Consumer<Song>
@Override
public void run()
{
- download(song);
+ try
+ {
+ List<Format> formats = getFormats(song);
+ Format chosenFormat = chooseFormat(formats);
+ download(song, chosenFormat);
+ } catch (Exception ex)
+ {
+ ex.printStackTrace();
+ }
}
});
}
@@ -67,6 +78,95 @@ public class Downloader implements Consumer<Song>
this.next = next;
}
+ private Format chooseFormat(List<Format> formats)
+ {
+ if (formats.isEmpty())
+ return null;
+ System.out.println("Choosing from " + formats.size() + " formats:");
+ System.out.println(formats);
+ formats.sort((Format a, Format b) ->
+ {
+ // audio only preferred to video
+ System.out.println("sort entered; a=" + a.toString() + " b=" + b.toString());
+ int comp = 0;
+ comp = Boolean.compare(a.isAudioOnly(), b.isAudioOnly());
+ System.out.println("\tCompared on audio only: " + comp);
+ if (comp == 0)
+ {
+ // known preferred to unknown
+ if (a.getBitrate() == b.getBitrate())
+ comp = 0;
+ else if (a.getBitrate() <= 0)
+ comp = 1;
+ else if (b.getBitrate() <= 0)
+ comp = -1;
+ else // closer to the target bitrate is best
+ {
+ int aDist = Math.abs(a.getBitrate() - BITRATE_TARGET);
+ int bDist = Math.abs(b.getBitrate() - BITRATE_TARGET);
+ comp = Integer.compare(bDist, aDist);
+ System.out.println("\tCompared on bitrate distance: " + comp);
+ }
+ }
+ if (comp == 0)
+ {
+ // known preferred to unknown
+ if (a.getSize() == b.getSize())
+ comp = 0;
+ if (a.getSize() <= 0)
+ comp = 1;
+ else if (b.getSize() <= 0)
+ comp = -1;
+ else // smaller is better
+ {
+ comp = Long.compare(b.getSize(), a.getSize());
+ System.out.println("\tCompared on filesize: " + comp);
+ }
+ }
+ System.out.println("\tOverall: " + comp);
+ return comp;
+ });
+ System.out.println("Sorting done! Formats:" + formats);
+ return formats.get(formats.size() - 1);
+ }
+
+ private List<Format> getFormats(Song song)
+ {
+ //
+ try
+ {
+ String cmd = "/usr/bin/youtube-dl --skip-download -F " + song.getUrl().toString();
+ Process exec = runCommand(cmd, 5);
+ InputStream input = exec.getInputStream();
+ String output = new String(input.readAllBytes(), Charset.defaultCharset());
+
+ List<Format> formats = new ArrayList<>();
+
+ List<String> list = output.lines().collect(Collectors.toList());
+ int i = 0;
+ while (!list.get(i).contains("Available formats"))
+ i++;
+ i++;
+ for (; i < list.size(); i++)
+ {
+ String line = list.get(i);
+ String[] split = line.split("\\s\\s+", 4);
+ if (split.length < 4)
+ continue;
+ formats.add(new Format(split[0], split[1], split[2], split[3]));
+ }
+// Matcher matcher = FORMAT_PATTERN.matcher(output);
+// while (matcher.find())
+// formats.add(new Format(matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(4)));
+ return formats;
+
+ } catch (Exception ex)
+ {
+ Logger.getLogger(Downloader.class.getName()).log(Level.SEVERE, null, ex);
+ return List.of();
+ }
+ }
+
private void getInfo(Song song)
{
try
@@ -93,13 +193,20 @@ public class Downloader implements Consumer<Song>
return downloadDir;
}
- private void download(Song song)
+ private void download(Song song, Format format)
{
+ final String formatCodes = format != null ? format.getCode() + "/" : "";
+
try
{
-
messageHandler.accept(song, null);
- String cmd = "/usr/bin/youtube-dl -x -f=worstaudio/worst --audio-format=wav --no-playlist -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s " + song.getUrl().toString();
+ String cmd = "/usr/bin/youtube-dl -x"
+ + " -f " + formatCodes + "worstaudio/bestaudio/worst/best "
+ + " --audio-format=wav"
+ + " --no-playlist"
+ + " -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s "
+ + song.getUrl().toString();
+
Process exec = runCommand(cmd, 300);
InputStream in = exec.getInputStream();
String output = new String(in.readAllBytes(), Charset.defaultCharset());
diff --git a/src/main/java/moe/nekojimi/chords/Format.java b/src/main/java/moe/nekojimi/chords/Format.java
new file mode 100644
index 0000000..81a3808
--- /dev/null
+++ b/src/main/java/moe/nekojimi/chords/Format.java
@@ -0,0 +1,104 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package moe.nekojimi.chords;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ *
+ * @author jimj316
+ */
+class Format
+{
+
+ private static final Pattern SIZE_PATTERN = Pattern.compile("\\b([0-9]+\\.?[0-9]*)([kkMmGg])i?[bB]\\b");
+ private static final Pattern BITRATE_PATTERN = Pattern.compile("\\b([0-9]+)k(b(ps?))?\\b");
+
+ private final String code;
+ private final String extension;
+ private final String resolution;
+ private final String note;
+
+ public Format(String code, String extension, String resolution, String note)
+ {
+ this.code = code;
+ this.extension = extension;
+ this.resolution = resolution;
+ this.note = note;
+ }
+
+ public boolean isAudioOnly()
+ {
+ return resolution.trim().toLowerCase().contains("audio only");
+ }
+
+ public long getSize()
+ {
+ // try to find eg. "1.32MiB" inside note
+ Matcher matcher = SIZE_PATTERN.matcher(note);
+ if (matcher.find())
+ {
+ double value = Double.parseDouble(matcher.group(1));
+ String mag = matcher.group(2).toUpperCase();
+ long mult = 1;
+ switch (mag)
+ {
+ case "K":
+ mult = 1024;
+ break;
+ case "M":
+ mult = 1024 * 1024;
+ break;
+ case "G":
+ mult = 1024 * 1024 * 1024;
+ break;
+ }
+ value *= mult;
+ return (long) value;
+
+ }
+ return -1;
+ }
+
+ public int getBitrate()
+ {
+ // try to find eg. "51k" inside note
+ Matcher matcher = BITRATE_PATTERN.matcher(note);
+ if (matcher.find())
+ {
+ return Integer.parseInt(matcher.group(1)) * 1000;
+ }
+ return -1;
+ }
+
+ public String getCode()
+ {
+ return code;
+ }
+
+ public String getExtension()
+ {
+ return extension;
+ }
+
+ public String getResolution()
+ {
+ return resolution;
+ }
+
+ public String getNote()
+ {
+ return note;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Format{" + "code=" + code + ", extension=" + extension + ", resolution=" + resolution + ", note=" + note + '}';
+ }
+
+}