Nekojimi
/
Chords
1
0
Fork 0
Code Issues 14 Pull Requests Projects Releases Wiki Activity

Optimise downloads by selecting download format manually.

This commit is contained in:
Nekojimi 2021-11-20 00:40:52 +00:00
parent cc17eb0bd9
commit 83bb9e5805
2 changed files with 218 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
src/main/java/moe/nekojimi/chords/Downloader.java
View File

@ -10,11 +10,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ -24,6 +22,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
@ -34,6 +33,10 @@ import javax.json.JsonReader;
*/
public class Downloader implements Consumer<Song>
{
private static final int BITRATE_TARGET = 64_000;
private static final Pattern FORMAT_PATTERN = Pattern.compile("^([\\w]+)\\s+([\\w]+)\\s+(\\w+ ?\\w*)\\s+(.*)$");
private final List<Song> downloadQueue = new LinkedList<>();
private final LinkedBlockingDeque<Runnable> workQueue = new LinkedBlockingDeque<>();
private final ThreadPoolExecutor exec = new ThreadPoolExecutor(2, 4, 30, TimeUnit.SECONDS, workQueue);
@ -57,7 +60,15 @@ public class Downloader implements Consumer<Song>
@Override
public void run()
{
download(song);
try
{
List<Format> formats = getFormats(song);
Format chosenFormat = chooseFormat(formats);
download(song, chosenFormat);
} catch (Exception ex)
{
ex.printStackTrace();
}
}
});
}
@ -67,6 +78,95 @@ public class Downloader implements Consumer<Song>
this.next = next;
}
private Format chooseFormat(List<Format> formats)
{
if (formats.isEmpty())
return null;
System.out.println("Choosing from " + formats.size() + " formats:");
System.out.println(formats);
formats.sort((Format a, Format b) ->
{
// audio only preferred to video
System.out.println("sort entered; a=" + a.toString() + " b=" + b.toString());
int comp = 0;
comp = Boolean.compare(a.isAudioOnly(), b.isAudioOnly());
System.out.println("\tCompared on audio only: " + comp);
if (comp == 0)
{
// known preferred to unknown
if (a.getBitrate() == b.getBitrate())
comp = 0;
else if (a.getBitrate() <= 0)
comp = 1;
else if (b.getBitrate() <= 0)
comp = -1;
else // closer to the target bitrate is best
{
int aDist = Math.abs(a.getBitrate() - BITRATE_TARGET);
int bDist = Math.abs(b.getBitrate() - BITRATE_TARGET);
comp = Integer.compare(bDist, aDist);
System.out.println("\tCompared on bitrate distance: " + comp);
}
}
if (comp == 0)
{
// known preferred to unknown
if (a.getSize() == b.getSize())
comp = 0;
if (a.getSize() <= 0)
comp = 1;
else if (b.getSize() <= 0)
comp = -1;
else // smaller is better
{
comp = Long.compare(b.getSize(), a.getSize());
System.out.println("\tCompared on filesize: " + comp);
}
}
System.out.println("\tOverall: " + comp);
return comp;
});
System.out.println("Sorting done! Formats:" + formats);
return formats.get(formats.size() - 1);
}
private List<Format> getFormats(Song song)
{
//
try
{
String cmd = "/usr/bin/youtube-dl --skip-download -F " + song.getUrl().toString();
Process exec = runCommand(cmd, 5);
InputStream input = exec.getInputStream();
String output = new String(input.readAllBytes(), Charset.defaultCharset());
List<Format> formats = new ArrayList<>();
List<String> list = output.lines().collect(Collectors.toList());
int i = 0;
while (!list.get(i).contains("Available formats"))
i++;
i++;
for (; i < list.size(); i++)
{
String line = list.get(i);
String[] split = line.split("\\s\\s+", 4);
if (split.length < 4)
continue;
formats.add(new Format(split[0], split[1], split[2], split[3]));
}
// Matcher matcher = FORMAT_PATTERN.matcher(output);
// while (matcher.find())
// formats.add(new Format(matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(4)));
return formats;
} catch (Exception ex)
{
Logger.getLogger(Downloader.class.getName()).log(Level.SEVERE, null, ex);
return List.of();
}
}
private void getInfo(Song song)
{
try
@ -93,13 +193,20 @@ public class Downloader implements Consumer<Song>
return downloadDir;
}
private void download(Song song)
private void download(Song song, Format format)
{
final String formatCodes = format != null ? format.getCode() + "/" : "";
try
{
messageHandler.accept(song, null);
String cmd = "/usr/bin/youtube-dl -x -f=worstaudio/worst --audio-format=wav --no-playlist -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s " + song.getUrl().toString();
String cmd = "/usr/bin/youtube-dl -x"
+ " -f " + formatCodes + "worstaudio/bestaudio/worst/best "
+ " --audio-format=wav"
+ " --no-playlist"
+ " -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s "
+ song.getUrl().toString();
Process exec = runCommand(cmd, 300);
InputStream in = exec.getInputStream();
String output = new String(in.readAllBytes(), Charset.defaultCharset());

104
src/main/java/moe/nekojimi/chords/Format.java Normal file
View File

@ -0,0 +1,104 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package moe.nekojimi.chords;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
*
* @author jimj316
*/
class Format
{
private static final Pattern SIZE_PATTERN = Pattern.compile("\\b([0-9]+\\.?[0-9]*)([kkMmGg])i?[bB]\\b");
private static final Pattern BITRATE_PATTERN = Pattern.compile("\\b([0-9]+)k(b(ps?))?\\b");
private final String code;
private final String extension;
private final String resolution;
private final String note;
public Format(String code, String extension, String resolution, String note)
{
this.code = code;
this.extension = extension;
this.resolution = resolution;
this.note = note;
}
public boolean isAudioOnly()
{
return resolution.trim().toLowerCase().contains("audio only");
}
public long getSize()
{
// try to find eg. "1.32MiB" inside note
Matcher matcher = SIZE_PATTERN.matcher(note);
if (matcher.find())
{
double value = Double.parseDouble(matcher.group(1));
String mag = matcher.group(2).toUpperCase();
long mult = 1;
switch (mag)
{
case "K":
mult = 1024;
break;
case "M":
mult = 1024 * 1024;
break;
case "G":
mult = 1024 * 1024 * 1024;
break;
}
value *= mult;
return (long) value;
}
return -1;
}
public int getBitrate()
{
// try to find eg. "51k" inside note
Matcher matcher = BITRATE_PATTERN.matcher(note);
if (matcher.find())
{
return Integer.parseInt(matcher.group(1)) * 1000;
}
return -1;
}
public String getCode()
{
return code;
}
public String getExtension()
{
return extension;
}
public String getResolution()
{
return resolution;
}
public String getNote()
{
return note;
}
@Override
public String toString()
{
return "Format{" + "code=" + code + ", extension=" + extension + ", resolution=" + resolution + ", note=" + note + '}';
}
}