|
|
|
@ -8,15 +8,11 @@ package moe.nekojimi.chords; |
|
|
|
|
import java.io.File; |
|
|
|
|
import java.io.IOException; |
|
|
|
|
import java.io.InputStream; |
|
|
|
|
import java.io.StringReader; |
|
|
|
|
import java.nio.charset.Charset; |
|
|
|
|
import java.nio.file.Files; |
|
|
|
|
import java.util.ArrayList; |
|
|
|
|
import java.util.LinkedList; |
|
|
|
|
import java.util.List; |
|
|
|
|
import java.util.Scanner; |
|
|
|
|
import java.util.concurrent.LinkedBlockingDeque; |
|
|
|
|
import java.util.concurrent.ThreadPoolExecutor; |
|
|
|
|
import java.util.concurrent.TimeUnit; |
|
|
|
|
import java.util.*; |
|
|
|
|
import java.util.concurrent.*; |
|
|
|
|
import java.util.function.BiConsumer; |
|
|
|
|
import java.util.function.Consumer; |
|
|
|
|
import java.util.logging.Level; |
|
|
|
@ -39,7 +35,7 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
{ |
|
|
|
|
|
|
|
|
|
private static final int DOWNLOAD_TIMEOUT = 300; |
|
|
|
|
private static final int INFO_TIMEOUT = 30; |
|
|
|
|
private static final int INFO_TIMEOUT = 60; |
|
|
|
|
private static final int FORMAT_TIMEOUT = 5; |
|
|
|
|
|
|
|
|
|
private static final int BITRATE_TARGET = (int) AudioSendHandler.INPUT_FORMAT.getSampleRate(); |
|
|
|
@ -47,15 +43,19 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
public static final Pattern DESTINATION_PATTERN = Pattern.compile("Destination: (.*\\.wav)"); |
|
|
|
|
public static final Pattern PROGRESS_PATTERN = Pattern.compile("\\[download\\].*?([\\d\\.]+)%"); |
|
|
|
|
private static final Pattern ETA_PATTERN = Pattern.compile("\\[download\\].*?ETA\\s+(\\d{1,2}:\\d{2})"); |
|
|
|
|
private static final Pattern DOWNLOAD_ITEM_PATTERN = Pattern.compile("\\[download\\] Downloading item (\\d+) of (\\d+)"); |
|
|
|
|
|
|
|
|
|
private final List<DownloadTask> downloadQueue = new LinkedList<>(); |
|
|
|
|
private final LinkedBlockingDeque<Runnable> workQueue = new LinkedBlockingDeque<>(); |
|
|
|
|
private final ThreadPoolExecutor executor = new ThreadPoolExecutor(2, 4, 30, TimeUnit.SECONDS, workQueue); |
|
|
|
|
private final ScheduledExecutorService scheduler = new ScheduledThreadPoolExecutor(1); |
|
|
|
|
// private Consumer<Song> next;
|
|
|
|
|
private BiConsumer<SongRequest, Exception> messageHandler; |
|
|
|
|
|
|
|
|
|
private File downloadDir = null; |
|
|
|
|
|
|
|
|
|
private int trackNumber = 1; |
|
|
|
|
|
|
|
|
|
public Downloader() |
|
|
|
|
{ |
|
|
|
|
|
|
|
|
@ -64,21 +64,22 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
@Override |
|
|
|
|
public void accept(DownloadTask task) |
|
|
|
|
{ |
|
|
|
|
// if already downloaded, just skip
|
|
|
|
|
Song song = task.request.getSong(); |
|
|
|
|
if (song.isDownloaded()) |
|
|
|
|
// if all songs of the request are already downloaded, just skip
|
|
|
|
|
if (!task.request.getSongs().isEmpty() && task.request.getSongs().stream().allMatch((t) -> t.isDownloaded())) |
|
|
|
|
{ |
|
|
|
|
for (Song song : task.request.getSongs()) |
|
|
|
|
task.getDestination().accept(song); |
|
|
|
|
return; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
downloadQueue.add(task); |
|
|
|
|
getInfo(song); |
|
|
|
|
getInfo(task.request); |
|
|
|
|
// TODO: get info should also use the thread pool
|
|
|
|
|
executor.submit(() -> |
|
|
|
|
{ |
|
|
|
|
try |
|
|
|
|
{ |
|
|
|
|
getFormats(song); |
|
|
|
|
// getFormats(song);
|
|
|
|
|
download(task); |
|
|
|
|
} catch (Exception ex) |
|
|
|
|
{ |
|
|
|
@ -87,11 +88,9 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
}); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void chooseFormats(Song song) |
|
|
|
|
private List<Format> sortFormats(Collection<Format> input) |
|
|
|
|
{ |
|
|
|
|
List<Format> formats = song.getFormats(); |
|
|
|
|
if (formats.isEmpty()) |
|
|
|
|
return; |
|
|
|
|
List<Format> formats = new ArrayList<>(input); |
|
|
|
|
formats.sort((Format a, Format b) -> |
|
|
|
|
{ |
|
|
|
|
// audio only preferred to video
|
|
|
|
@ -127,7 +126,7 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
} |
|
|
|
|
return -comp; |
|
|
|
|
}); |
|
|
|
|
song.setFormats(formats); |
|
|
|
|
return formats; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void getFormats(Song song) |
|
|
|
@ -205,14 +204,26 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void getInfo(Song song) |
|
|
|
|
private List<Song> getInfo(SongRequest request) |
|
|
|
|
{ |
|
|
|
|
List<Song> ret = new ArrayList<>(); |
|
|
|
|
try |
|
|
|
|
{ |
|
|
|
|
String cmd = Chords.getSettings().getYtdlCommand() + " --skip-download --print-json " + song.getUrl().toString(); |
|
|
|
|
String cmd = Chords.getSettings().getYtdlCommand() + " --skip-download --print-json " + request.getUrl().toString(); |
|
|
|
|
Process exec = runCommand(cmd, INFO_TIMEOUT); |
|
|
|
|
InputStream input = exec.getInputStream(); |
|
|
|
|
JsonReader reader = Json.createReader(input); |
|
|
|
|
|
|
|
|
|
// read each line as JSON, turn each into a song object
|
|
|
|
|
Scanner sc = new Scanner(input); |
|
|
|
|
while (sc.hasNextLine()) |
|
|
|
|
{ |
|
|
|
|
Song song = new Song(request.getUrl()); |
|
|
|
|
song.setNumber(trackNumber); |
|
|
|
|
trackNumber++; |
|
|
|
|
request.addSong(song); |
|
|
|
|
|
|
|
|
|
String line = sc.nextLine(); |
|
|
|
|
JsonReader reader = Json.createReader(new StringReader(line)); |
|
|
|
|
JsonObject object = reader.readObject(); |
|
|
|
|
if (song.getTitle() == null) |
|
|
|
|
song.setTitle(object.getString("title", null)); |
|
|
|
@ -231,10 +242,14 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
} |
|
|
|
|
song.setFormats(formats); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
ret.add(song); |
|
|
|
|
} |
|
|
|
|
} catch (Exception ex) |
|
|
|
|
{ |
|
|
|
|
Logger.getLogger(Downloader.class.getName()).log(Level.SEVERE, null, ex); |
|
|
|
|
} |
|
|
|
|
return ret; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private File getDownloadDir() throws IOException |
|
|
|
@ -244,15 +259,35 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
return downloadDir; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private Song getSongFromRequest(SongRequest request, int idx) |
|
|
|
|
{ |
|
|
|
|
// if there's less songs in the request than expected, fill the array
|
|
|
|
|
while (idx >= request.getSongs().size()) |
|
|
|
|
{ |
|
|
|
|
Song song = new Song(request.getUrl()); |
|
|
|
|
song.setNumber(trackNumber); |
|
|
|
|
trackNumber++; |
|
|
|
|
request.addSong(song); |
|
|
|
|
} |
|
|
|
|
return request.getSongs().get(idx); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void download(DownloadTask task) |
|
|
|
|
{ |
|
|
|
|
Song song = task.request.getSong(); |
|
|
|
|
chooseFormats(song); |
|
|
|
|
Set<Format> uniqueFormats = new HashSet<>(); |
|
|
|
|
for (Song song : task.request.getSongs()) |
|
|
|
|
{ |
|
|
|
|
uniqueFormats.addAll(song.getFormats()); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
List<Format> sortedFormats = sortFormats(uniqueFormats); |
|
|
|
|
String formatCodes = ""; |
|
|
|
|
final List<Format> formats = song.getFormats(); |
|
|
|
|
for (int i = 0; i < 3 && i < song.getFormats().size(); i++) |
|
|
|
|
final List<Format> formats = sortedFormats; |
|
|
|
|
for (int i = 0; i < 5 && i < sortedFormats.size(); i++) |
|
|
|
|
formatCodes += formats.get(i).getCode() + "/"; |
|
|
|
|
|
|
|
|
|
int downloadIdx = 0; |
|
|
|
|
|
|
|
|
|
try |
|
|
|
|
{ |
|
|
|
|
messageHandler.accept(task.request, null); |
|
|
|
@ -263,7 +298,7 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
+ " --no-playlist" |
|
|
|
|
+ " --extractor-args youtube:player_client=android" |
|
|
|
|
+ " -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s " |
|
|
|
|
+ song.getUrl().toString(); |
|
|
|
|
+ task.request.getUrl().toString(); |
|
|
|
|
|
|
|
|
|
Process exec = runCommand(cmd, DOWNLOAD_TIMEOUT); |
|
|
|
|
InputStream in = exec.getInputStream(); |
|
|
|
@ -274,30 +309,47 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
String line = sc.nextLine(); |
|
|
|
|
System.out.println(line); |
|
|
|
|
|
|
|
|
|
Matcher itemMatcher = DOWNLOAD_ITEM_PATTERN.matcher(line); |
|
|
|
|
if (itemMatcher.find()) |
|
|
|
|
{ |
|
|
|
|
int idx = Integer.parseInt(itemMatcher.group(1)) - 1; |
|
|
|
|
int total = Integer.parseInt(cmd); |
|
|
|
|
|
|
|
|
|
downloadIdx = idx; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Matcher progMatcher = PROGRESS_PATTERN.matcher(line); |
|
|
|
|
if (progMatcher.find()) |
|
|
|
|
{ |
|
|
|
|
task.request.setProgress(Double.parseDouble(progMatcher.group(1))); |
|
|
|
|
getSongFromRequest(task.request, downloadIdx).setProgress(Double.parseDouble(progMatcher.group(1))); |
|
|
|
|
messageHandler.accept(task.request, null); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Matcher destMatcher = DESTINATION_PATTERN.matcher(line); |
|
|
|
|
if (destMatcher.find()) |
|
|
|
|
{ |
|
|
|
|
Song song = getSongFromRequest(task.request, downloadIdx); |
|
|
|
|
|
|
|
|
|
song.setLocation(new File(destMatcher.group(1))); |
|
|
|
|
|
|
|
|
|
// this is currently our criteria for completion; submit the song and move on
|
|
|
|
|
if (task.getDestination() != null) |
|
|
|
|
task.getDestination().accept(song); |
|
|
|
|
|
|
|
|
|
downloadIdx++; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// String output = new String(in.readAllBytes(), Charset.defaultCharset());
|
|
|
|
|
// String output = new String(in.readAllBytes(), Charset.defaultCharset());
|
|
|
|
|
String error = new String(exec.getErrorStream().readAllBytes(), Charset.defaultCharset()); |
|
|
|
|
// System.out.println(output);
|
|
|
|
|
// System.out.println(output);
|
|
|
|
|
|
|
|
|
|
if (exec.exitValue() != 0) |
|
|
|
|
throw new RuntimeException("youtube-dl failed with error " + exec.exitValue() + ", output:\n" + error); |
|
|
|
|
|
|
|
|
|
task.request.setProgress(100); |
|
|
|
|
// return true;
|
|
|
|
|
// task.request.setProgress(100);
|
|
|
|
|
// return true;
|
|
|
|
|
|
|
|
|
|
if (task.getDestination() != null) |
|
|
|
|
task.getDestination().accept(song); |
|
|
|
|
downloadQueue.remove(task); |
|
|
|
|
messageHandler.accept(task.request, null); |
|
|
|
|
} catch (Exception ex) |
|
|
|
@ -314,12 +366,20 @@ public class Downloader implements Consumer<DownloadTask> |
|
|
|
|
System.out.println("Running command: " + cmd); |
|
|
|
|
// Process exec = Runtime.getRuntime().exec().split(" "));
|
|
|
|
|
Process exec = new ProcessBuilder(cmd.split(" ")).redirectOutput(ProcessBuilder.Redirect.PIPE).redirectError(ProcessBuilder.Redirect.PIPE).start(); |
|
|
|
|
boolean done = exec.waitFor(timeoutSecs, TimeUnit.SECONDS); |
|
|
|
|
if (!done) |
|
|
|
|
scheduler.schedule(() -> |
|
|
|
|
{ |
|
|
|
|
if (exec.isAlive()) |
|
|
|
|
{ |
|
|
|
|
exec.destroyForcibly(); |
|
|
|
|
throw new RuntimeException("Took too long, giving up."); |
|
|
|
|
} |
|
|
|
|
System.err.println("Process " + cmd + " took too long, killing process."); |
|
|
|
|
} |
|
|
|
|
}, timeoutSecs, TimeUnit.SECONDS); |
|
|
|
|
// boolean done = exec.waitFor(timeoutSecs, TimeUnit.SECONDS);
|
|
|
|
|
// if (!done)
|
|
|
|
|
// {
|
|
|
|
|
// exec.destroyForcibly();
|
|
|
|
|
// throw new RuntimeException("Took too long, giving up.");
|
|
|
|
|
// }
|
|
|
|
|
return exec; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|