Add support for downloading entire playlists.

master
Nekojimi 1 year ago
parent de807b4524
commit 310158a39b
  1. 88
      src/main/java/moe/nekojimi/chords/Chords.java
  2. 134
      src/main/java/moe/nekojimi/chords/Downloader.java
  3. 148
      src/main/java/moe/nekojimi/chords/MusicHandler.java
  4. 61
      src/main/java/moe/nekojimi/chords/Song.java
  5. 34
      src/main/java/moe/nekojimi/chords/SongRequest.java

@ -30,7 +30,6 @@ import net.dv8tion.jda.api.entities.*;
import net.dv8tion.jda.api.entities.channel.middleman.AudioChannel;
import net.dv8tion.jda.api.events.guild.voice.GuildVoiceUpdateEvent;
import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
import net.dv8tion.jda.api.hooks.ListenerAdapter;
import net.dv8tion.jda.api.managers.AudioManager;
import net.dv8tion.jda.api.requests.GatewayIntent;
@ -65,8 +64,6 @@ public final class Chords extends ListenerAdapter
private AudioChannel currentVoiceChannel = null;
private int trackNumber = 1;
private final Map<String, Playlist> playlists = new HashMap<>();
/**
@ -256,49 +253,29 @@ public final class Chords extends ListenerAdapter
}
}
public Song queueDownload(SongRequest request)
{
Song song;
if (request.getUrl() != null)
{
song = new Song(request.getUrl());
} else
public void queueDownload(SongRequest request)
{
// interpret search result
throw new UnsupportedOperationException("Not supported yet.");
}
if (request.getInvocation().getRequestMessage() != null)
{
song.setRequestedBy(request.getInvocation().getRequestMessage().getAuthor().getName());
song.setRequestedIn(request.getInvocation().getRequestMessage().getChannel().getId());
}
song.setNumber(trackNumber);
trackNumber++;
request.setSong(song);
request.getInvocation().respond("Request pending...");
downloader.accept(new Downloader.DownloadTask(request, queueManager));
return song;
}
// public Song queueDownload(final URL url, GuildMessageReceivedEvent event)
// Song song;
// if (request.getUrl() != null)
// {
// Song song = new Song(url);
// song.setRequestedBy(event.getAuthor().getName());
// song.setRequestedIn(event.getChannel().getId());
// song.setNumber(trackNumber);
// trackNumber++;
// downloader.accept(new Downloader.DownloadTask(song, musicHandler));
// return song;
// song = new Song(request.getUrl());
// } else
// {
// // interpret search result
// throw new UnsupportedOperationException("Not supported yet.");
// }
//
// public Song queueDownload(Result res, GuildMessageReceivedEvent event)
// if (request.getInvocation().getRequestMessage() != null)
// {
// Song song = queueDownload(res.getLink(), event);
// song.setArtist(res.getArtist());
// song.setTitle(res.getTitle());
//
// }
// song.setNumber(trackNumber);
// trackNumber++;
// request.addSong(song);
request.getInvocation().respond("Request pending...");
downloader.accept(new Downloader.DownloadTask(request, queueManager));
// return song;
// }
}
public void setStatus(Song nowPlaying)
{
jda.getPresence().setActivity(Activity.listening(nowPlaying.toString()));
@ -414,11 +391,6 @@ public final class Chords extends ListenerAdapter
return queueManager;
}
public int getTrackNumber()
{
return trackNumber;
}
public static Settings getSettings()
{
return settings;
@ -437,13 +409,17 @@ public final class Chords extends ListenerAdapter
@Override
public void accept(SongRequest request, Exception ex)
{
Song song = request.getSong();
String response = "";
if (request.getSongs().size() > 1)
response += "Downloading " + request.getSongs().size() + " tracks:\n";
for (Song song : request.getSongs())
{
// TextChannel channel = jda.getTextChannelById(song.getRequestedIn());
// String bracketNo = "[" + song.getNumber() + "] ";
if (ex == null)
if (song.getLocation() != null && request.getProgress() >= 100)
if (song.getLocation() != null && song.getProgress() >= 100)
{
request.getInvocation().respond("Finished downloading " + song + ", added to queue!");
response += ("Finished downloading " + song + ", added to queue!");
log("DOWN", "Downloaded " + song);
} else
{
@ -457,19 +433,23 @@ public final class Chords extends ListenerAdapter
String bitFmt = (bitrate <= 0 ? "??" : bitrate) + "k";
formatDetails = " (" + bitFmt + ", " + sizeFmt + ")";
}
String progressDetails = "";
if (request.getProgress() >= 0)
{
progressDetails = " [" + String.format("%.1f", request.getProgress()) + "%]";
}
request.getInvocation().respond("Now downloading " + song + formatDetails + progressDetails + " ...");
if (song.getProgress() >= 0)
progressDetails = " [" + String.format("%.1f", song.getProgress()) + "%]";
response += ("Now downloading " + song + formatDetails + progressDetails + " ...");
log("DOWN", "Downloading " + song + "...");
}
else
{
request.getInvocation().respond("Failed to download " + song + "! Reason: " + ex.getMessage());
response += ("Failed to download " + song + "! Reason: " + ex.getMessage());
log("DOWN", "Failed to download " + song + "! Reason: " + ex.getMessage());
}
response += "\n";
}
if (!response.isEmpty())
request.getInvocation().respond(response);
}
}

@ -8,15 +8,11 @@ package moe.nekojimi.chords;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.logging.Level;
@ -39,7 +35,7 @@ public class Downloader implements Consumer<DownloadTask>
{
private static final int DOWNLOAD_TIMEOUT = 300;
private static final int INFO_TIMEOUT = 30;
private static final int INFO_TIMEOUT = 60;
private static final int FORMAT_TIMEOUT = 5;
private static final int BITRATE_TARGET = (int) AudioSendHandler.INPUT_FORMAT.getSampleRate();
@ -47,15 +43,19 @@ public class Downloader implements Consumer<DownloadTask>
public static final Pattern DESTINATION_PATTERN = Pattern.compile("Destination: (.*\\.wav)");
public static final Pattern PROGRESS_PATTERN = Pattern.compile("\\[download\\].*?([\\d\\.]+)%");
private static final Pattern ETA_PATTERN = Pattern.compile("\\[download\\].*?ETA\\s+(\\d{1,2}:\\d{2})");
private static final Pattern DOWNLOAD_ITEM_PATTERN = Pattern.compile("\\[download\\] Downloading item (\\d+) of (\\d+)");
private final List<DownloadTask> downloadQueue = new LinkedList<>();
private final LinkedBlockingDeque<Runnable> workQueue = new LinkedBlockingDeque<>();
private final ThreadPoolExecutor executor = new ThreadPoolExecutor(2, 4, 30, TimeUnit.SECONDS, workQueue);
private final ScheduledExecutorService scheduler = new ScheduledThreadPoolExecutor(1);
// private Consumer<Song> next;
private BiConsumer<SongRequest, Exception> messageHandler;
private File downloadDir = null;
private int trackNumber = 1;
public Downloader()
{
@ -64,21 +64,22 @@ public class Downloader implements Consumer<DownloadTask>
@Override
public void accept(DownloadTask task)
{
// if already downloaded, just skip
Song song = task.request.getSong();
if (song.isDownloaded())
// if all songs of the request are already downloaded, just skip
if (!task.request.getSongs().isEmpty() && task.request.getSongs().stream().allMatch((t) -> t.isDownloaded()))
{
for (Song song : task.request.getSongs())
task.getDestination().accept(song);
return;
}
downloadQueue.add(task);
getInfo(song);
getInfo(task.request);
// TODO: get info should also use the thread pool
executor.submit(() ->
{
try
{
getFormats(song);
// getFormats(song);
download(task);
} catch (Exception ex)
{
@ -87,11 +88,9 @@ public class Downloader implements Consumer<DownloadTask>
});
}
private void chooseFormats(Song song)
private List<Format> sortFormats(Collection<Format> input)
{
List<Format> formats = song.getFormats();
if (formats.isEmpty())
return;
List<Format> formats = new ArrayList<>(input);
formats.sort((Format a, Format b) ->
{
// audio only preferred to video
@ -127,7 +126,7 @@ public class Downloader implements Consumer<DownloadTask>
}
return -comp;
});
song.setFormats(formats);
return formats;
}
private void getFormats(Song song)
@ -205,14 +204,26 @@ public class Downloader implements Consumer<DownloadTask>
}
}
private void getInfo(Song song)
private List<Song> getInfo(SongRequest request)
{
List<Song> ret = new ArrayList<>();
try
{
String cmd = Chords.getSettings().getYtdlCommand() + " --skip-download --print-json " + song.getUrl().toString();
String cmd = Chords.getSettings().getYtdlCommand() + " --skip-download --print-json " + request.getUrl().toString();
Process exec = runCommand(cmd, INFO_TIMEOUT);
InputStream input = exec.getInputStream();
JsonReader reader = Json.createReader(input);
// read each line as JSON, turn each into a song object
Scanner sc = new Scanner(input);
while (sc.hasNextLine())
{
Song song = new Song(request.getUrl());
song.setNumber(trackNumber);
trackNumber++;
request.addSong(song);
String line = sc.nextLine();
JsonReader reader = Json.createReader(new StringReader(line));
JsonObject object = reader.readObject();
if (song.getTitle() == null)
song.setTitle(object.getString("title", null));
@ -231,10 +242,14 @@ public class Downloader implements Consumer<DownloadTask>
}
song.setFormats(formats);
}
ret.add(song);
}
} catch (Exception ex)
{
Logger.getLogger(Downloader.class.getName()).log(Level.SEVERE, null, ex);
}
return ret;
}
private File getDownloadDir() throws IOException
@ -244,15 +259,35 @@ public class Downloader implements Consumer<DownloadTask>
return downloadDir;
}
private Song getSongFromRequest(SongRequest request, int idx)
{
// if there's less songs in the request than expected, fill the array
while (idx >= request.getSongs().size())
{
Song song = new Song(request.getUrl());
song.setNumber(trackNumber);
trackNumber++;
request.addSong(song);
}
return request.getSongs().get(idx);
}
private void download(DownloadTask task)
{
Song song = task.request.getSong();
chooseFormats(song);
Set<Format> uniqueFormats = new HashSet<>();
for (Song song : task.request.getSongs())
{
uniqueFormats.addAll(song.getFormats());
}
List<Format> sortedFormats = sortFormats(uniqueFormats);
String formatCodes = "";
final List<Format> formats = song.getFormats();
for (int i = 0; i < 3 && i < song.getFormats().size(); i++)
final List<Format> formats = sortedFormats;
for (int i = 0; i < 5 && i < sortedFormats.size(); i++)
formatCodes += formats.get(i).getCode() + "/";
int downloadIdx = 0;
try
{
messageHandler.accept(task.request, null);
@ -263,7 +298,7 @@ public class Downloader implements Consumer<DownloadTask>
+ " --no-playlist"
+ " --extractor-args youtube:player_client=android"
+ " -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s "
+ song.getUrl().toString();
+ task.request.getUrl().toString();
Process exec = runCommand(cmd, DOWNLOAD_TIMEOUT);
InputStream in = exec.getInputStream();
@ -274,30 +309,47 @@ public class Downloader implements Consumer<DownloadTask>
String line = sc.nextLine();
System.out.println(line);
Matcher itemMatcher = DOWNLOAD_ITEM_PATTERN.matcher(line);
if (itemMatcher.find())
{
int idx = Integer.parseInt(itemMatcher.group(1)) - 1;
int total = Integer.parseInt(cmd);
downloadIdx = idx;
}
Matcher progMatcher = PROGRESS_PATTERN.matcher(line);
if (progMatcher.find())
{
task.request.setProgress(Double.parseDouble(progMatcher.group(1)));
getSongFromRequest(task.request, downloadIdx).setProgress(Double.parseDouble(progMatcher.group(1)));
messageHandler.accept(task.request, null);
}
Matcher destMatcher = DESTINATION_PATTERN.matcher(line);
if (destMatcher.find())
{
Song song = getSongFromRequest(task.request, downloadIdx);
song.setLocation(new File(destMatcher.group(1)));
// this is currently our criteria for completion; submit the song and move on
if (task.getDestination() != null)
task.getDestination().accept(song);
downloadIdx++;
}
}
// String output = new String(in.readAllBytes(), Charset.defaultCharset());
// String output = new String(in.readAllBytes(), Charset.defaultCharset());
String error = new String(exec.getErrorStream().readAllBytes(), Charset.defaultCharset());
// System.out.println(output);
// System.out.println(output);
if (exec.exitValue() != 0)
throw new RuntimeException("youtube-dl failed with error " + exec.exitValue() + ", output:\n" + error);
task.request.setProgress(100);
// return true;
// task.request.setProgress(100);
// return true;
if (task.getDestination() != null)
task.getDestination().accept(song);
downloadQueue.remove(task);
messageHandler.accept(task.request, null);
} catch (Exception ex)
@ -314,12 +366,20 @@ public class Downloader implements Consumer<DownloadTask>
System.out.println("Running command: " + cmd);
// Process exec = Runtime.getRuntime().exec().split(" "));
Process exec = new ProcessBuilder(cmd.split(" ")).redirectOutput(ProcessBuilder.Redirect.PIPE).redirectError(ProcessBuilder.Redirect.PIPE).start();
boolean done = exec.waitFor(timeoutSecs, TimeUnit.SECONDS);
if (!done)
scheduler.schedule(() ->
{
if (exec.isAlive())
{
exec.destroyForcibly();
throw new RuntimeException("Took too long, giving up.");
}
System.err.println("Process " + cmd + " took too long, killing process.");
}
}, timeoutSecs, TimeUnit.SECONDS);
// boolean done = exec.waitFor(timeoutSecs, TimeUnit.SECONDS);
// if (!done)
// {
// exec.destroyForcibly();
// throw new RuntimeException("Took too long, giving up.");
// }
return exec;
}

@ -7,12 +7,13 @@ package moe.nekojimi.chords;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.sound.sampled.*;
import net.dv8tion.jda.api.audio.AudioSendHandler;
import org.apache.commons.io.input.buffer.CircularByteBuffer;
/**
*
@ -37,7 +38,8 @@ public class MusicHandler implements AudioSendHandler, Closeable
}
private Song currentSong;
private TrackPlayer player;
// private TrackPlayer player;
private final List<TrackPlayer> playingTracks = new ArrayList<>();
private File debugOutFile;
private BufferedOutputStream debugOut;
@ -67,12 +69,11 @@ public class MusicHandler implements AudioSendHandler, Closeable
nextSong(true);
}
// public boolean restartSong()
// {
//// songQueue.addFirst(currentSong);
// currentSong = null;
// return nextSong(true);
// }
public void playOver(Song song)
{
}
private boolean nextSong()
{
@ -84,8 +85,7 @@ public class MusicHandler implements AudioSendHandler, Closeable
if (immediate)
{
System.out.println("Immediate next - clearing buffer");
player = null;
// audioBuffer.clear();
playingTracks.clear();
}
try
@ -108,7 +108,8 @@ public class MusicHandler implements AudioSendHandler, Closeable
System.out.println("Playing song " + currentSong.getLocation().getAbsolutePath());
arrayErr = false;
byteCount = 3840;
player = new TrackPlayer(currentSong);
TrackPlayer player = new TrackPlayer(currentSong);
playingTracks.add(player);
// System.out.println("Queue filled to " + audioBuffer.getCurrentNumberOfBytes());
return true;
} catch (UnsupportedAudioFileException | IOException ex)
@ -122,7 +123,7 @@ public class MusicHandler implements AudioSendHandler, Closeable
public boolean isPlaying()
{
return player != null;
return !playingTracks.isEmpty();
}
public boolean isShouldPlay()
@ -140,7 +141,12 @@ public class MusicHandler implements AudioSendHandler, Closeable
@Override
public boolean canProvide()
{
return player != null && player.has(1);
if (playingTracks.isEmpty())
return false;
for (TrackPlayer player : playingTracks)
if (player.has(1))
return true;
return false;
// If we have something in our buffer we can provide it to the send system
// return audioBuffer.getCurrentNumberOfBytes() > byteCount && shouldPlay;
}
@ -149,28 +155,36 @@ public class MusicHandler implements AudioSendHandler, Closeable
public ByteBuffer provide20MsAudio()
{
ByteBuffer ret = ByteBuffer.allocate(byteCount);
while (ret.position() < byteCount && player != null)
while (ret.position() < byteCount && !playingTracks.isEmpty())
{
boolean outOfInput = true;
List<ByteBuffer> mixes = new ArrayList<>();
List<TrackPlayer> emptyPlayers = new ArrayList<>();
for (TrackPlayer player : playingTracks)
{
// System.out.println("Position: " + ret.position() + " Remaining: " + ret.remaining());
try
{
ByteBuffer read = player.read(ret.remaining());
// System.out.println("SAMPLES from player: " + Util.printSamples(read));
// System.out.println("Wanted: " + byteCount + " Space:" + space + " Available: " + din.available() + " To read: " + bytesToRead + " Read: " + read);
// System.out.println("Read: " + read.remaining());
ret.put(read);
if (ret.limit() + read.position() >= byteCount)
outOfInput = false;
mixes.add(read);
// ret.put(read);
} catch (TrackPlayer.OutOfInputException | IOException ex)
{
// System.out.println("Track ended, starting next.");
// System.out.println("Track player " + player + " stopped giving input: " + ex.getMessage());
emptyPlayers.add(player);
// System.out.println("Track ended, starting next.");
// outOfInput = true;
}
}
playingTracks.removeAll(emptyPlayers);
ret.put(mixBuffers(mixes));
if (outOfInput)
{
boolean foundNext = nextSong();
if (!foundNext)
{
// System.out.println("Out of tracks!");
break;
}
}
}
// System.out.println("Buffer filled, submitting.");
@ -180,56 +194,6 @@ public class MusicHandler implements AudioSendHandler, Closeable
}
// private void fillBuffer(boolean canSkip)
// {
// // use what we have in our buffer to send audio as PCM
// while (audioBuffer.getCurrentNumberOfBytes() < DESIRED_BUFFER_SIZE)
// if (!readData())
// if (!canSkip || !nextSong())
// break;
// }
//
// private boolean readData()
// {
// if (din == null)
// return false;
// try
// {
// // if (din.available() == 0)
// // return false;
// int bytesToRead = DESIRED_BUFFER_SIZE - audioBuffer.getCurrentNumberOfBytes();
// int space = audioBuffer.getSpace();
// if (din.available() > 0 && din.available() < bytesToRead)
// bytesToRead = din.available();
// if (bytesToRead > space)
// bytesToRead = space;
// if (bytesToRead == 0)
// return false;
// byte[] bytes = new byte[bytesToRead];
// // byte[] bytes = din.readNBytes(bytesToRead);
// int read = din.read(bytes);
//// System.out.println("Wanted: " + byteCount + " Space:" + space + " Available: " + din.available() + " To read: " + bytesToRead + " Read: " + read);
// if (read < 0)
// return false;
//// queue.add(bytes);
//
// audioBuffer.add(bytes, 0, read);
// } catch (IOException ex)
// {
// Logger.getLogger(Chords.class.getName()).log(Level.SEVERE, null, ex);
// return false;
// } catch (ArrayIndexOutOfBoundsException ex)
// {
// if (!arrayErr)
// arrayErr = true;
// else
// {
// Logger.getLogger(Chords.class.getName()).log(Level.SEVERE, null, ex);
// return false;
// }
// }
// return true;
// }
public Song getCurrentSong()
{
return currentSong;
@ -246,4 +210,38 @@ public class MusicHandler implements AudioSendHandler, Closeable
{
}
private ByteBuffer mixBuffers(List<ByteBuffer> mixes)
{
// System.out.println("Mixing " + mixes.size() + " buffers");
if (mixes.size() == 1)
return mixes.get(0);
int maxSize = 0;
for (ByteBuffer buf : mixes)
{
if (buf.limit() > maxSize)
maxSize = buf.position();
}
ByteBuffer ret = ByteBuffer.allocate(maxSize);
for (int i = 0; i < ret.limit(); i++)
{
int byteTotal = 0;
int mixCount = 0;
for (ByteBuffer buf : mixes)
{
if (i < buf.limit())
{
byteTotal += buf.get(i);
mixCount++;
}
}
double avg = ((double) byteTotal) / mixCount;
byte byteVal = (byte) Math.round(avg);
ret.put(byteVal);
}
ret.rewind();
return ret;
}
}

@ -19,7 +19,7 @@ import java.util.List;
*
* @author jimj316
*/
public class Song
public class Song implements Comparable<Song>
{
private String title;
private String artist;
@ -28,10 +28,11 @@ public class Song
private int number;
private List<Format> formats = new ArrayList<>();
private String requestedBy;
private String requestedIn;
private boolean kept = false;
private double progress = -1;
private double eta = -1;
public Song(URL url)
{
this.url = url;
@ -50,8 +51,8 @@ public class Song
.add("location", location.getAbsolutePath())
.add("num", Integer.toString(number))
.add("formats", build.build())
.add("requestedBy", requestedBy)
.add("requestedIn", requestedIn)
// .add("requestedBy", requestedBy)
// .add("requestedIn", requestedIn)
.add("kept", Boolean.toString(kept))
.build();
}
@ -64,8 +65,8 @@ public class Song
song.setLocation(new File(map.string("location")));
song.setNumber(map.integer("num"));
song.setKept(Boolean.parseBoolean(map.string("kept")));
song.setRequestedBy(map.string("requestedBy"));
song.setRequestedIn(map.string("requestedIn"));
// song.setRequestedBy(map.string("requestedBy"));
// song.setRequestedIn(map.string("requestedIn"));
List<Format> formats = new ArrayList<>();
YamlSequence formatSeq = map.yamlSequence("formats");
@ -133,26 +134,6 @@ public class Song
location = null;
}
public String getRequestedBy()
{
return requestedBy;
}
public void setRequestedBy(String requestedBy)
{
this.requestedBy = requestedBy;
}
public String getRequestedIn()
{
return requestedIn;
}
public void setRequestedIn(String requestedIn)
{
this.requestedIn = requestedIn;
}
public boolean isKept()
{
return kept;
@ -197,4 +178,30 @@ public class Song
return formats.get(0);
}
public double getProgress()
{
return progress;
}
public void setProgress(double progress)
{
this.progress = progress;
}
public double getEta()
{
return eta;
}
public void setEta(double eta)
{
this.eta = eta;
}
@Override
public int compareTo(Song o)
{
return Integer.compare(number, o.number);
}
}

@ -17,6 +17,7 @@
package moe.nekojimi.chords;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import moe.nekojimi.chords.commands.Invocation;
import moe.nekojimi.musicsearcher.Result;
@ -36,10 +37,10 @@ public class SongRequest
private Result result;
private Song song;
private final List<Song> songs = new ArrayList<>();
private double progress = -1;
private double eta = -1;
private String requestedBy;
private String requestedIn;
public List<Result> getSearchResults()
@ -70,6 +71,8 @@ public class SongRequest
public void setInvocation(Invocation invocation)
{
this.invocation = invocation;
requestedBy = invocation.getRequestMessage().getAuthor().getName();
requestedIn = invocation.getRequestMessage().getChannel().getId();
}
// public Message getRequestMessage()
@ -112,34 +115,29 @@ public class SongRequest
this.url = url;
}
public Song getSong()
public List<Song> getSongs()
{
return song;
return songs;
}
public void setSong(Song song)
public void addSong(Song song)
{
this.song = song;
songs.add(song);
}
public double getProgress()
public void clearSongs()
{
return progress;
songs.clear();
}
public void setProgress(double progress)
public String getRequestedBy()
{
this.progress = progress;
return requestedBy;
}
public double getEta()
public String getRequestedIn()
{
return eta;
}
public void setEta(double eta)
{
this.eta = eta;
return requestedIn;
}
}

Loading…
Cancel
Save