WIP: initial work on streaming instead of downloading.

src/main/java/moe/nekojimi/chords/Downloader.java

@@ -5,10 +5,8 @@
  */
 package moe.nekojimi.chords;
 
-import java.io.File;
+import com.beust.jcommander.Strings;
-import java.io.IOException;
+import java.io.*;
-import java.io.InputStream;
-import java.io.StringReader;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.util.*;
@@ -40,6 +38,7 @@ public class Downloader extends QueueThing<TrackRequest, Track>
     private static final int BITRATE_TARGET = (int) AudioSendHandler.INPUT_FORMAT.getSampleRate();
     private static final Pattern FORMAT_PATTERN = Pattern.compile("^([\\w]+)\\s+([\\w]+)\\s+(\\w+ ?\\w*)\\s+(.*)$");
     public static final Pattern DESTINATION_PATTERN = Pattern.compile("Destination: (.*\\.wav)");
+    public static final Pattern STREAM_PATTERN = Pattern.compile("Destination: -");
     public static final Pattern PROGRESS_PATTERN = Pattern.compile("\\[download\\].*?([\\d\\.]+)%");
     private static final Pattern INFO_JSON_PATTERN = Pattern.compile("Writing video metadata as JSON to: (.*\\.info\\.json)");
     private static final Pattern ETA_PATTERN = Pattern.compile("\\[download\\].*?ETA\\s+(\\d{1,2}:\\d{2})");
@@ -99,7 +98,8 @@ public class Downloader extends QueueThing<TrackRequest, Track>
             {
 
                 getInfo(request);
-                download(promise);
+                boolean streamOutput = request.getTracks().size() == 1;
+                download(promise, streamOutput);
             } catch (Exception ex)
             {
                 ex.printStackTrace();
@@ -157,8 +157,8 @@ public class Downloader extends QueueThing<TrackRequest, Track>
             return;
         try
         {
-            String cmd = Chords.getSettings().getYtdlCommand() + " --skip-download -F " + track.getUrl().toString();
+//            String cmd =  + " --skip-download -F " + track.getUrl().toString();
-            Process exec = runCommand(cmd, FORMAT_TIMEOUT);
+            Process exec = runCommand(List.of(Chords.getSettings().getYtdlCommand(), "--skip-download", "-F", track.getUrl().toString()), FORMAT_TIMEOUT);
             InputStream input = exec.getInputStream();
             String output = new String(input.readAllBytes(), Charset.defaultCharset());
 
@@ -231,8 +231,8 @@ public class Downloader extends QueueThing<TrackRequest, Track>
         List<Track> ret = new ArrayList<>();
         try
         {
-            String cmd = Chords.getSettings().getYtdlCommand() + " --skip-download --print-json " + request.getUrl().toString();
+//            String cmd = Chords.getSettings().getYtdlCommand() + " --skip-download --print-json " + request.getUrl().toString();
-            Process exec = runCommand(cmd, INFO_TIMEOUT);
+            Process exec = runCommand(List.of(Chords.getSettings().getYtdlCommand(), "--skip-download", "--print-json", request.getUrl().toString()), INFO_TIMEOUT);
             InputStream input = exec.getInputStream();
 
             // read each line as JSON, turn each into a track object
@@ -330,7 +330,7 @@ public class Downloader extends QueueThing<TrackRequest, Track>
         return request.getTracks().get(idx);
     }
 
-    private void download(Promise<TrackRequest, Track> promise) throws InterruptedException, ExecutionException
+    private void download(Promise<TrackRequest, Track> promise, boolean streamOutput) throws InterruptedException, ExecutionException
     {
         TrackRequest request = promise.getInput();
         Set<Format> uniqueFormats = new HashSet<>();
@@ -351,69 +351,109 @@ public class Downloader extends QueueThing<TrackRequest, Track>
         try
         {
             messageHandler.accept(request, null);
-            String cmd = Chords.getSettings().getYtdlCommand()
+            List<String> cmd = new ArrayList<>();
-                    + " -x"
+            cmd.add(Chords.getSettings().getYtdlCommand());
-                    + " -f " + formatCodes + "worstaudio/bestaudio/worst/best"
+            cmd.add("-x");
-                    + " --audio-format=wav"
+            cmd.add("-f=" + formatCodes + "worstaudio/bestaudio/worst/best");
-                    + " --no-playlist"
+            cmd.add("--audio-format=wav");
-                    //                    + " --extractor-args youtube:player_client=android"
+            cmd.add("--no-playlist");
-                    + " -N 8"
+//            cmd.add(" --extractor-args youtube:player_client=android";
-                    + " -o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s "
+            cmd.add("-N8");
-                    + request.getUrl().toString();
+            if (streamOutput)
-
-            Process exec = runCommand(cmd, DOWNLOAD_TIMEOUT);
-            InputStream in = exec.getInputStream();
-
-            Scanner sc = new Scanner(in);
-            while (sc.hasNextLine())
             {
-                String line = sc.nextLine();
+                cmd.add("--downloader=ffmpeg"); // download using FFMpeg
-                System.out.println(line);
+                cmd.add("--downloader-args=ffmpeg:-f wav -c:a pcm_s16le"); // tell FFMpeg to convert to wav
-
+                cmd.add("-o"); // output to stdout
-                Matcher itemMatcher = DOWNLOAD_ITEM_PATTERN.matcher(line);
+                cmd.add("-");
-                if (itemMatcher.find())
-                {
-                    int idx = Integer.parseInt(itemMatcher.group(1)) - 1;
-//                    int total = Integer.parseInt(itemMatcher.group(2));
-
-                    downloadIdx = idx;
-                }
-
-                Matcher progMatcher = PROGRESS_PATTERN.matcher(line);
-                if (progMatcher.find())
-                {
-                    getTrackFromRequest(request, downloadIdx).setProgress(Double.parseDouble(progMatcher.group(1)));
-                    messageHandler.accept(request, null);
-                }
-
-                Matcher destMatcher = DESTINATION_PATTERN.matcher(line);
-                if (destMatcher.find())
-                {
-                    Track track = getTrackFromRequest(request, downloadIdx);
-
-                    track.setLocation(new File(destMatcher.group(1)));
-
-                    // this is currently our criteria for completion; submit the track and move on
-                    promise.complete(track);
-
-                    track.setProgress(100.0);
-
-                    messageHandler.accept(request, null);
-
-                    downloadIdx++;
-                }
-            }
-
-            boolean exited = exec.waitFor(DOWNLOAD_TIMEOUT, TimeUnit.SECONDS);
-            if (exited)
-            {
-                String error = new String(exec.getErrorStream().readAllBytes(), Charset.defaultCharset());
-
-                if (exec.exitValue() != 0)
-                    throw new RuntimeException("youtube-dl failed with error " + exec.exitValue() + ", output:\n" + error);
             } else
             {
-                throw new RuntimeException("youtube-dl failed to exit.");
+                cmd.add("-o " + getDownloadDir().getAbsolutePath() + "/%(title)s.%(ext)s");
+            }
+            cmd.add(request.getUrl().toString());
+
+            Process exec = runCommand(cmd, streamOutput ? 0 : DOWNLOAD_TIMEOUT);
+
+            if (streamOutput)
+            {
+                Scanner sc = new Scanner(exec.getErrorStream());
+                while (sc.hasNextLine())
+                {
+                    String line = sc.nextLine();
+                    System.out.println(line);
+                    Matcher streamMatcher = STREAM_PATTERN.matcher(line);
+                    if (streamMatcher.find())
+                    {
+                        break;
+                    }
+                }
+
+                if (!exec.isAlive() && exec.exitValue() != 0)
+                    throw new RuntimeException("yt-dlp failed with error code " + exec.exitValue());
+
+                Track track = getTrackFromRequest(request, 0);
+                BufferedInputStream inBuf = new BufferedInputStream(exec.getInputStream());
+                inBuf.mark(128);
+                final byte[] headerBytes = inBuf.readNBytes(80);
+                String header = new String(headerBytes);
+                System.out.println("streaming data header: " + header);
+                if (!header.startsWith("RIFF"))
+                    throw new RuntimeException("Streaming data has bad header!");
+                inBuf.reset();
+                track.setInputStream(inBuf);
+                promise.complete(track);
+                track.setProgress(100.0);
+                messageHandler.accept(request, null);
+            } else
+            {
+                Scanner sc = new Scanner(exec.getInputStream());
+                while (sc.hasNextLine())
+                {
+                    String line = sc.nextLine();
+                    System.out.println(line);
+
+                    Matcher itemMatcher = DOWNLOAD_ITEM_PATTERN.matcher(line);
+                    if (itemMatcher.find())
+                    {
+                        int idx = Integer.parseInt(itemMatcher.group(1)) - 1;
+                        downloadIdx = idx;
+                    }
+
+                    Matcher progMatcher = PROGRESS_PATTERN.matcher(line);
+                    if (progMatcher.find())
+                    {
+                        getTrackFromRequest(request, downloadIdx).setProgress(Double.parseDouble(progMatcher.group(1)));
+                        messageHandler.accept(request, null);
+                    }
+
+                    Matcher destMatcher = DESTINATION_PATTERN.matcher(line);
+                    if (destMatcher.find())
+                    {
+                        Track track = getTrackFromRequest(request, downloadIdx);
+
+                        track.setLocation(new File(destMatcher.group(1)));
+
+                        // this is currently our criteria for completion; submit the track and move on
+                        promise.complete(track);
+
+                        track.setProgress(100.0);
+
+                        messageHandler.accept(request, null);
+
+                        downloadIdx++;
+                    }
+                }
+
+                boolean exited = exec.waitFor(DOWNLOAD_TIMEOUT, TimeUnit.SECONDS);
+                if (exited)
+                {
+                    String error = new String(exec.getErrorStream().readAllBytes(), Charset.defaultCharset());
+
+                    if (exec.exitValue() != 0)
+                        throw new RuntimeException("youtube-dl failed with error " + exec.exitValue() + ", output:\n" + error);
+                } else
+                {
+                    throw new RuntimeException("youtube-dl failed to exit.");
+                }
             }
 
             messageHandler.accept(request, null);
@@ -426,25 +466,21 @@ public class Downloader extends QueueThing<TrackRequest, Track>
         }
     }
 
-    private Process runCommand(String cmd, int timeoutSecs) throws RuntimeException, IOException, InterruptedException
+    private Process runCommand(List<String> cmd, int timeoutSecs) throws RuntimeException, IOException, InterruptedException
     {
         System.out.println("Running command: " + cmd);
-        //		Process exec = Runtime.getRuntime().exec().split(" "));
+        Process exec = new ProcessBuilder(cmd).redirectOutput(ProcessBuilder.Redirect.PIPE).redirectError(ProcessBuilder.Redirect.PIPE).start();
-        Process exec = new ProcessBuilder(cmd.split(" ")).redirectOutput(ProcessBuilder.Redirect.PIPE).redirectError(ProcessBuilder.Redirect.PIPE).start();
+        if (timeoutSecs > 0)
-        scheduler.schedule(() ->
         {
-            if (exec.isAlive())
+            scheduler.schedule(() ->
             {
-                exec.destroyForcibly();
+                if (exec.isAlive())
-                System.err.println("Process " + cmd + " took too long, killing process.");
+                {
-            }
+                    exec.destroyForcibly();
-        }, timeoutSecs, TimeUnit.SECONDS);
+                    System.err.println("Process " + cmd + " took too long, killing process.");
-//        boolean done = exec.waitFor(timeoutSecs, TimeUnit.SECONDS);
+                }
-//        if (!done)
+            }, timeoutSecs, TimeUnit.SECONDS);
-//        {
+        }
-//            exec.destroyForcibly();
-//            throw new RuntimeException("Took too long, giving up.");
-//        }
         return exec;
     }
 

src/main/java/moe/nekojimi/chords/TrackPlayer.java

@@ -5,6 +5,7 @@
  */
 package moe.nekojimi.chords;
 
+import java.io.BufferedInputStream;
 import java.io.Closeable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -39,30 +40,32 @@ public class TrackPlayer implements Closeable
     {
         AudioInputStream in = null;
         AudioFormat decodedFormat = null;
-        for (int retry = 0; retry < RETRY_COUNT; retry++)
+        int retry = 0;
+        while (in == null)
         {
             try
             {
-                in = AudioSystem.getAudioInputStream(track.getInputStream());
+                in = AudioSystem.getAudioInputStream(new BufferedInputStream(track.getInputStream()));
                 decodedFormat = AudioSendHandler.INPUT_FORMAT;
 
                 break; // it worked!
             } catch (Exception ex)
             {
-                try
+                retry++;
+                if (retry < RETRY_COUNT)
                 {
-                    Thread.sleep(((long) Math.pow(2, retry)) * RETRY_DELAY);
+                    System.err.println("Open file " + track.getLocation() + " failed because " + ex.getMessage() + "  retry " + retry + "...");
-                } catch (InterruptedException ex1)
+                    try
-                {
-                    if (retry < RETRY_COUNT)
                     {
-                        System.err.println("Open file " + track.getLocation() + " failed, retry " + retry + "...");
+                        Thread.sleep(((long) Math.pow(2, retry)) * RETRY_DELAY);
-                        continue;
+                    } catch (InterruptedException ex1)
-                    } else
                     {
-                        throw ex;
                     }
+                } else
+                {
+                    throw ex;
                 }
+
             }
         }
         input = AudioSystem.getAudioInputStream(decodedFormat, in);
@@ -88,7 +91,7 @@ public class TrackPlayer implements Closeable
 //            throw new OutOfInputException();
 
         int toRead = Math.min(length, audioBuffer.getCurrentNumberOfBytes());
-//        System.out.println("To read: " + toRead + " from " + audioBuffer.getCurrentNumberOfBytes());
+        System.out.println("To read: " + toRead + " from " + audioBuffer.getCurrentNumberOfBytes());
         if (toRead <= 0)
             throw new OutOfInputException();
 
@@ -166,6 +169,7 @@ public class TrackPlayer implements Closeable
     }
 
 
+
     public static class OutOfInputException extends RuntimeException
     {