Nekojimi
/
MusicSearcher
generated from Nekojimi/JavaMavenTemplate
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: Nekojimi:4df2f55922c09cd6b11cc5d63abfb74ccba3be5a
Branches Tags
Nekojimi:master
..
compare: Nekojimi:0bdf5d614d6e0d2ca2a58b26ed240ee71cd5798c
Branches Tags
Nekojimi:master

No commits in common. "4df2f55922c09cd6b11cc5d63abfb74ccba3be5a" and "0bdf5d614d6e0d2ca2a58b26ed240ee71cd5798c" have entirely different histories.
4df2f55922 ... 0bdf5d614d

6 changed files with 84 additions and 212 deletions
Show Stats Expand all files Collapse all files

5
pom.xml
View File

@ -73,11 +73,6 @@
<version>1.1.4</version> <version>1.1.4</version>
<classifier>module</classifier> <classifier>module</classifier>
</dependency> </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
</dependencies> </dependencies>
<name>MusicSearcher</name> <name>MusicSearcher</name>
</project> </project>

61
src/main/java/moe/nekojimi/musicsearcher/Main.java
View File

@ -19,48 +19,55 @@ import java.util.Map;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import moe.nekojimi.musicsearcher.providers.MetaSearcher;
import moe.nekojimi.musicsearcher.providers.Searcher; import moe.nekojimi.musicsearcher.providers.Searcher;
/** /**
* *
* @author jim * @author jim
*/ */
public class Main public class Main
{ {
private static final Map<String, Searcher> searchers = new HashMap<>(); private static final Map<String, Searcher> searchers = new HashMap<>();
/** /**
* @param args the command line arguments * @param args the command line arguments
*/ */
public static void main(String[] args) throws IOException public static void main(String[] args) throws IOException
{ {
// System.out.println("Hello world!"); // System.out.println("Hello world!");
YamlInput input = Yaml.createYamlInput(new File("searchproviders.yml"));
YamlSequence seq = input.readYamlSequence();
for (int i = 0; i < seq.size(); i++)
{
try
{
YamlMapping map = seq.yamlMapping(i);
String type = map.string("type");
Class<? extends Searcher> clazz = (Class<? extends Searcher>) Class.forName("moe.nekojimi.musicsearcher.providers." + type);
Constructor<? extends Searcher> constructor = clazz.getConstructor(YamlMapping.class);
Searcher searcher = constructor.newInstance(map);
searchers.put(searcher.getName(), searcher);
} catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
}
}
System.out.println(searchers); System.out.println(searchers);
String query = "eminem lose yourself"; String query = "everybodys circulation";
for (Searcher searcher: searchers.values())
MetaSearcher metaSearch = MetaSearcher.loadYAML(new File("searchproviders.yml"));
try
{ {
List<Result> results = metaSearch.searchAndWait(Query.fullText(query)); System.out.println("Searching " + searcher.getName() + " for " + query);
// for (Searcher searcher: searchers.values()) try
// { {
// System.out.println("Searching " + searcher.getName() + " for " + query); List<Result> results = searcher.searchAndWait(Query.fullText(query));
// try for (Result result: results)
// { {
// List<Result> results = searcher.searchAndWait(Query.fullText(query)); System.out.println("\t" + result);
for (Result result : results) }
System.out.println("\t" + result); } catch (InterruptedException | ExecutionException ex) {
// } catch (InterruptedException | ExecutionException ex) { Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
// Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex); }
// }
// }
} catch (ExecutionException | InterruptedException ex)
{
Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
} }
} }
} }

52
src/main/java/moe/nekojimi/musicsearcher/Result.java
View File

@ -11,18 +11,13 @@ import java.net.URL;
* *
* @author jim * @author jim
*/ */
public class Result implements Comparable<Result> public class Result
{ {
private URL link; private URL link;
private String artist; private String artist;
private String album; private String album;
private String title; private String title;
private String sourceName;
private String sourceAbbr;
private double score;
public URL getLink() { public URL getLink() {
return link; return link;
} }
@ -55,39 +50,12 @@ public class Result implements Comparable<Result>
this.title = title; this.title = title;
} }
public String getSourceName()
{
return sourceName;
}
public void setSource(String name, String abbr)
{
sourceName = name;
sourceAbbr = abbr;
}
public String getSourceAbbr()
{
return sourceAbbr;
}
public double getScore()
{
return score;
}
public void setScore(double score)
{
this.score = score;
}
@Override @Override
public String toString() public String toString() {
{ return "Result{" + "link=" + link + ", artist=" + artist + ", album=" + album + ", title=" + title + '}';
return "Result{" + "artist=" + artist + ", album=" + album + ", title=" + title + ", sourceAbbr=" + sourceAbbr + ", score=" + score + '}';
} }
public void setAlbumArtist(String field) public void setAlbumArtist(String field)
{ {
// System.out.println("Parsing album-artist: " + field); // System.out.println("Parsing album-artist: " + field);
String fieldLower = field.toLowerCase(); String fieldLower = field.toLowerCase();
@ -110,17 +78,9 @@ public class Result implements Comparable<Result>
else else
album += word + " "; album += word + " ";
} }
} }
artist = artist.trim();
album = album.trim();
} }
} }
@Override
public int compareTo(Result t)
{
return (int) Math.signum(score - t.score);
}
} }

77
src/main/java/moe/nekojimi/musicsearcher/providers/MetaSearcher.java
View File

@ -5,26 +5,11 @@
*/ */
package moe.nekojimi.musicsearcher.providers; package moe.nekojimi.musicsearcher.providers;
import com.amihaiemil.eoyaml.Yaml;
import com.amihaiemil.eoyaml.YamlInput;
import com.amihaiemil.eoyaml.YamlMapping;
import com.amihaiemil.eoyaml.YamlSequence;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.Collection;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import moe.nekojimi.musicsearcher.Main;
import moe.nekojimi.musicsearcher.Query; import moe.nekojimi.musicsearcher.Query;
import moe.nekojimi.musicsearcher.Result; import moe.nekojimi.musicsearcher.Result;
@ -34,44 +19,18 @@ import moe.nekojimi.musicsearcher.Result;
*/ */
public class MetaSearcher extends Searcher public class MetaSearcher extends Searcher
{ {
private final Set<Searcher> searchers = new HashSet<>(); private final Set<Searcher> searchers = new HashSet<>();
private int minSearchTime = 10000; // ms private int minSearchTime = 10000; // ms
private int maxSearchTime = 30000; // ms private int maxSearchTime = 30000; // ms
public MetaSearcher(Collection<Searcher> searchers) public MetaSearcher()
{ {
super("meta"); super("meta");
this.searchers.addAll(searchers);
}
public static MetaSearcher loadYAML(File file)
{
try
{
List<Searcher> searchers = new ArrayList<>();
YamlInput input = Yaml.createYamlInput(file);
YamlSequence seq = input.readYamlSequence();
for (int i = 0; i < seq.size(); i++)
{
YamlMapping map = seq.yamlMapping(i);
String type = map.string("type");
Class<? extends Searcher> clazz = (Class<? extends Searcher>) Class.forName("moe.nekojimi.musicsearcher.providers." + type);
Constructor<? extends Searcher> constructor = clazz.getConstructor(YamlMapping.class);
Searcher searcher = constructor.newInstance(map);
searchers.add(searcher);
}
MetaSearcher metaSearch = new MetaSearcher(searchers);
return metaSearch;
} catch (IOException | ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex)
{
throw new IllegalArgumentException("Could not load MetaSearcher config from YAML file " + file.getAbsolutePath(), ex);
}
} }
@Override @Override
protected List<Result> doSearch(Query query) protected List<Result> doSearch(Query query)
{ {
List<Result> results = new ArrayList<>(); List<Result> results = new ArrayList<>();
List<CompletableFuture<List<Result>>> searches = new ArrayList<>(); List<CompletableFuture<List<Result>>> searches = new ArrayList<>();
@ -79,28 +38,18 @@ public class MetaSearcher extends Searcher
{ {
CompletableFuture<List<Result>> search = searcher.search(query); CompletableFuture<List<Result>> search = searcher.search(query);
searches.add(search); searches.add(search);
// search.whenComplete((t, u) -> search.whenComplete((t, u) ->
// { {
// if (u == null) if (u == null)
// { {
// results.addAll(t); results.addAll(t);
//// searches.remove(search); // searches.remove(search);
// } }
// }); });
} }
// CompletableFuture.allOf((CompletableFuture<?>[]) searches.toArray()); CompletableFuture.allOf((CompletableFuture<?>[]) searches.toArray());
for (CompletableFuture<List<Result>> search : searches)
try
{
List<Result> result = search.get(30, TimeUnit.SECONDS);
results.addAll(result);
} catch (TimeoutException | InterruptedException | ExecutionException ex)
{
Logger.getLogger(MetaSearcher.class.getName()).log(Level.SEVERE, null, ex);
}
return results; return results;
} }
} }

64
src/main/java/moe/nekojimi/musicsearcher/providers/Searcher.java
View File

@ -6,10 +6,7 @@
package moe.nekojimi.musicsearcher.providers; package moe.nekojimi.musicsearcher.providers;
import com.amihaiemil.eoyaml.YamlMapping; import com.amihaiemil.eoyaml.YamlMapping;
import java.util.Collections;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinPool;
@ -18,97 +15,62 @@ import java.util.concurrent.TimeoutException;
import moe.nekojimi.musicsearcher.Query; import moe.nekojimi.musicsearcher.Query;
import moe.nekojimi.musicsearcher.QueryFieldUnsupportedException; import moe.nekojimi.musicsearcher.QueryFieldUnsupportedException;
import moe.nekojimi.musicsearcher.Result; import moe.nekojimi.musicsearcher.Result;
import org.apache.commons.lang3.StringUtils;
/** /**
* *
* @author jim * @author jim
*/ */
public abstract class Searcher public abstract class Searcher
{ {
protected final String name; final String name;
protected String abbr;
private final ForkJoinPool executor; private final ForkJoinPool executor;
public Searcher(String name) public Searcher(String name)
{ {
this.name = name; this.name = name;
this.abbr = name.substring(0, 2);
this.executor = new ForkJoinPool(); this.executor = new ForkJoinPool();
} }
public Searcher(YamlMapping yaml) public Searcher(YamlMapping yaml)
{ {
this(yaml.string("name")); this(yaml.string("name"));
abbr = yaml.string("abbr");
assert yaml.string("type").equals(this.getClass().getSimpleName()); assert yaml.string("type").equals(this.getClass().getSimpleName());
} }
public String getName() public String getName()
{ {
return name; return name;
} }
public List<Result> searchAndWait(Query query) throws InterruptedException, ExecutionException public List<Result> searchAndWait(Query query) throws InterruptedException, ExecutionException
{ {
try try
{ {
return searchAndWait(query, Long.MAX_VALUE, TimeUnit.DAYS); return searchAndWait(query, Long.MAX_VALUE, TimeUnit.DAYS);
} catch (TimeoutException ex) } catch (TimeoutException ex)
{ {
throw new RuntimeException("This should never happen!",ex); throw new RuntimeException("This should never happen!",ex);
} }
} }
public List<Result> searchAndWait(Query query, long limit, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException public List<Result> searchAndWait(Query query, long limit, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
{ {
return search(query).get(limit, unit); return search(query).get(limit, unit);
} }
public CompletableFuture<List<Result>> search(Query query) public CompletableFuture<List<Result>> search(Query query)
{ {
CompletableFuture<List<Result>> future = new CompletableFuture<>(); CompletableFuture<List<Result>> future = new CompletableFuture<>();
future.completeAsync(() -> searchWrapper(query), executor); future.completeAsync(()->doSearch(query), executor);
return future; return future;
} }
protected List<Result> searchWrapper(Query query)
{
List<Result> results = doSearch(query);
for (Result result : results)
{
double artistScore = 1;
double titleScore = 1;
double albumScore = 1;
double fullTextScore = 0;
Set<String> toCompare = new HashSet<>();
toCompare.add(result.getAlbum());
toCompare.add(result.getArtist());
toCompare.add(result.getTitle());
for (String comparison : toCompare)
{
if (comparison == null)
continue;
double score = StringUtils.getJaroWinklerDistance(query.getTextSearch(), comparison);
if (score > fullTextScore)
fullTextScore = score;
}
result.setScore(albumScore * artistScore * titleScore * fullTextScore);
}
Collections.sort(results, Collections.reverseOrder());
return results;
}
protected abstract List<Result> doSearch(Query query) throws QueryFieldUnsupportedException; protected abstract List<Result> doSearch(Query query) throws QueryFieldUnsupportedException;
@Override @Override
public String toString() { public String toString() {
return this.getClass().getSimpleName() + "{" + "name=" + name + '}'; return this.getClass().getSimpleName() + "{" + "name=" + name + '}';
} }
} }

37
src/main/java/moe/nekojimi/musicsearcher/providers/WebScraperSearcher.java
View File

@ -35,15 +35,15 @@ public class WebScraperSearcher extends Searcher
{ {
protected String searchUrl; protected String searchUrl;
protected URL rootURL; protected URL rootURL;
protected String resultSelector; protected String resultSelector;
protected String resultArtistSelector; protected String resultArtistSelector;
protected String resultTitleSelector; protected String resultTitleSelector;
protected String resultLinkSelector; protected String resultLinkSelector;
protected String resultAlbumArtistSelector; protected String resultAlbumArtistSelector;
protected Map<String,String> searchFields = new HashMap<>(); protected Map<String,String> searchFields = new HashMap<>();
protected Parser<?,?> parser; protected Parser<?,?> parser;
public WebScraperSearcher(String name) public WebScraperSearcher(String name)
@ -51,7 +51,7 @@ public class WebScraperSearcher extends Searcher
super(name); super(name);
} }
public WebScraperSearcher(YamlMapping yaml) throws MalformedURLException public WebScraperSearcher(YamlMapping yaml) throws MalformedURLException
{ {
super(yaml); super(yaml);
searchUrl = yaml.string("search_url"); searchUrl = yaml.string("search_url");
@ -59,7 +59,7 @@ public class WebScraperSearcher extends Searcher
// rootURL = fillURL(Query.fullText("")); // rootURL = fillURL(Query.fullText(""));
// rootURL = new URL(rootURL.getProtocol(), rootURL.getHost(), ""); // rootURL = new URL(rootURL.getProtocol(), rootURL.getHost(), "");
resultSelector = yaml.string("result_selector"); resultSelector = yaml.string("result_selector");
YamlMapping fields = yaml.yamlMapping("result_fields"); YamlMapping fields = yaml.yamlMapping("result_fields");
if (fields != null) if (fields != null)
{ {
@ -68,11 +68,11 @@ public class WebScraperSearcher extends Searcher
resultLinkSelector = fields.string("link"); resultLinkSelector = fields.string("link");
resultAlbumArtistSelector = fields.string("album_artist"); resultAlbumArtistSelector = fields.string("album_artist");
} }
YamlMapping searchFieldMap = yaml.yamlMapping("search_fields"); YamlMapping searchFieldMap = yaml.yamlMapping("search_fields");
for (YamlNode key: searchFieldMap.keys()) for (YamlNode key: searchFieldMap.keys())
searchFields.put(key.asScalar().value(), searchFieldMap.string(key)); searchFields.put(key.asScalar().value(), searchFieldMap.string(key));
String formatName = yaml.string("format"); String formatName = yaml.string("format");
switch(formatName) switch(formatName)
{ {
@ -86,21 +86,21 @@ public class WebScraperSearcher extends Searcher
} }
@Override @Override
protected List<Result> doSearch(Query query) protected List<Result> doSearch(Query query)
{ {
try try
{ {
URL url = fillURL(query); URL url = fillURL(query);
InputStream input = url.openStream(); InputStream input = url.openStream();
return processResults(parser, input); return processResults(parser, input);
} catch (IOException ex) } catch (IOException ex)
{ {
Logger.getLogger(WebScraperSearcher.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(WebScraperSearcher.class.getName()).log(Level.SEVERE, null, ex);
return List.of(); return List.of();
} }
} }
protected URL fillURL(Query query) throws MalformedURLException protected URL fillURL(Query query) throws MalformedURLException
{ {
// URL url = new URL(searchUrl.replaceAll("\\$QUERY", URLEncoder.encode(query.getTextSearch(), Charset.forName("utf-8")))); // URL url = new URL(searchUrl.replaceAll("\\$QUERY", URLEncoder.encode(query.getTextSearch(), Charset.forName("utf-8"))));
try try
@ -122,12 +122,12 @@ public class WebScraperSearcher extends Searcher
throw new MalformedURLException(); throw new MalformedURLException();
} }
} }
protected String transformSearchString(String search) protected String transformSearchString(String search)
{ {
return search; return search;
} }
protected <E> List<Result> processResults(Parser<?,E> parser, InputStream input) protected <E> List<Result> processResults(Parser<?,E> parser, InputStream input)
{ {
Collection<E> resultEles = parser.getResults(input, resultSelector); Collection<E> resultEles = parser.getResults(input, resultSelector);
@ -142,21 +142,20 @@ public class WebScraperSearcher extends Searcher
try try
{ {
Result res = new Result(); Result res = new Result();
res.setSource(name, abbr);
// Artist // Artist
if (resultArtistSelector != null) if (resultArtistSelector != null)
res.setArtist(parser.getField(ele, resultArtistSelector).trim()); res.setArtist(parser.getField(ele, resultArtistSelector));
// Title // Title
if (resultTitleSelector != null) if (resultTitleSelector != null)
res.setTitle(parser.getField(ele, resultTitleSelector).trim()); res.setTitle(parser.getField(ele, resultTitleSelector));
// Link // Link
if (resultLinkSelector != null) if (resultLinkSelector != null)
res.setLink(parser.getURLField(ele, rootURL, resultLinkSelector)); res.setLink(parser.getURLField(ele, rootURL, resultLinkSelector));
// Artist + Album // Artist + Album
if (resultAlbumArtistSelector != null) if (resultAlbumArtistSelector != null)
res.setAlbumArtist(parser.getField(ele, resultAlbumArtistSelector).trim()); res.setAlbumArtist(parser.getField(ele, resultAlbumArtistSelector));
// Artist + Title // Artist + Title
return res; return res;