Nekojimi
/
MusicSearcher
generated from Nekojimi/JavaMavenTemplate
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: Nekojimi:4df2f55922c09cd6b11cc5d63abfb74ccba3be5a
Branches Tags
Nekojimi:master
..
compare: Nekojimi:0bdf5d614d6e0d2ca2a58b26ed240ee71cd5798c
Branches Tags
Nekojimi:master

No commits in common. "4df2f55922c09cd6b11cc5d63abfb74ccba3be5a" and "0bdf5d614d6e0d2ca2a58b26ed240ee71cd5798c" have entirely different histories.
4df2f55922 ... 0bdf5d614d

6 changed files with 84 additions and 212 deletions
Show Stats Expand all files Collapse all files

5
pom.xml
View File

@ -73,11 +73,6 @@
<version>1.1.4</version>
<classifier>module</classifier>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
</dependencies>
<name>MusicSearcher</name>
</project>

61
src/main/java/moe/nekojimi/musicsearcher/Main.java
View File

@ -19,48 +19,55 @@ import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.logging.Logger;
import moe.nekojimi.musicsearcher.providers.MetaSearcher;
import moe.nekojimi.musicsearcher.providers.Searcher;
/**
*
* @author jim
*/
public class Main
public class Main
{
private static final Map<String, Searcher> searchers = new HashMap<>();
/**
* @param args the command line arguments
*/
public static void main(String[] args) throws IOException
public static void main(String[] args) throws IOException
{
// System.out.println("Hello world!");
YamlInput input = Yaml.createYamlInput(new File("searchproviders.yml"));
YamlSequence seq = input.readYamlSequence();
for (int i = 0; i < seq.size(); i++)
{
try
{
YamlMapping map = seq.yamlMapping(i);
String type = map.string("type");
Class<? extends Searcher> clazz = (Class<? extends Searcher>) Class.forName("moe.nekojimi.musicsearcher.providers." + type);
Constructor<? extends Searcher> constructor = clazz.getConstructor(YamlMapping.class);
Searcher searcher = constructor.newInstance(map);
searchers.put(searcher.getName(), searcher);
} catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
}
}
System.out.println(searchers);
String query = "eminem lose yourself";
MetaSearcher metaSearch = MetaSearcher.loadYAML(new File("searchproviders.yml"));
try
String query = "everybodys circulation";
for (Searcher searcher: searchers.values())
{
List<Result> results = metaSearch.searchAndWait(Query.fullText(query));
// for (Searcher searcher: searchers.values())
// {
// System.out.println("Searching " + searcher.getName() + " for " + query);
// try
// {
// List<Result> results = searcher.searchAndWait(Query.fullText(query));
for (Result result : results)
System.out.println("\t" + result);
// } catch (InterruptedException | ExecutionException ex) {
// Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
// }
// }
} catch (ExecutionException | InterruptedException ex)
{
Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
System.out.println("Searching " + searcher.getName() + " for " + query);
try
{
List<Result> results = searcher.searchAndWait(Query.fullText(query));
for (Result result: results)
{
System.out.println("\t" + result);
}
} catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}

52
src/main/java/moe/nekojimi/musicsearcher/Result.java
View File

@ -11,18 +11,13 @@ import java.net.URL;
*
* @author jim
*/
public class Result implements Comparable<Result>
public class Result
{
private URL link;
private String artist;
private String album;
private String title;
private String sourceName;
private String sourceAbbr;
private double score;
public URL getLink() {
return link;
}
@ -55,39 +50,12 @@ public class Result implements Comparable<Result>
this.title = title;
}
public String getSourceName()
{
return sourceName;
}
public void setSource(String name, String abbr)
{
sourceName = name;
sourceAbbr = abbr;
}
public String getSourceAbbr()
{
return sourceAbbr;
}
public double getScore()
{
return score;
}
public void setScore(double score)
{
this.score = score;
}
@Override
public String toString()
{
return "Result{" + "artist=" + artist + ", album=" + album + ", title=" + title + ", sourceAbbr=" + sourceAbbr + ", score=" + score + '}';
public String toString() {
return "Result{" + "link=" + link + ", artist=" + artist + ", album=" + album + ", title=" + title + '}';
}
public void setAlbumArtist(String field)
public void setAlbumArtist(String field)
{
// System.out.println("Parsing album-artist: " + field);
String fieldLower = field.toLowerCase();
@ -110,17 +78,9 @@ public class Result implements Comparable<Result>
else
album += word + " ";
}
}
artist = artist.trim();
album = album.trim();
}
}
@Override
public int compareTo(Result t)
{
return (int) Math.signum(score - t.score);
}
}

77
src/main/java/moe/nekojimi/musicsearcher/providers/MetaSearcher.java
View File

@ -5,26 +5,11 @@
*/
package moe.nekojimi.musicsearcher.providers;
import com.amihaiemil.eoyaml.Yaml;
import com.amihaiemil.eoyaml.YamlInput;
import com.amihaiemil.eoyaml.YamlMapping;
import com.amihaiemil.eoyaml.YamlSequence;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.Collection;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import moe.nekojimi.musicsearcher.Main;
import moe.nekojimi.musicsearcher.Query;
import moe.nekojimi.musicsearcher.Result;
@ -34,44 +19,18 @@ import moe.nekojimi.musicsearcher.Result;
*/
public class MetaSearcher extends Searcher
{
private final Set<Searcher> searchers = new HashSet<>();
private int minSearchTime = 10000; // ms
private int maxSearchTime = 30000; // ms
public MetaSearcher(Collection<Searcher> searchers)
public MetaSearcher()
{
super("meta");
this.searchers.addAll(searchers);
}
public static MetaSearcher loadYAML(File file)
{
try
{
List<Searcher> searchers = new ArrayList<>();
YamlInput input = Yaml.createYamlInput(file);
YamlSequence seq = input.readYamlSequence();
for (int i = 0; i < seq.size(); i++)
{
YamlMapping map = seq.yamlMapping(i);
String type = map.string("type");
Class<? extends Searcher> clazz = (Class<? extends Searcher>) Class.forName("moe.nekojimi.musicsearcher.providers." + type);
Constructor<? extends Searcher> constructor = clazz.getConstructor(YamlMapping.class);
Searcher searcher = constructor.newInstance(map);
searchers.add(searcher);
}
MetaSearcher metaSearch = new MetaSearcher(searchers);
return metaSearch;
} catch (IOException | ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex)
{
throw new IllegalArgumentException("Could not load MetaSearcher config from YAML file " + file.getAbsolutePath(), ex);
}
}
@Override
protected List<Result> doSearch(Query query)
protected List<Result> doSearch(Query query)
{
List<Result> results = new ArrayList<>();
List<CompletableFuture<List<Result>>> searches = new ArrayList<>();
@ -79,28 +38,18 @@ public class MetaSearcher extends Searcher
{
CompletableFuture<List<Result>> search = searcher.search(query);
searches.add(search);
// search.whenComplete((t, u) ->
// {
// if (u == null)
// {
// results.addAll(t);
//// searches.remove(search);
// }
// });
search.whenComplete((t, u) ->
{
if (u == null)
{
results.addAll(t);
// searches.remove(search);
}
});
}
// CompletableFuture.allOf((CompletableFuture<?>[]) searches.toArray());
for (CompletableFuture<List<Result>> search : searches)
try
{
List<Result> result = search.get(30, TimeUnit.SECONDS);
results.addAll(result);
} catch (TimeoutException | InterruptedException | ExecutionException ex)
{
Logger.getLogger(MetaSearcher.class.getName()).log(Level.SEVERE, null, ex);
}
CompletableFuture.allOf((CompletableFuture<?>[]) searches.toArray());
return results;
}
}

64
src/main/java/moe/nekojimi/musicsearcher/providers/Searcher.java
View File

@ -6,10 +6,7 @@
package moe.nekojimi.musicsearcher.providers;
import com.amihaiemil.eoyaml.YamlMapping;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
@ -18,97 +15,62 @@ import java.util.concurrent.TimeoutException;
import moe.nekojimi.musicsearcher.Query;
import moe.nekojimi.musicsearcher.QueryFieldUnsupportedException;
import moe.nekojimi.musicsearcher.Result;
import org.apache.commons.lang3.StringUtils;
/**
*
* @author jim
*/
public abstract class Searcher
public abstract class Searcher
{
protected final String name;
protected String abbr;
final String name;
private final ForkJoinPool executor;
public Searcher(String name)
public Searcher(String name)
{
this.name = name;
this.abbr = name.substring(0, 2);
this.executor = new ForkJoinPool();
}
public Searcher(YamlMapping yaml)
{
this(yaml.string("name"));
abbr = yaml.string("abbr");
assert yaml.string("type").equals(this.getClass().getSimpleName());
}
public String getName()
public String getName()
{
return name;
}
public List<Result> searchAndWait(Query query) throws InterruptedException, ExecutionException
{
try
try
{
return searchAndWait(query, Long.MAX_VALUE, TimeUnit.DAYS);
} catch (TimeoutException ex)
} catch (TimeoutException ex)
{
throw new RuntimeException("This should never happen!",ex);
}
}
public List<Result> searchAndWait(Query query, long limit, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException
{
return search(query).get(limit, unit);
}
public CompletableFuture<List<Result>> search(Query query)
{
CompletableFuture<List<Result>> future = new CompletableFuture<>();
future.completeAsync(() -> searchWrapper(query), executor);
future.completeAsync(()->doSearch(query), executor);
return future;
}
protected List<Result> searchWrapper(Query query)
{
List<Result> results = doSearch(query);
for (Result result : results)
{
double artistScore = 1;
double titleScore = 1;
double albumScore = 1;
double fullTextScore = 0;
Set<String> toCompare = new HashSet<>();
toCompare.add(result.getAlbum());
toCompare.add(result.getArtist());
toCompare.add(result.getTitle());
for (String comparison : toCompare)
{
if (comparison == null)
continue;
double score = StringUtils.getJaroWinklerDistance(query.getTextSearch(), comparison);
if (score > fullTextScore)
fullTextScore = score;
}
result.setScore(albumScore * artistScore * titleScore * fullTextScore);
}
Collections.sort(results, Collections.reverseOrder());
return results;
}
protected abstract List<Result> doSearch(Query query) throws QueryFieldUnsupportedException;
@Override
public String toString() {
return this.getClass().getSimpleName() + "{" + "name=" + name + '}';
}
}

37
src/main/java/moe/nekojimi/musicsearcher/providers/WebScraperSearcher.java
View File

@ -35,15 +35,15 @@ public class WebScraperSearcher extends Searcher
{
protected String searchUrl;
protected URL rootURL;
protected String resultSelector;
protected String resultArtistSelector;
protected String resultTitleSelector;
protected String resultLinkSelector;
protected String resultAlbumArtistSelector;
protected Map<String,String> searchFields = new HashMap<>();
protected Parser<?,?> parser;
public WebScraperSearcher(String name)
@ -51,7 +51,7 @@ public class WebScraperSearcher extends Searcher
super(name);
}
public WebScraperSearcher(YamlMapping yaml) throws MalformedURLException
public WebScraperSearcher(YamlMapping yaml) throws MalformedURLException
{
super(yaml);
searchUrl = yaml.string("search_url");
@ -59,7 +59,7 @@ public class WebScraperSearcher extends Searcher
// rootURL = fillURL(Query.fullText(""));
// rootURL = new URL(rootURL.getProtocol(), rootURL.getHost(), "");
resultSelector = yaml.string("result_selector");
YamlMapping fields = yaml.yamlMapping("result_fields");
if (fields != null)
{
@ -68,11 +68,11 @@ public class WebScraperSearcher extends Searcher
resultLinkSelector = fields.string("link");
resultAlbumArtistSelector = fields.string("album_artist");
}
YamlMapping searchFieldMap = yaml.yamlMapping("search_fields");
for (YamlNode key: searchFieldMap.keys())
searchFields.put(key.asScalar().value(), searchFieldMap.string(key));
String formatName = yaml.string("format");
switch(formatName)
{
@ -86,21 +86,21 @@ public class WebScraperSearcher extends Searcher
}
@Override
protected List<Result> doSearch(Query query)
protected List<Result> doSearch(Query query)
{
try
try
{
URL url = fillURL(query);
InputStream input = url.openStream();
return processResults(parser, input);
} catch (IOException ex)
} catch (IOException ex)
{
Logger.getLogger(WebScraperSearcher.class.getName()).log(Level.SEVERE, null, ex);
return List.of();
}
}
}
protected URL fillURL(Query query) throws MalformedURLException
protected URL fillURL(Query query) throws MalformedURLException
{
// URL url = new URL(searchUrl.replaceAll("\\$QUERY", URLEncoder.encode(query.getTextSearch(), Charset.forName("utf-8"))));
try
@ -122,12 +122,12 @@ public class WebScraperSearcher extends Searcher
throw new MalformedURLException();
}
}
protected String transformSearchString(String search)
{
return search;
}
protected <E> List<Result> processResults(Parser<?,E> parser, InputStream input)
{
Collection<E> resultEles = parser.getResults(input, resultSelector);
@ -142,21 +142,20 @@ public class WebScraperSearcher extends Searcher
try
{
Result res = new Result();
res.setSource(name, abbr);
// Artist
if (resultArtistSelector != null)
res.setArtist(parser.getField(ele, resultArtistSelector).trim());
res.setArtist(parser.getField(ele, resultArtistSelector));
// Title
if (resultTitleSelector != null)
res.setTitle(parser.getField(ele, resultTitleSelector).trim());
res.setTitle(parser.getField(ele, resultTitleSelector));
// Link
if (resultLinkSelector != null)
res.setLink(parser.getURLField(ele, rootURL, resultLinkSelector));
// Artist + Album
if (resultAlbumArtistSelector != null)
res.setAlbumArtist(parser.getField(ele, resultAlbumArtistSelector).trim());
res.setAlbumArtist(parser.getField(ele, resultAlbumArtistSelector));
// Artist + Title
return res;