270 lines
10 KiB
C#
270 lines
10 KiB
C#
using AngleSharp.Parser.Html;
|
|
using DuoVia.FuzzyStrings;
|
|
using FireSharp.Config;
|
|
using FireSharp.Interfaces;
|
|
using Herse.Models;
|
|
using Newtonsoft.Json;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Configuration;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace BillboardPlaylistUpdater
|
|
{
|
|
class Program
|
|
{
|
|
static List<Song> songs = null;
|
|
static List<SongList> songList = null;
|
|
|
|
static void Main(string[] args)
|
|
{
|
|
//args = new string[] { "bsully5150" };
|
|
if (args.Length != 1)
|
|
{
|
|
Console.WriteLine("usage: songcrawler partyid songspath");
|
|
return;
|
|
}
|
|
string controller = args[0];
|
|
|
|
IFirebaseConfig config = new FirebaseConfig
|
|
{
|
|
AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"],
|
|
BasePath = ConfigurationManager.AppSettings["Firebase.Path"]
|
|
};
|
|
FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config);
|
|
songs = client.Get(string.Format("controllers/{0}/songs", controller)).ResultAs<List<Song>>();
|
|
|
|
string firepath = "songList";
|
|
Console.WriteLine("Loading current library");
|
|
songList = client.Get(firepath).ResultAs<List<SongList>>();
|
|
|
|
if (songList != null)
|
|
Console.WriteLine(string.Format("{0} songList loaded", songList.Count));
|
|
else
|
|
songList = new List<SongList>();
|
|
|
|
RunTest();
|
|
//update Shared Charts and save
|
|
UpdateCurrentCharts();
|
|
client.Set(firepath, songList);
|
|
|
|
//update Controller Charts for Local Search and save
|
|
UpdateSearchLists();
|
|
client.Set(string.Format("controllers/{0}/songList", controller), songList);
|
|
}
|
|
|
|
static void RunTest()
|
|
{
|
|
var testArtist = "Linkin Park Featuring Kiiara".RemoveCrap().ToLower();
|
|
var testTitle = "Heavy";
|
|
var psongs = songs.Where(s => s.Title.Contains(testTitle)).ToList();
|
|
foreach (var item in psongs)
|
|
{
|
|
var ia = item.Artist.RemoveCrap();
|
|
var it = item.Title.RemoveCrap();
|
|
var artist = DoesMatch(ia, testArtist);
|
|
var title = DoesMatch(it, testTitle);
|
|
}
|
|
}
|
|
|
|
static void UpdateCurrentCharts()
|
|
{
|
|
SongList hot100 = Download("Hot 100", "https://www.billboard.com/charts/hot-100");
|
|
SongList pop = Download("Pop-Songs", "https://www.billboard.com/charts/pop-songs");
|
|
SongList rock = Download("Rock-Songs", "https://www.billboard.com/charts/rock-songs");
|
|
SongList country = Download("Country-Songs", "https://www.billboard.com/charts/country-songs");
|
|
SongList hiphop = Download("R-B-Hip-Hop-Songs", "https://www.billboard.com/charts/r-b-hip-hop-songs");
|
|
List<SongList> localSongList = new List<SongList>();
|
|
localSongList.Add(pop);
|
|
localSongList.Add(rock);
|
|
localSongList.Add(country);
|
|
localSongList.Add(hiphop);
|
|
localSongList.Add(hot100);
|
|
|
|
foreach (SongList sl in localSongList)
|
|
{
|
|
try
|
|
{
|
|
Console.WriteLine(string.Format("Checking for {0}", sl.Title));
|
|
var found = songList.Where(s => s.Title.ToLower() == sl.Title.ToLower());
|
|
if (found != null)
|
|
{
|
|
var items = found.ToList();
|
|
foreach (var item in items)
|
|
{
|
|
songList.Remove(item);
|
|
}
|
|
}
|
|
songList.Add(sl);
|
|
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
}
|
|
|
|
songList = songList.OrderByDescending(l => l.Title).ToList();
|
|
}
|
|
|
|
static void UpdateSearchLists()
|
|
{
|
|
//update the controller SongLists
|
|
foreach (var list in songList)
|
|
{
|
|
Console.WriteLine("********************************************************");
|
|
Console.WriteLine(string.Format("Matching Controllers Songs for {0}", list.Title));
|
|
Console.WriteLine("********************************************************");
|
|
Search(list);
|
|
}
|
|
}
|
|
|
|
static void Search(SongList list)
|
|
{
|
|
foreach (var song in list.Songs)
|
|
{
|
|
song.FoundSongs.Clear();
|
|
var bA = song.Artist.RemoveCrap().ToLower();
|
|
var bT = song.Title.RemoveCrap().ToLower();
|
|
|
|
foreach (var item in songs)
|
|
{
|
|
if (item.Artist != null && item.Title != null)
|
|
{
|
|
var t = item.Title.RemoveCrap().ToLower();
|
|
var a = item.Artist.RemoveCrap().ToLower();
|
|
bool titleMatch = DoesMatch(bT, t);
|
|
if (titleMatch && DoesMatch(bA, a))
|
|
{
|
|
song.FoundSongs.Add(item);
|
|
}
|
|
}
|
|
}
|
|
Console.WriteLine("Found ({0}) Song:{1} - {2}", song.FoundSongs.Count(), song.Artist, song.Title);
|
|
|
|
}
|
|
}
|
|
|
|
static bool DoesMatch(string primary, string toMatch)
|
|
{
|
|
if (primary.Contains(toMatch) || toMatch.Contains(primary)) { return true; }
|
|
int diff = primary.LevenshteinDistance(toMatch);
|
|
int distance = 4;
|
|
if (toMatch.Length < 6) { distance = 2; }
|
|
return diff < distance;
|
|
}
|
|
|
|
static SongList Download(string listName, string url)
|
|
{
|
|
DateTime now = DateTime.Now;
|
|
string title = now.Year + " - " + listName;
|
|
|
|
Console.WriteLine("Downloading " + title);
|
|
|
|
string html = DownloadHtml(url);
|
|
|
|
SongList list = null;
|
|
List<SongListSong> songs = Parse(title, html);
|
|
if (songs != null)
|
|
{
|
|
list = new SongList();
|
|
list.Title = title;
|
|
list.Songs = songs;
|
|
}
|
|
return list;
|
|
}
|
|
|
|
static List<SongListSong> Parse(string name, string html)
|
|
{
|
|
List<SongListSong> songs = null;
|
|
var parser = new HtmlParser();
|
|
var document = parser.Parse(html);
|
|
//2-?
|
|
var articles = document.QuerySelectorAll("div.chart-list-item ");
|
|
if (articles.Count() > 0)
|
|
{
|
|
Console.WriteLine("Found " + articles.Count() + " Songs");
|
|
songs = new List<SongListSong>();
|
|
}
|
|
|
|
//1
|
|
var number1artist = document.QuerySelectorAll("div.chart-number-one__artist ").First().InnerHtml.TrimStart().TrimEnd();
|
|
var number1title = document.QuerySelectorAll("div.chart-number-one__title ").First().InnerHtml.TrimStart().TrimEnd();
|
|
var number1 = new SongListSong();
|
|
number1.Artist = number1artist;
|
|
number1.Title = number1title;
|
|
number1.Position = 1;
|
|
if (number1artist.Contains("href"))
|
|
{
|
|
var start = number1artist.IndexOf(">") + 1;
|
|
var end = number1artist.IndexOf("<",1) - 1;
|
|
var art = number1artist.Substring(start, end - start);
|
|
number1.Artist = art.TrimStart().TrimEnd();
|
|
}
|
|
songs.Add(number1);
|
|
|
|
|
|
var i = 1;
|
|
foreach (var article in articles)
|
|
{
|
|
var title = article.Attributes["data-title"].Value;
|
|
var artist = article.Attributes["data-artist"].Value;
|
|
var position = article.Attributes["data-rank"].Value;
|
|
var song = new SongListSong();
|
|
song.Artist = artist;
|
|
song.Title = title;
|
|
song.Position = Convert.ToInt32(position);
|
|
songs.Add(song);
|
|
i++;
|
|
}
|
|
Console.Write("Parsed " + songs.Count() + " Songs");
|
|
return songs;
|
|
}
|
|
|
|
static string DownloadHtml(string url)
|
|
{
|
|
string data = null;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
|
|
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true;
|
|
ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
|
request.Method = "GET";
|
|
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
|
|
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
Stream receiveStream = response.GetResponseStream();
|
|
StreamReader readStream = null;
|
|
if (response.CharacterSet == null)
|
|
{
|
|
readStream = new StreamReader(receiveStream);
|
|
}
|
|
else
|
|
{
|
|
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
|
|
}
|
|
data = readStream.ReadToEnd();
|
|
|
|
response.Close();
|
|
readStream.Close();
|
|
}
|
|
return data;
|
|
}
|
|
}
|
|
|
|
static class StingExtension
|
|
{
|
|
public static string RemoveCrap(this String str)
|
|
{
|
|
string regex = "(\\[.*\\])|(\".*\")|('.*')|(\\(.*\\))";
|
|
return Regex.Replace(str, regex, "").ToLower().Replace("ft.", "").Replace("feat.", "").Replace("featured", "").Replace("featuring", "").Replace("'", "").Replace(" "," ").Trim();
|
|
}
|
|
}
|
|
}
|
|
|