using AngleSharp.Parser.Html; using DuoVia.FuzzyStrings; using FireSharp.Config; using FireSharp.Interfaces; using Herse.Models; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Configuration; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace BillboardPlaylistUpdater { class Program { static List songs = null; static List songList = null; static void Main(string[] args) { args = new string[] { "mbrucedogs" }; if (args.Length != 1) { Console.WriteLine("usage: songcrawler partyid songspath"); return; } string controller = args[0]; IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config); songs = client.Get(string.Format("controllers/{0}/songs", controller)).ResultAs>(); string firepath = "songList"; Console.WriteLine("Loading current library"); songList = client.Get(firepath).ResultAs>(); if (songList != null) Console.WriteLine(string.Format("{0} songList loaded", songList.Count)); else songList = new List(); RunTest(); //update Shared Charts and save UpdateCurrentCharts(); client.Set(firepath, songList); //update Controller Charts for Local Search and save UpdateSearchLists(); client.Set(string.Format("controllers/{0}/songList", controller), songList); } static void RunTest() { var testArtist = "Linkin Park Featuring Kiiara".RemoveCrap().ToLower(); var testTitle = "Heavy"; var psongs = songs.Where(s => s.Title.Contains(testTitle)).ToList(); foreach (var item in psongs) { var ia = item.Artist.RemoveCrap(); var it = item.Title.RemoveCrap(); var artist = DoesMatch(ia, testArtist); var title = DoesMatch(it, testTitle); } } static void UpdateCurrentCharts() { SongList hot100 = Download("Hot 100", "https://www.billboard.com/charts/hot-100"); SongList pop = Download("Pop-Songs", "https://www.billboard.com/charts/pop-songs"); SongList rock = Download("Rock-Songs", "https://www.billboard.com/charts/rock-songs"); SongList country = Download("Country-Songs", "https://www.billboard.com/charts/country-songs"); SongList hiphop = Download("R-B-Hip-Hop-Songs", "https://www.billboard.com/charts/r-b-hip-hop-songs"); List localSongList = new List(); localSongList.Add(pop); localSongList.Add(rock); localSongList.Add(country); localSongList.Add(hiphop); localSongList.Add(hot100); foreach (SongList sl in localSongList) { try { Console.WriteLine(string.Format("Checking for {0}", sl.Title)); var found = songList.Where(s => s.Title.ToLower() == sl.Title.ToLower()); if (found != null) { var items = found.ToList(); foreach (var item in items) { songList.Remove(item); } } songList.Add(sl); } catch (Exception ex) { Console.WriteLine(ex.Message); } } songList = songList.OrderByDescending(l => l.Title).ToList(); } static void UpdateSearchLists() { //update the controller SongLists foreach (var list in songList) { Console.WriteLine("********************************************************"); Console.WriteLine(string.Format("Matching Controllers Songs for {0}", list.Title)); Console.WriteLine("********************************************************"); Search(list); } } static void Search(SongList list) { foreach (var song in list.Songs) { song.FoundSongs.Clear(); var bA = song.Artist.RemoveCrap().ToLower(); var bT = song.Title.RemoveCrap().ToLower(); foreach (var item in songs) { if (item.Artist != null && item.Title != null) { var t = item.Title.RemoveCrap().ToLower(); var a = item.Artist.RemoveCrap().ToLower(); bool titleMatch = DoesMatch(bT, t); if (titleMatch && DoesMatch(bA, a)) { song.FoundSongs.Add(item); } } } Console.WriteLine("Found ({0}) Song:{1} - {2}", song.FoundSongs.Count(), song.Artist, song.Title); } } static bool DoesMatch(string primary, string toMatch) { if (primary.Contains(toMatch) || toMatch.Contains(primary)) { return true; } int diff = primary.LevenshteinDistance(toMatch); int distance = 4; if (toMatch.Length < 6) { distance = 2; } return diff < distance; } static SongList Download(string listName, string url) { DateTime now = DateTime.Now; string title = now.Year + " - " + listName; Console.WriteLine("Downloading " + title); string html = DownloadHtml(url); SongList list = null; List songs = Parse(title, html); if (songs != null) { list = new SongList(); list.Title = title; list.Songs = songs; } return list; } static List Parse(string name, string html) { List songs = null; var parser = new HtmlParser(); var document = parser.Parse(html); //2-? var articles = document.QuerySelectorAll("div.chart-list-item "); if (articles.Count() > 0) { Console.WriteLine("Found " + articles.Count() + " Songs"); songs = new List(); } //1 var number1artist = document.QuerySelectorAll("div.chart-number-one__artist ").First().InnerHtml.TrimStart().TrimEnd(); var number1title = document.QuerySelectorAll("div.chart-number-one__title ").First().InnerHtml.TrimStart().TrimEnd(); var number1 = new SongListSong(); number1.Artist = number1artist; number1.Title = number1title; number1.Position = 1; if (number1artist.Contains("href")) { var start = number1artist.IndexOf(">") + 1; var end = number1artist.IndexOf("<",1) - 1; var art = number1artist.Substring(start, end - start); number1.Artist = art.TrimStart().TrimEnd(); } songs.Add(number1); var i = 1; foreach (var article in articles) { var title = article.Attributes["data-title"].Value; var artist = article.Attributes["data-artist"].Value; var position = article.Attributes["data-rank"].Value; var song = new SongListSong(); song.Artist = artist; song.Title = title; song.Position = Convert.ToInt32(position); songs.Add(song); i++; } Console.Write("Parsed " + songs.Count() + " Songs"); return songs; } static string DownloadHtml(string url) { string data = null; ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12; ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true; ServicePointManager.ServerCertificateValidationCallback = delegate { return true; }; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.Method = "GET"; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode == HttpStatusCode.OK) { Stream receiveStream = response.GetResponseStream(); StreamReader readStream = null; if (response.CharacterSet == null) { readStream = new StreamReader(receiveStream); } else { readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet)); } data = readStream.ReadToEnd(); response.Close(); readStream.Close(); } return data; } } static class StingExtension { public static string RemoveCrap(this String str) { string regex = "(\\[.*\\])|(\".*\")|('.*')|(\\(.*\\))"; return Regex.Replace(str, regex, "").ToLower().Replace("ft.", "").Replace("feat.", "").Replace("featured", "").Replace("featuring", "").Replace("'", "").Replace(" "," ").Trim(); } } }