using AngleSharp.Parser.Html; using FireSharp.Config; using FireSharp.Interfaces; using Herse.Models; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Configuration; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Threading.Tasks; namespace BillboardPlaylistUpdater { class Program { static void Main(string[] args) { IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config); string firepath = "SongList"; Console.WriteLine("Loading current library"); List songList = client.Get(firepath).ResultAs>(); if (songList != null) Console.WriteLine(string.Format("{0} songList loaded", songList.Count)); else songList = new List(); SongList pop = Download("Pop-Songs","http://www.billboard.com/charts/pop-songs"); SongList rock = Download("Rock-Songs","http://www.billboard.com/charts/rock-songs"); SongList country = Download("Country-Songs", "http://www.billboard.com/charts/country-songs"); SongList hiphop = Download("R-B-Hip-Hop-Songs", "http://www.billboard.com/charts/r-b-hip-hop-songs"); List localSongList = new List(); localSongList.Add(pop); localSongList.Add(rock); localSongList.Add(country); localSongList.Add(hiphop); foreach (SongList sl in localSongList) { try { Console.WriteLine(string.Format("Checking for {0}", sl.Title)); var found = songList.Where(s => s.Title.ToLower() == sl.Title.ToLower()); if(found != null) { var items = found.ToList(); foreach (var item in items) { songList.Remove(item); } } songList.Add(sl); } catch (Exception ex) { Console.WriteLine(ex.Message); } } client.Set(firepath, songList); } static SongList Download(string listName, string url) { DateTime now = DateTime.Now; string title = now.Year + " - " + listName; Console.WriteLine("Downloading " + title); string html = DownloadHtml(url); SongList list = null; List songs = Parse(title, html); if (songs != null) { list = new SongList(); list.Title = title; list.Songs = songs; } return list; } static List Parse(string name, string html) { List songs = null; var parser = new HtmlParser(); var document = parser.Parse(html); var articles = document.QuerySelectorAll("article.chart-row"); if(articles.Count() > 0) { Console.WriteLine("Found " + articles.Count() + " Songs"); songs = new List(); } var i = 1; foreach (var article in articles) { var title = article.QuerySelector("h2.chart-row__song"); var artist = article.QuerySelector("a.chart-row__artist"); var position = article.QuerySelector("span.chart-row__current-week"); if (artist == null) { artist = article.QuerySelector("h3.chart-row__artist"); } if(title != null && artist != null && position != null) { SongListSong song = new SongListSong(); song.Artist = artist.InnerHtml.Trim().Replace("\n",""); song.Title = title.InnerHtml.Trim().Replace("\n",""); song.Position = Int32.Parse(position.InnerHtml.Trim()); songs.Add(song); } else { Console.WriteLine("couldn't find objects in " + title + " for Song #" + i); } i++; } Console.Write("Parsed " + songs.Count() + " Songs"); return songs; } static string DownloadHtml(string url) { string data = null; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); HttpWebResponse response = (HttpWebResponse)request.GetResponse(); if(response.StatusCode == HttpStatusCode.OK) { Stream receiveStream = response.GetResponseStream(); StreamReader readStream = null; if(response.CharacterSet == null) { readStream = new StreamReader(receiveStream); }else { readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet)); } data = readStream.ReadToEnd(); response.Close(); readStream.Close(); } return data; } } }