KaraokePC/BillboardPlaylistUpdater/Program.cs

using AngleSharp.Parser.Html;
using DuoVia.FuzzyStrings;
using FireSharp.Config;
using FireSharp.Interfaces;
using Herse.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace BillboardPlaylistUpdater
{
    class Program
    {
        static List<Song> songs = null;
        static List<SongList> songList = null;

        static void Main(string[] args)
        {
            //args = new string[] { "mbrucedogs" };
            if (args.Length != 1)
            {
                Console.WriteLine("usage: songcrawler partyid songspath");
                return;
            }
            string controller = args[0];

            IFirebaseConfig config = new FirebaseConfig
            {
                AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"],
                BasePath = ConfigurationManager.AppSettings["Firebase.Path"]
            };
            FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config);
            songs = client.Get(string.Format("controllers/{0}/songs", controller)).ResultAs<List<Song>>();

            string firepath = "songList";
            Console.WriteLine("Loading current library");
            songList = client.Get(firepath).ResultAs<List<SongList>>();

            if (songList != null)
                Console.WriteLine(string.Format("{0} songList loaded", songList.Count));
            else
                songList = new List<SongList>();

            RunTest();
            //update Shared Charts and save
            UpdateCurrentCharts();
            client.Set(firepath, songList);

            //update Controller Charts for Local Search and save
            UpdateSearchLists();
            client.Set(string.Format("controllers/{0}/songList", controller), songList);
        }

        static void RunTest()
        {
            var testArtist = "Linkin Park Featuring Kiiara".RemoveCrap().ToLower();
            var testTitle = "Heavy";
            var psongs = songs.Where(s => s.Title.Contains(testTitle)).ToList();
            foreach (var item in psongs)
            {
                var ia = item.Artist.RemoveCrap();
                var it = item.Title.RemoveCrap();
                var artist = DoesMatch(ia, testArtist);
                var title = DoesMatch(it, testTitle);
            }
        }  

        static void UpdateCurrentCharts()
        {
            SongList hot100 = Download("Hot 100", "http://www.billboard.com/charts/hot-100");
            SongList pop = Download("Pop-Songs", "http://www.billboard.com/charts/pop-songs");
            SongList rock = Download("Rock-Songs", "http://www.billboard.com/charts/rock-songs");
            SongList country = Download("Country-Songs", "http://www.billboard.com/charts/country-songs");
            SongList hiphop = Download("R-B-Hip-Hop-Songs", "http://www.billboard.com/charts/r-b-hip-hop-songs");

            List<SongList> localSongList = new List<SongList>();
            localSongList.Add(pop);
            localSongList.Add(rock);
            localSongList.Add(country);
            localSongList.Add(hiphop);
            localSongList.Add(hot100);

            foreach (SongList sl in localSongList)
            {
                try
                {
                    Console.WriteLine(string.Format("Checking for {0}", sl.Title));
                    var found = songList.Where(s => s.Title.ToLower() == sl.Title.ToLower());
                    if (found != null)
                    {
                        var items = found.ToList();
                        foreach (var item in items)
                        {
                            songList.Remove(item);
                        }
                    }
                    songList.Add(sl);

                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.Message);
                }
            }

            songList = songList.OrderByDescending(l => l.Title).ToList();
        }

        static void UpdateSearchLists()
        {
            //update the controller SongLists
            foreach (var list in songList)
            {
                Console.WriteLine("********************************************************");
                Console.WriteLine(string.Format("Matching Controllers Songs for {0}", list.Title));
                Console.WriteLine("********************************************************");
                Search(list);
            }
        }

        static void Search(SongList list)
        {
            foreach (var song in list.Songs)
            {
                song.FoundSongs.Clear();
                var bA = song.Artist.RemoveCrap().ToLower();
                var bT = song.Title.RemoveCrap().ToLower();

                foreach (var item in songs)
                {
                    if (item.Artist != null && item.Title != null)
                    {
                        var t = item.Title.RemoveCrap().ToLower();
                        var a = item.Artist.RemoveCrap().ToLower();
                        bool titleMatch = DoesMatch(bT, t);
                        if (titleMatch && DoesMatch(bA, a))
                        {
                            song.FoundSongs.Add(item);
                        }
                    }
                }
                Console.WriteLine("Found ({0}) Song:{1} - {2}", song.FoundSongs.Count(), song.Artist, song.Title);

            }
        }

        static bool DoesMatch(string primary, string toMatch)
        {
            if (primary.Contains(toMatch) || toMatch.Contains(primary)) { return true; }
            int diff = primary.LevenshteinDistance(toMatch);
            int distance = 4;
            if (toMatch.Length < 6) { distance = 2; }
            return diff < distance;
        }

        static SongList Download(string listName, string url)
        {
            DateTime now = DateTime.Now;
            string title = now.Year + " - " + listName;

            Console.WriteLine("Downloading " + title);

            string html = DownloadHtml(url);

            SongList list = null;
            List<SongListSong> songs = Parse(title, html);
            if (songs != null)
            {
                list = new SongList();
                list.Title = title;
                list.Songs = songs;
            }
            return list;
        }

        static List<SongListSong> Parse(string name, string html)
        {
            List<SongListSong> songs = null;
            var parser = new HtmlParser();
            var document = parser.Parse(html);
            var articles = document.QuerySelectorAll("article.chart-row");
            if (articles.Count() > 0)
            {
                Console.WriteLine("Found " + articles.Count() + " Songs");
                songs = new List<SongListSong>();
            }
            var i = 1;
            foreach (var article in articles)
            {
                var title = article.QuerySelector("h2.chart-row__song");
                var artist = article.QuerySelector("span.chart-row__artist");
                var position = article.QuerySelector("span.chart-row__current-week");
                if (artist == null)
                {
                    artist = article.QuerySelector("h3.chart-row__artist");
                }
                if (artist == null)
                {
                    artist = article.QuerySelector("a.chart-row__artist");
                }
                if (title != null && artist != null && position != null)
                {
                    SongListSong song = new SongListSong();
                    song.Artist = WebUtility.HtmlDecode(artist.InnerHtml.Trim().Replace("\n", ""));
                    song.Title = WebUtility.HtmlDecode(title.InnerHtml.Trim().Replace("\n", ""));
                    song.Position = Int32.Parse(position.InnerHtml.Trim());
                    songs.Add(song);
                }
                else
                {
                    Console.WriteLine("couldn't find objects in " + title + " for Song #" + i);
                }
                i++;
            }
            Console.Write("Parsed " + songs.Count() + " Songs");
            return songs;
        }

        static string DownloadHtml(string url)
        {
            string data = null;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            if (response.StatusCode == HttpStatusCode.OK)
            {
                Stream receiveStream = response.GetResponseStream();
                StreamReader readStream = null;
                if (response.CharacterSet == null)
                {
                    readStream = new StreamReader(receiveStream);
                }
                else
                {
                    readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
                }
                data = readStream.ReadToEnd();

                response.Close();
                readStream.Close();
            }
            return data;
        }
    }

    static class StingExtension
    {
        public static string RemoveCrap(this String str)
        {
            string regex = "(\\[.*\\])|(\".*\")|('.*')|(\\(.*\\))";
            return Regex.Replace(str, regex, "").ToLower().Replace("ft.", "").Replace("feat.", "").Replace("featured", "").Replace("featuring", "").Replace("'", "").Replace("  "," ").Trim();
        }
    }
}