using FireSharp.Config; using FireSharp.Interfaces; using FireSharp.Response; using Herse.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO.Compression; using System.Configuration; using Newtonsoft.Json; using System.Security.Cryptography; namespace SongCrawler { class Program { // Properties to store commonly used values from args private static string Controller { get; set; } private static string SongsFolderPath { get; set; } private static string[] Args { get; set; } private static FireSharp.FirebaseClient FirebaseClient { get; set; } private static Guid GuidFromString(string input) { using (MD5 md5 = MD5.Create()) { byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(input)); return new Guid(hash); } } // Helper methods to eliminate duplication private static FireSharp.FirebaseClient CreateFirebaseClient() { IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; return new FireSharp.FirebaseClient(config); } private static string GetControllerPath(string controller, string pathType) { return string.Format("controllers/{0}/{1}", controller, pathType); } private static List GetAllMusicFiles(string songpath) { List files = new List(); files.AddRange(FindFiles("mp4", songpath)); files.AddRange(FindFiles("mp3", songpath)); files.AddRange(FindFiles("zip", songpath)); return files; } private static void ValidateArgs(string[] args, int expectedLength, string usage) { if (args.Length != expectedLength) { Console.WriteLine(usage); return; } } private static void InitializeFromArgs(string[] args) { Args = args; if (args.Length > 0) { Controller = args[0]; } if (args.Length > 1) { SongsFolderPath = args[1]; } FirebaseClient = CreateFirebaseClient(); } // Additional helper methods for more refactoring private static List LoadFirebaseData(FireSharp.FirebaseClient client, string path) where T : class { try { return client.Get(path).ResultAs>(); } catch { return convertToList(client.Get(path).Body); } } private static void ProcessFilesToSongs(List files, List songs, bool checkDuplicates = false, bool debug = false, int debugLimit = 1000) { Song song = null; int i = 0; foreach (string filepath in files) { i++; try { song = MakeSong(filepath); Console.WriteLine(string.Format("{0:000000}/{1} - {2} - {3}", i, files.Count, song.Artist, song.Title)); if (checkDuplicates) { if (!songs.Any(s => s.Title.ToLower() == song.Title.ToLower() && s.Artist.ToLower() == song.Artist.ToLower())) { songs.Add(song); } } else { songs.Add(song); } } catch (Exception ex) { Console.WriteLine(ex.Message); } if (debug && i > debugLimit) { break; } } } private static void SyncSongProperties(List songs, List sourceList, Action propertySetter) { if (sourceList != null && sourceList.Count > 0) { sourceList.ForEach(s => { if (s != null) { var found = songs.Find(ls => s.Path.ToLower() == ls.Path.ToLower()); if (found != null) { propertySetter(found); } } }); } } private static Dictionary> FindDuplicateSongs(List songs, Func filter = null) { Dictionary> dupes = new Dictionary>(); int i = 0; foreach (var localsong in songs) { i++; Console.WriteLine(string.Format("Checking for {0:000000}/{1}) {2} - {3}", i, songs.Count, localsong.Artist, localsong.Title)); if (!string.IsNullOrEmpty(localsong.Artist) && !string.IsNullOrEmpty(localsong.Title)) { if (filter != null && !filter(localsong)) continue; string key = localsong.Artist + " - " + localsong.Title; if (!dupes.ContainsKey(key)) { var dsongs = songs.Where(s => s.Title.ToLower() == localsong.Title.ToLower() && s.Artist.ToLower() == localsong.Artist.ToLower()); if (dsongs.Count() > 1) { List d = new List(); d.AddRange(dsongs.ToList()); dupes.Add(key, d); } } } } return dupes; } static void Main(string[] args) { InitializeFromArgs(args); if (args.Count() == 3) { DeleteSongs(); } else { CrawlSongs(); } } private static void CrawlSongs() { //string [] test = { "mbrucedogs", "z://" }; //args = test; var debug = false; ValidateArgs(Args, 2, "usage: songcrawler partyid songspath"); if (Args.Length != 2) return; string songsPath = GetControllerPath(Controller, "songs"); string favoritesPath = GetControllerPath(Controller, "favorites"); string disabledPath = GetControllerPath(Controller, "disabled"); Console.WriteLine("Loading current library"); List songs = new List(); List disabled = LoadFirebaseData(FirebaseClient, disabledPath); List favorited = LoadFirebaseData(FirebaseClient, favoritesPath); FirebaseClient.Set(songsPath, songs); List files = GetAllMusicFiles(SongsFolderPath); ProcessFilesToSongs(files, songs, debug: debug, debugLimit: 1000); //sync all favorite, history, disabled SyncSongProperties(songs, favorited, song => song.Favorite = true); SyncSongProperties(songs, disabled, song => song.Disabled = true); FirebaseClient.Set(songsPath, songs); //string test = string.Format("controllers/{0}/testsongs", Controller); //Dictionary testSongs = new Dictionary(); //foreach (Song s in songs) //{ // testSongs[s.Guid] = s; //} //FirebaseClient.Set(test, testSongs); var created = songs.Select(s => new CreatedSong(File.GetCreationTime(s.Path), s)).ToList(); var first200 = created.Where(s => s.created != null).OrderByDescending(s => s.created).Take(200); var added = first200.Select(s => new PathOnly(path: s.song.Path)).ToList(); string newSongs = GetControllerPath(Controller, "newSongs"); FirebaseClient.Set(newSongs, added); } private class PathOnly { [JsonProperty("path")] public String Path { get; set; } public PathOnly(string path) { this.Path = path; } } private static List convertToList(dynamic json) where T : class { dynamic data = JsonConvert.DeserializeObject(json); var list = new List(); foreach (var itemDynamic in data) { var fjson = itemDynamic.Value.ToString(); var f = JsonConvert.DeserializeObject(fjson); list.Add(f); } return list; } private static void DeleteSongs() { ValidateArgs(Args, 3, "usage: songcrawler partyid songspath delete"); if (Args.Length != 3) return; string firepath = GetControllerPath(Controller, "songs"); Console.WriteLine("Deleting Songs ..."); List songs = new List(); FirebaseClient.Set(firepath, songs); } private static void fixNewSongs() { string songsPath = GetControllerPath(Controller, "songs"); string newSongsPath = GetControllerPath(Controller, "newSongs"); List songs = FirebaseClient.Get(songsPath).ResultAs>(); List newSongs = FirebaseClient.Get(newSongsPath).ResultAs>(); List updated = new List(); foreach (Song n in newSongs){ var found = songs.First(s => s.Path == n.Path); if(found != null) { updated.Add(found); } } FirebaseClient.Set(newSongsPath, updated); } private static void fixHistory() { string historyPath = GetControllerPath(Controller, "history"); List history = null; try { history = FirebaseClient.Get(historyPath).ResultAs>(); } catch { dynamic data = JsonConvert.DeserializeObject(FirebaseClient.Get(historyPath).Body); history = new List(); foreach (var itemDynamic in data) { var fjson = itemDynamic.Value.ToString(); var f = JsonConvert.DeserializeObject(fjson); history.Add(f); } } FirebaseClient.Set(historyPath, history); } public class CreatedSong { public DateTime created { get; set; } public Song song { get; set; } public CreatedSong(DateTime created, Song song) { this.song = song; this.created = created; } } private static void CrawlNoDupeSongs() { ValidateArgs(Args, 2, "usage: songcrawler partyid songspath"); if (Args.Length != 2) return; string firepath = GetControllerPath(Controller, "songs"); Console.WriteLine("Loading current library"); List songs = new List(); List files = GetAllMusicFiles(SongsFolderPath); ProcessFilesToSongs(files, songs, checkDuplicates: true); Console.WriteLine(string.Format("{0:000000}/{1} unique songs", songs.Count(), files.Count())); FirebaseClient.Set(firepath, songs); } private static void FindDuplicates(string[] args) { string songpath = @"D:\KaraokeData\Karaoke"; // args[0]; List songs = songs = new List(); List files = GetAllMusicFiles(songpath); ProcessFilesToSongs(files, songs); var dupes = FindDuplicateSongs(songs); File.WriteAllText(@"D:\dupliates.json", JsonConvert.SerializeObject(dupes.OrderBy(o => o.Key))); } private static void DisableDuplicates() { ValidateArgs(Args, 1, "usage: songcrawler partyid songspath"); if (Args.Length != 1) return; string firepath = GetControllerPath(Controller, "songs"); Console.WriteLine("Loading current library"); List songs = FirebaseClient.Get(firepath).ResultAs>(); var dupes = FindDuplicateSongs(songs, song => !song.Disabled && song.Path.Contains(".mp4")); // Disable duplicate songs (keep the first one, disable the rest) foreach (var duplicateGroup in dupes.Values) { for (int i = 1; i < duplicateGroup.Count; i++) // Skip first one, disable the rest { duplicateGroup[i].Disabled = true; } } FirebaseClient.Set(firepath, songs); } private static Song MakeSong(string filepath) { Song song = null; var ext = Path.GetExtension(filepath).ToLower(); switch (ext) { case ".mp3": case ".mp4": song = ReadId3(filepath); break; case ".zip": song = ReadId3FromZip(filepath); break; } CheckTitle(song); song.Path = filepath; return song; } private static string[] FindFiles(string ext, string path) { Console.Write(string.Format("\rscanning {0} for {1} - ", path, ext)); string[] files = Directory.GetFiles(path, "*." + ext, SearchOption.AllDirectories); Console.WriteLine(string.Format("{0} found", files.Length)); return files; } private static void CheckTitle(Song song) { if(string.IsNullOrEmpty(song.Title)) { string file = Path.GetFileNameWithoutExtension(song.Path); string[] parts = file.Split('-'); if (parts.Length == 1) { song.Title = parts[0].Trim(); } else if (parts.Length > 1) { song.Artist = parts[parts.Length - 2].Trim(); song.Title = parts[parts.Length - 1].Trim(); } } } private static Song ReadId3FromZip(string zippath) { ZipFile.ExtractToDirectory(zippath, "c:\\temp"); string filepath = Directory.GetFiles("c:\\temp", "*.mp3")[0]; Song song = ReadId3(filepath); foreach (string file in Directory.GetFiles("c:\\temp")) File.Delete(file); return song; } static Song ReadId3(string path) { Song song = new Song(); TagLib.File tagFile; try { tagFile = TagLib.File.Create(path); song.Title = tagFile.Tag.Title.Trim(); song.Artist = tagFile.Tag.FirstPerformer.Trim(); } catch { // do nothing; } song.Path = path; return song; } } }