using FireSharp.Config; using FireSharp.Interfaces; using FireSharp.Response; using Herse.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO.Compression; using System.Configuration; using Newtonsoft.Json; using System.Security.Cryptography; namespace SongCrawler { class Program { private static Guid GuidFromString(string input) { using (MD5 md5 = MD5.Create()) { byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(input)); return new Guid(hash); } } // Helper methods to eliminate duplication private static FireSharp.FirebaseClient CreateFirebaseClient() { IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; return new FireSharp.FirebaseClient(config); } private static string GetControllerPath(string controller, string pathType) { return string.Format("controllers/{0}/{1}", controller, pathType); } private static List GetAllMusicFiles(string songpath) { List files = new List(); files.AddRange(FindFiles("mp4", songpath)); files.AddRange(FindFiles("mp3", songpath)); files.AddRange(FindFiles("zip", songpath)); return files; } private static void ValidateArgs(string[] args, int expectedLength, string usage) { if (args.Length != expectedLength) { Console.WriteLine(usage); return; } } // Additional helper methods for more refactoring private static List LoadFirebaseData(FireSharp.FirebaseClient client, string path) where T : class { try { return client.Get(path).ResultAs>(); } catch { return convertToList(client.Get(path).Body); } } private static void ProcessFilesToSongs(List files, List songs, bool checkDuplicates = false, bool debug = false, int debugLimit = 1000) { Song song = null; int i = 0; foreach (string filepath in files) { i++; try { song = MakeSong(filepath); Console.WriteLine(string.Format("{0:000000}/{1} - {2} - {3}", i, files.Count, song.Artist, song.Title)); if (checkDuplicates) { if (!songs.Any(s => s.Title.ToLower() == song.Title.ToLower() && s.Artist.ToLower() == song.Artist.ToLower())) { songs.Add(song); } } else { songs.Add(song); } } catch (Exception ex) { Console.WriteLine(ex.Message); } if (debug && i > debugLimit) { break; } } } private static void SyncSongProperties(List songs, List sourceList, Action propertySetter) { if (sourceList != null && sourceList.Count > 0) { sourceList.ForEach(s => { if (s != null) { var found = songs.Find(ls => s.Path.ToLower() == ls.Path.ToLower()); if (found != null) { propertySetter(found); } } }); } } private static Dictionary> FindDuplicateSongs(List songs, Func filter = null) { Dictionary> dupes = new Dictionary>(); int i = 0; foreach (var localsong in songs) { i++; Console.WriteLine(string.Format("Checking for {0:000000}/{1}) {2} - {3}", i, songs.Count, localsong.Artist, localsong.Title)); if (!string.IsNullOrEmpty(localsong.Artist) && !string.IsNullOrEmpty(localsong.Title)) { if (filter != null && !filter(localsong)) continue; string key = localsong.Artist + " - " + localsong.Title; if (!dupes.ContainsKey(key)) { var dsongs = songs.Where(s => s.Title.ToLower() == localsong.Title.ToLower() && s.Artist.ToLower() == localsong.Artist.ToLower()); if (dsongs.Count() > 1) { List d = new List(); d.AddRange(dsongs.ToList()); dupes.Add(key, d); } } } } return dupes; } static void Main(string[] args) { if (args.Count() == 3) { DeleteSongs(args); } else { CrawlSongs(args); } } private static void CrawlSongs(string[] args) { //string [] test = { "mbrucedogs", "z://" }; //args = test; var debug = false; ValidateArgs(args, 2, "usage: songcrawler partyid songspath"); if (args.Length != 2) return; string controller = args[0]; string songpath = args[1]; FireSharp.FirebaseClient client = CreateFirebaseClient(); string songsPath = GetControllerPath(controller, "songs"); string favoritesPath = GetControllerPath(controller, "favorites"); string disabledPath = GetControllerPath(controller, "disabled"); Console.WriteLine("Loading current library"); List songs = new List(); List disabled = LoadFirebaseData(client, disabledPath); List favorited = LoadFirebaseData(client, favoritesPath); client.Set(songsPath, songs); List files = GetAllMusicFiles(songpath); ProcessFilesToSongs(files, songs, debug: debug, debugLimit: 1000); //sync all favorite, history, disabled SyncSongProperties(songs, favorited, song => song.Favorite = true); SyncSongProperties(songs, disabled, song => song.Disabled = true); client.Set(songsPath, songs); //string test = string.Format("controllers/{0}/testsongs", controller); //Dictionary testSongs = new Dictionary(); //foreach (Song s in songs) //{ // testSongs[s.Guid] = s; //} //client.Set(test, testSongs); var created = songs.Select(s => new CreatedSong(File.GetCreationTime(s.Path), s)).ToList(); var first200 = created.Where(s => s.created != null).OrderByDescending(s => s.created).Take(200); var added = first200.Select(s => new PathOnly(path: s.song.Path)).ToList(); string newSongs = GetControllerPath(controller, "newSongs"); client.Set(newSongs, added); } private class PathOnly { [JsonProperty("path")] public String Path { get; set; } public PathOnly(string path) { this.Path = path; } } private static List convertToList(dynamic json) { dynamic data = JsonConvert.DeserializeObject(json); var list = new List(); foreach (var itemDynamic in data) { var fjson = itemDynamic.Value.ToString(); var f = JsonConvert.DeserializeObject(fjson); list.Add(f); } return list; } private static void DeleteSongs(string[] args) { ValidateArgs(args, 3, "usage: songcrawler partyid songspath delete"); if (args.Length != 3) return; string controller = args[0]; string songpath = args[1]; FireSharp.FirebaseClient client = CreateFirebaseClient(); string firepath = GetControllerPath(controller, "songs"); Console.WriteLine("Deleting Songs ..."); List songs = new List(); client.Set(firepath, songs); } private static void fixNewSongs(FireSharp.FirebaseClient client, String controller) { string songsPath = GetControllerPath(controller, "songs"); string newSongsPath = GetControllerPath(controller, "newSongs"); List songs = client.Get(songsPath).ResultAs>(); List newSongs = client.Get(newSongsPath).ResultAs>(); List updated = new List(); foreach (Song n in newSongs){ var found = songs.First(s => s.Path == n.Path); if(found != null) { updated.Add(found); } } client.Set(newSongsPath, updated); } private static void fixHistory(FireSharp.FirebaseClient client, String controller) { string historyPath = GetControllerPath(controller, "history"); List history = null; try { history = client.Get(historyPath).ResultAs>(); } catch { dynamic data = JsonConvert.DeserializeObject(client.Get(historyPath).Body); history = new List(); foreach (var itemDynamic in data) { var fjson = itemDynamic.Value.ToString(); var f = JsonConvert.DeserializeObject(fjson); history.Add(f); } } client.Set(historyPath, history); } public class CreatedSong { public DateTime created { get; set; } public Song song { get; set; } public CreatedSong(DateTime created, Song song) { this.song = song; this.created = created; } } private static void CrawlNoDupeSongs(string[] args) { ValidateArgs(args, 2, "usage: songcrawler partyid songspath"); if (args.Length != 2) return; string controller = args[0]; string songpath = args[1]; FireSharp.FirebaseClient client = CreateFirebaseClient(); string firepath = GetControllerPath(controller, "songs"); Console.WriteLine("Loading current library"); List songs = new List(); List files = GetAllMusicFiles(songpath); ProcessFilesToSongs(files, songs, checkDuplicates: true); Console.WriteLine(string.Format("{0:000000}/{1} unique songs", songs.Count(), files.Count())); client.Set(firepath, songs); } private static void FindDuplicates(string[] args) { string songpath = @"D:\KaraokeData\Karaoke"; // args[0]; List songs = songs = new List(); List files = GetAllMusicFiles(songpath); ProcessFilesToSongs(files, songs); var dupes = FindDuplicateSongs(songs); File.WriteAllText(@"D:\dupliates.json", JsonConvert.SerializeObject(dupes.OrderBy(o => o.Key))); } private static void DisableDuplicates(string[] args) { ValidateArgs(args, 1, "usage: songcrawler partyid songspath"); if (args.Length != 1) return; string controller = args[0]; FireSharp.FirebaseClient client = CreateFirebaseClient(); string firepath = GetControllerPath(controller, "songs"); Console.WriteLine("Loading current library"); List songs = client.Get(firepath).ResultAs>(); var dupes = FindDuplicateSongs(songs, song => !song.Disabled && song.Path.Contains(".mp4")); // Disable duplicate songs (keep the first one, disable the rest) foreach (var duplicateGroup in dupes.Values) { for (int i = 1; i < duplicateGroup.Count; i++) // Skip first one, disable the rest { duplicateGroup[i].Disabled = true; } } client.Set(firepath, songs); } private static Song MakeSong(string filepath) { Song song = null; var ext = Path.GetExtension(filepath).ToLower(); switch (ext) { case ".mp3": case ".mp4": song = ReadId3(filepath); break; case ".zip": song = ReadId3FromZip(filepath); break; } CheckTitle(song); song.Path = filepath; return song; } private static string[] FindFiles(string ext, string path) { Console.Write(string.Format("\rscanning {0} for {1} - ", path, ext)); string[] files = Directory.GetFiles(path, "*." + ext, SearchOption.AllDirectories); Console.WriteLine(string.Format("{0} found", files.Length)); return files; } private static void CheckTitle(Song song) { if(string.IsNullOrEmpty(song.Title)) { string file = Path.GetFileNameWithoutExtension(song.Path); string[] parts = file.Split('-'); if (parts.Length == 1) { song.Title = parts[0].Trim(); } else if (parts.Length > 1) { song.Artist = parts[parts.Length - 2].Trim(); song.Title = parts[parts.Length - 1].Trim(); } } } private static Song ReadId3FromZip(string zippath) { ZipFile.ExtractToDirectory(zippath, "c:\\temp"); string filepath = Directory.GetFiles("c:\\temp", "*.mp3")[0]; Song song = ReadId3(filepath); foreach (string file in Directory.GetFiles("c:\\temp")) File.Delete(file); return song; } static Song ReadId3(string path) { Song song = new Song(); TagLib.File tagFile; try { tagFile = TagLib.File.Create(path); song.Title = tagFile.Tag.Title.Trim(); song.Artist = tagFile.Tag.FirstPerformer.Trim(); } catch { // do nothing; } song.Path = path; return song; } } }