using FireSharp.Config; using FireSharp.Interfaces; using FireSharp.Response; using Herse.Models; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO.Compression; using System.Configuration; using Newtonsoft.Json; namespace SongCrawler { class Program { static void Main(string[] args) { //FindDuplicates(args); if (args.Last() == "-sl") { UploadSongList(args); } else { CrawlSongs(args); } } private static void UploadSongList(string[] args) { string songlistpath = args[0]; if (args.Length != 2) { Console.WriteLine("usage: songcrawler jsonpath"); return; } IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config); string firepath = "SongList"; Console.WriteLine("Loading current library"); List songList = client.Get(firepath).ResultAs>(); if (songList != null) Console.WriteLine(string.Format("{0} songList loaded", songList.Count)); else songList = new List(); List localSongList = JsonConvert.DeserializeObject>(File.ReadAllText(songlistpath)); foreach (SongList sl in localSongList) { try { Console.WriteLine(string.Format("Checking for {0}", sl.Title)); if (!songList.Any(s => s.Title.ToLower() == sl.Title.ToLower())) songList.Add(sl); } catch (Exception ex) { Console.WriteLine(ex.Message); } } client.Set(firepath, songList); } private static void CrawlSongs(string[] args) { if (args.Length != 2) { Console.WriteLine("usage: songcrawler partyid songspath"); return; } string controller = args[0]; string songpath = args[1]; IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config); string firepath = string.Format("controllers/{0}/songs", controller); Console.WriteLine("Loading current library"); Dictionary dict = client.Get(firepath).ResultAs>(); List songs = dict.Values.ToList(); if (songs != null) Console.WriteLine(string.Format("{0} songs loaded", songs.Count)); else songs = new List(); List files = new List(); files.AddRange(FindFiles("mp4", songpath)); files.AddRange(FindFiles("mp3", songpath)); files.AddRange(FindFiles("zip", songpath)); Song song = null; int i = 0; foreach (string filepath in files) { i++; try { song = MakeSong(filepath); Console.WriteLine(string.Format("{0:000000}/{1} - {2}", i, files.Count, song.Title)); if (!songs.Any(s => s.Path.ToLower() == song.Path.ToLower())) songs.Add(song); } catch (Exception ex) { Console.WriteLine(ex.Message); } } client.Set(firepath, songs); List created = new List(); foreach (Song s in songs) { created.Add(new CreatedSong(File.GetCreationTime(s.Path),s)); } var first200 = created.Where(s => s.created != null).OrderByDescending(s => s.created).Take(200); var added = first200.Select(s=>s.song).ToList(); string newSongs = string.Format("controllers/{0}/newSongs", controller); client.Set(newSongs, added); } public class CreatedSong { public DateTime created { get; set; } public Song song { get; set; } public CreatedSong(DateTime created, Song song) { this.song = song; this.created = created; } } private static void CrawlNoDupeSongs(string[] args) { if (args.Length != 2) { Console.WriteLine("usage: songcrawler partyid songspath"); return; } string controller = args[0]; string songpath = args[1]; IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config); string firepath = string.Format("controllers/{0}/songs", controller); Console.WriteLine("Loading current library"); List songs = new List(); List files = new List(); files.AddRange(FindFiles("mp4", songpath)); files.AddRange(FindFiles("mp3", songpath)); files.AddRange(FindFiles("zip", songpath)); Song song = null; int i = 0; foreach (string filepath in files) { i++; try { song = MakeSong(filepath); Console.WriteLine(string.Format("{0:000000}/{1} - {2}", i, files.Count(), song.Title)); if (!songs.Any(s => s.Title.ToLower() == song.Title.ToLower() && s.Artist.ToLower() == song.Artist.ToLower())) songs.Add(song); } catch (Exception ex) { Console.WriteLine(ex.Message); } } Console.WriteLine(string.Format("{0:000000}/{1} unique songs", songs.Count(), files.Count())); client.Set(firepath, songs); } private static void FindDuplicates(string[] args) { string songpath = @"D:\KaraokeData\Karaoke"; // args[0]; List songs = songs = new List(); List files = new List(); files.AddRange(FindFiles("mp3", songpath)); files.AddRange(FindFiles("zip", songpath)); files.AddRange(FindFiles("mp4", songpath)); Song song = null; int i = 0; foreach (string filepath in files) { i++; try { song = MakeSong(filepath); Console.WriteLine(string.Format("{0:000000}/{1} - {2}", i, files.Count, song.Title)); songs.Add(song); } catch (Exception ex) { Console.WriteLine(ex.Message); } } i = 0; Dictionary> dupes = new Dictionary>(); foreach (var localsong in songs) { i++; Console.WriteLine(string.Format("Checking for {0:000000}/{1}) {2} - {3}", i, files.Count, localsong.Artist, localsong.Title)); if (!string.IsNullOrEmpty(localsong.Artist) && !string.IsNullOrEmpty(localsong.Title)) { string key = localsong.Artist + " - " + localsong.Title; if (dupes.ContainsKey(key) == false) { var dsongs = songs.Where(s => s.Title.ToLower() == localsong.Title.ToLower() && s.Artist.ToLower() == localsong.Artist.ToLower()); if (dsongs.Count() > 1) { List d = new List(); d.AddRange(dsongs.ToList()); dupes.Add(key, d); } } } } File.WriteAllText(@"D:\dupliates.json", JsonConvert.SerializeObject(dupes.OrderBy(o => o.Key))); } private static void DisableDuplicates(string[] args) { if (args.Length != 1) { Console.WriteLine("usage: songcrawler partyid songspath"); return; } string controller = args[0]; IFirebaseConfig config = new FirebaseConfig { AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"], BasePath = ConfigurationManager.AppSettings["Firebase.Path"] }; FireSharp.FirebaseClient client = new FireSharp.FirebaseClient(config); string firepath = string.Format("controllers/{0}/songs", controller); Console.WriteLine("Loading current library"); List songs = client.Get(firepath).ResultAs>(); Dictionary> dupes = new Dictionary>(); int i = 0; foreach (var localsong in songs) { i++; Console.WriteLine(string.Format("Checking for {0:000000}/{1}) {2} - {3}", i, songs.Count, localsong.Artist, localsong.Title)); if (!string.IsNullOrEmpty(localsong.Artist) && !string.IsNullOrEmpty(localsong.Title) && localsong.Disabled == false && localsong.Path.Contains(".mp4")) { string key = localsong.Artist + " - " + localsong.Title; if (dupes.ContainsKey(key) == false) { var dsongs = songs.Where(s => s.Path != localsong.Path && s.Title.ToLower() == localsong.Title.ToLower() && s.Artist.ToLower() == localsong.Artist.ToLower() && localsong.Disabled == false); if (dsongs.Count() > 1) { List d = new List(); d.AddRange(dsongs.ToList()); dupes.Add(key, d); } foreach (var item in dsongs) { item.Disabled = true; } } } } client.Set(firepath, songs); } private static Song MakeSong(string filepath) { Song song = null; var ext = Path.GetExtension(filepath).ToLower(); switch (ext) { case ".mp3": case ".mp4": song = ReadId3(filepath); break; case ".zip": song = ReadId3FromZip(filepath); break; } CheckTitle(song); song.Path = filepath; return song; } private static string[] FindFiles(string ext, string path) { Console.Write(string.Format("\rscanning {0} for {1} - ", path, ext)); string[] files = Directory.GetFiles(path, "*." + ext, SearchOption.AllDirectories); Console.WriteLine(string.Format("{0} found", files.Length)); return files; } private static void CheckTitle(Song song) { if(string.IsNullOrEmpty(song.Title)) { string file = Path.GetFileNameWithoutExtension(song.Path); string[] parts = file.Split('-'); if (parts.Length == 1) { song.Title = parts[0].Trim(); } else if (parts.Length > 1) { song.Artist = parts[parts.Length - 2].Trim(); song.Title = parts[parts.Length - 1].Trim(); } } } private static Song ReadId3FromZip(string zippath) { ZipFile.ExtractToDirectory(zippath, "c:\\temp"); string filepath = Directory.GetFiles("c:\\temp", "*.mp3")[0]; Song song = ReadId3(filepath); foreach (string file in Directory.GetFiles("c:\\temp")) File.Delete(file); return song; } static Song ReadId3(string path) { Song song = new Song(); TagLib.File tagFile; try { tagFile = TagLib.File.Create(path); song.Title = tagFile.Tag.Title.Trim(); song.Artist = tagFile.Tag.FirstPerformer.Trim(); song.Genre = tagFile.Tag.FirstGenre.Trim(); song.Year = (int)tagFile.Tag.Year; } catch { // do nothing; } song.Path = path; return song; } } }