438 lines
16 KiB
C#
438 lines
16 KiB
C#
using FireSharp.Config;
|
|
using FireSharp.Interfaces;
|
|
using FireSharp.Response;
|
|
using Herse.Models;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using System.IO.Compression;
|
|
using System.Configuration;
|
|
using Newtonsoft.Json;
|
|
using System.Security.Cryptography;
|
|
|
|
namespace SongCrawler
|
|
{
|
|
|
|
class Program
|
|
{
|
|
private static Guid GuidFromString(string input)
|
|
{
|
|
using (MD5 md5 = MD5.Create())
|
|
{
|
|
byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(input));
|
|
return new Guid(hash);
|
|
}
|
|
}
|
|
|
|
// Helper methods to eliminate duplication
|
|
private static FireSharp.FirebaseClient CreateFirebaseClient()
|
|
{
|
|
IFirebaseConfig config = new FirebaseConfig
|
|
{
|
|
AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"],
|
|
BasePath = ConfigurationManager.AppSettings["Firebase.Path"]
|
|
};
|
|
return new FireSharp.FirebaseClient(config);
|
|
}
|
|
|
|
private static string GetControllerPath(string controller, string pathType)
|
|
{
|
|
return string.Format("controllers/{0}/{1}", controller, pathType);
|
|
}
|
|
|
|
private static List<string> GetAllMusicFiles(string songpath)
|
|
{
|
|
List<string> files = new List<string>();
|
|
files.AddRange(FindFiles("mp4", songpath));
|
|
files.AddRange(FindFiles("mp3", songpath));
|
|
files.AddRange(FindFiles("zip", songpath));
|
|
return files;
|
|
}
|
|
|
|
private static void ValidateArgs(string[] args, int expectedLength, string usage)
|
|
{
|
|
if (args.Length != expectedLength)
|
|
{
|
|
Console.WriteLine(usage);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Additional helper methods for more refactoring
|
|
private static List<T> LoadFirebaseData<T>(FireSharp.FirebaseClient client, string path) where T : class
|
|
{
|
|
try
|
|
{
|
|
return client.Get(path).ResultAs<List<T>>();
|
|
}
|
|
catch
|
|
{
|
|
return convertToList(client.Get(path).Body);
|
|
}
|
|
}
|
|
|
|
private static void ProcessFilesToSongs(List<string> files, List<Song> songs, bool checkDuplicates = false, bool debug = false, int debugLimit = 1000)
|
|
{
|
|
Song song = null;
|
|
int i = 0;
|
|
foreach (string filepath in files)
|
|
{
|
|
i++;
|
|
try
|
|
{
|
|
song = MakeSong(filepath);
|
|
Console.WriteLine(string.Format("{0:000000}/{1} - {2} - {3}", i, files.Count, song.Artist, song.Title));
|
|
|
|
if (checkDuplicates)
|
|
{
|
|
if (!songs.Any(s => s.Title.ToLower() == song.Title.ToLower() && s.Artist.ToLower() == song.Artist.ToLower()))
|
|
{
|
|
songs.Add(song);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
songs.Add(song);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
if (debug && i > debugLimit)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void SyncSongProperties(List<Song> songs, List<Song> sourceList, Action<Song> propertySetter)
|
|
{
|
|
if (sourceList != null && sourceList.Count > 0)
|
|
{
|
|
sourceList.ForEach(s =>
|
|
{
|
|
if (s != null)
|
|
{
|
|
var found = songs.Find(ls => s.Path.ToLower() == ls.Path.ToLower());
|
|
if (found != null)
|
|
{
|
|
propertySetter(found);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
private static Dictionary<string, List<Song>> FindDuplicateSongs(List<Song> songs, Func<Song, bool> filter = null)
|
|
{
|
|
Dictionary<string, List<Song>> dupes = new Dictionary<string, List<Song>>();
|
|
int i = 0;
|
|
|
|
foreach (var localsong in songs)
|
|
{
|
|
i++;
|
|
Console.WriteLine(string.Format("Checking for {0:000000}/{1}) {2} - {3}", i, songs.Count, localsong.Artist, localsong.Title));
|
|
|
|
if (!string.IsNullOrEmpty(localsong.Artist) && !string.IsNullOrEmpty(localsong.Title))
|
|
{
|
|
if (filter != null && !filter(localsong))
|
|
continue;
|
|
|
|
string key = localsong.Artist + " - " + localsong.Title;
|
|
if (!dupes.ContainsKey(key))
|
|
{
|
|
var dsongs = songs.Where(s => s.Title.ToLower() == localsong.Title.ToLower() && s.Artist.ToLower() == localsong.Artist.ToLower());
|
|
if (dsongs.Count() > 1)
|
|
{
|
|
List<Song> d = new List<Song>();
|
|
d.AddRange(dsongs.ToList());
|
|
dupes.Add(key, d);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return dupes;
|
|
}
|
|
|
|
static void Main(string[] args)
|
|
{
|
|
if (args.Count() == 3)
|
|
{
|
|
DeleteSongs(args);
|
|
}
|
|
else
|
|
{
|
|
CrawlSongs(args);
|
|
}
|
|
}
|
|
|
|
private static void CrawlSongs(string[] args)
|
|
{
|
|
//string [] test = { "mbrucedogs", "z://" };
|
|
//args = test;
|
|
var debug = false;
|
|
ValidateArgs(args, 2, "usage: songcrawler partyid songspath");
|
|
if (args.Length != 2) return;
|
|
|
|
string controller = args[0];
|
|
string songpath = args[1];
|
|
FireSharp.FirebaseClient client = CreateFirebaseClient();
|
|
string songsPath = GetControllerPath(controller, "songs");
|
|
string favoritesPath = GetControllerPath(controller, "favorites");
|
|
string disabledPath = GetControllerPath(controller, "disabled");
|
|
Console.WriteLine("Loading current library");
|
|
|
|
List<Song> songs = new List<Song>();
|
|
List<Song> disabled = LoadFirebaseData<Song>(client, disabledPath);
|
|
List<Song> favorited = LoadFirebaseData<Song>(client, favoritesPath);
|
|
|
|
client.Set(songsPath, songs);
|
|
|
|
List<string> files = GetAllMusicFiles(songpath);
|
|
|
|
ProcessFilesToSongs(files, songs, debug: debug, debugLimit: 1000);
|
|
|
|
//sync all favorite, history, disabled
|
|
SyncSongProperties(songs, favorited, song => song.Favorite = true);
|
|
SyncSongProperties(songs, disabled, song => song.Disabled = true);
|
|
client.Set(songsPath, songs);
|
|
|
|
//string test = string.Format("controllers/{0}/testsongs", controller);
|
|
//Dictionary<string, Song> testSongs = new Dictionary<string, Song>();
|
|
//foreach (Song s in songs)
|
|
//{
|
|
// testSongs[s.Guid] = s;
|
|
//}
|
|
//client.Set(test, testSongs);
|
|
|
|
var created = songs.Select(s => new CreatedSong(File.GetCreationTime(s.Path), s)).ToList();
|
|
var first200 = created.Where(s => s.created != null).OrderByDescending(s => s.created).Take(200);
|
|
var added = first200.Select(s => new PathOnly(path: s.song.Path)).ToList();
|
|
string newSongs = GetControllerPath(controller, "newSongs");
|
|
client.Set(newSongs, added);
|
|
|
|
}
|
|
private class PathOnly
|
|
{
|
|
[JsonProperty("path")]
|
|
public String Path { get; set; }
|
|
public PathOnly(string path)
|
|
{
|
|
this.Path = path;
|
|
}
|
|
}
|
|
|
|
private static List<Song> convertToList(dynamic json)
|
|
{
|
|
dynamic data = JsonConvert.DeserializeObject<dynamic>(json);
|
|
var list = new List<Song>();
|
|
foreach (var itemDynamic in data)
|
|
{
|
|
var fjson = itemDynamic.Value.ToString();
|
|
var f = JsonConvert.DeserializeObject<Song>(fjson);
|
|
list.Add(f);
|
|
}
|
|
return list;
|
|
}
|
|
|
|
private static void DeleteSongs(string[] args)
|
|
{
|
|
ValidateArgs(args, 3, "usage: songcrawler partyid songspath delete");
|
|
if (args.Length != 3) return;
|
|
|
|
string controller = args[0];
|
|
string songpath = args[1];
|
|
FireSharp.FirebaseClient client = CreateFirebaseClient();
|
|
string firepath = GetControllerPath(controller, "songs");
|
|
Console.WriteLine("Deleting Songs ...");
|
|
List<Song> songs = new List<Song>();
|
|
client.Set(firepath, songs);
|
|
|
|
}
|
|
|
|
private static void fixNewSongs(FireSharp.FirebaseClient client, String controller)
|
|
{
|
|
string songsPath = GetControllerPath(controller, "songs");
|
|
string newSongsPath = GetControllerPath(controller, "newSongs");
|
|
|
|
List<Song> songs = client.Get(songsPath).ResultAs<List<Song>>();
|
|
List<Song> newSongs = client.Get(newSongsPath).ResultAs<List<Song>>();
|
|
|
|
List<Song> updated = new List<Song>();
|
|
foreach (Song n in newSongs){
|
|
var found = songs.First(s => s.Path == n.Path);
|
|
if(found != null)
|
|
{
|
|
updated.Add(found);
|
|
}
|
|
}
|
|
client.Set(newSongsPath, updated);
|
|
}
|
|
|
|
private static void fixHistory(FireSharp.FirebaseClient client, String controller)
|
|
{
|
|
string historyPath = GetControllerPath(controller, "history");
|
|
List<History> history = null;
|
|
try
|
|
{
|
|
history = client.Get(historyPath).ResultAs<List<History>>();
|
|
}
|
|
catch
|
|
{
|
|
dynamic data = JsonConvert.DeserializeObject<dynamic>(client.Get(historyPath).Body);
|
|
history = new List<History>();
|
|
foreach (var itemDynamic in data)
|
|
{
|
|
var fjson = itemDynamic.Value.ToString();
|
|
var f = JsonConvert.DeserializeObject<History>(fjson);
|
|
history.Add(f);
|
|
}
|
|
}
|
|
client.Set(historyPath, history);
|
|
}
|
|
public class CreatedSong
|
|
{
|
|
public DateTime created { get; set; }
|
|
public Song song { get; set; }
|
|
public CreatedSong(DateTime created, Song song)
|
|
{
|
|
this.song = song;
|
|
this.created = created;
|
|
}
|
|
}
|
|
|
|
|
|
private static void CrawlNoDupeSongs(string[] args)
|
|
{
|
|
ValidateArgs(args, 2, "usage: songcrawler partyid songspath");
|
|
if (args.Length != 2) return;
|
|
|
|
string controller = args[0];
|
|
string songpath = args[1];
|
|
FireSharp.FirebaseClient client = CreateFirebaseClient();
|
|
string firepath = GetControllerPath(controller, "songs");
|
|
Console.WriteLine("Loading current library");
|
|
List<Song> songs = new List<Song>();
|
|
|
|
List<string> files = GetAllMusicFiles(songpath);
|
|
|
|
ProcessFilesToSongs(files, songs, checkDuplicates: true);
|
|
Console.WriteLine(string.Format("{0:000000}/{1} unique songs", songs.Count(), files.Count()));
|
|
client.Set(firepath, songs);
|
|
}
|
|
|
|
private static void FindDuplicates(string[] args)
|
|
{
|
|
string songpath = @"D:\KaraokeData\Karaoke"; // args[0];
|
|
List<Song> songs = songs = new List<Song>();
|
|
|
|
List<string> files = GetAllMusicFiles(songpath);
|
|
|
|
ProcessFilesToSongs(files, songs);
|
|
|
|
var dupes = FindDuplicateSongs(songs);
|
|
File.WriteAllText(@"D:\dupliates.json", JsonConvert.SerializeObject(dupes.OrderBy(o => o.Key)));
|
|
}
|
|
private static void DisableDuplicates(string[] args)
|
|
{
|
|
ValidateArgs(args, 1, "usage: songcrawler partyid songspath");
|
|
if (args.Length != 1) return;
|
|
|
|
string controller = args[0];
|
|
FireSharp.FirebaseClient client = CreateFirebaseClient();
|
|
string firepath = GetControllerPath(controller, "songs");
|
|
Console.WriteLine("Loading current library");
|
|
List<Song> songs = client.Get(firepath).ResultAs<List<Song>>();
|
|
|
|
var dupes = FindDuplicateSongs(songs, song => !song.Disabled && song.Path.Contains(".mp4"));
|
|
|
|
// Disable duplicate songs (keep the first one, disable the rest)
|
|
foreach (var duplicateGroup in dupes.Values)
|
|
{
|
|
for (int i = 1; i < duplicateGroup.Count; i++) // Skip first one, disable the rest
|
|
{
|
|
duplicateGroup[i].Disabled = true;
|
|
}
|
|
}
|
|
client.Set(firepath, songs);
|
|
}
|
|
|
|
private static Song MakeSong(string filepath)
|
|
{
|
|
|
|
Song song = null;
|
|
var ext = Path.GetExtension(filepath).ToLower();
|
|
switch (ext)
|
|
{
|
|
case ".mp3":
|
|
case ".mp4":
|
|
song = ReadId3(filepath);
|
|
break;
|
|
case ".zip":
|
|
song = ReadId3FromZip(filepath);
|
|
break;
|
|
}
|
|
CheckTitle(song);
|
|
song.Path = filepath;
|
|
return song;
|
|
}
|
|
|
|
private static string[] FindFiles(string ext, string path)
|
|
{
|
|
Console.Write(string.Format("\rscanning {0} for {1} - ", path, ext));
|
|
string[] files = Directory.GetFiles(path, "*." + ext, SearchOption.AllDirectories);
|
|
Console.WriteLine(string.Format("{0} found", files.Length));
|
|
return files;
|
|
}
|
|
|
|
private static void CheckTitle(Song song)
|
|
{
|
|
if(string.IsNullOrEmpty(song.Title))
|
|
{
|
|
string file = Path.GetFileNameWithoutExtension(song.Path);
|
|
string[] parts = file.Split('-');
|
|
if (parts.Length == 1)
|
|
{
|
|
song.Title = parts[0].Trim();
|
|
}
|
|
else if (parts.Length > 1)
|
|
{
|
|
song.Artist = parts[parts.Length - 2].Trim();
|
|
song.Title = parts[parts.Length - 1].Trim();
|
|
}
|
|
}
|
|
}
|
|
|
|
private static Song ReadId3FromZip(string zippath)
|
|
{
|
|
ZipFile.ExtractToDirectory(zippath, "c:\\temp");
|
|
string filepath = Directory.GetFiles("c:\\temp", "*.mp3")[0];
|
|
Song song = ReadId3(filepath);
|
|
foreach (string file in Directory.GetFiles("c:\\temp")) File.Delete(file);
|
|
return song;
|
|
}
|
|
|
|
static Song ReadId3(string path)
|
|
{
|
|
Song song = new Song();
|
|
TagLib.File tagFile;
|
|
try
|
|
{
|
|
tagFile = TagLib.File.Create(path);
|
|
song.Title = tagFile.Tag.Title.Trim();
|
|
song.Artist = tagFile.Tag.FirstPerformer.Trim();
|
|
}
|
|
catch
|
|
{
|
|
// do nothing;
|
|
}
|
|
song.Path = path;
|
|
return song;
|
|
}
|
|
}
|
|
}
|