623 lines
23 KiB
C#
623 lines
23 KiB
C#
using FireSharp.Config;
|
|
using FireSharp.Interfaces;
|
|
using FireSharp.Response;
|
|
using Herse.Models;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using System.IO.Compression;
|
|
using System.Configuration;
|
|
using Newtonsoft.Json;
|
|
using System.Security.Cryptography;
|
|
|
|
namespace SongCrawler
|
|
{
|
|
/// <summary>
|
|
/// Main program class for the SongCrawler application.
|
|
/// Handles crawling, indexing, and managing karaoke songs from file systems and Firebase.
|
|
/// </summary>
|
|
class Program
|
|
{
|
|
#region Properties and Configuration
|
|
|
|
/// <summary>
|
|
/// Controller identifier from command line arguments
|
|
/// </summary>
|
|
private static string Controller { get; set; }
|
|
|
|
/// <summary>
|
|
/// Path to the songs folder from command line arguments
|
|
/// </summary>
|
|
private static string SongsFolderPath { get; set; }
|
|
|
|
/// <summary>
|
|
/// Original command line arguments array
|
|
/// </summary>
|
|
private static string[] Args { get; set; }
|
|
|
|
/// <summary>
|
|
/// Shared Firebase client instance
|
|
/// </summary>
|
|
private static FireSharp.FirebaseClient FirebaseClient { get; set; }
|
|
|
|
#endregion
|
|
|
|
#region Main Entry Point
|
|
|
|
/// <summary>
|
|
/// Main entry point for the application.
|
|
/// Initializes properties from command line arguments and routes to appropriate functionality.
|
|
/// </summary>
|
|
/// <param name="args">Command line arguments: [controller] [songspath] [optional: delete]</param>
|
|
static void Main(string[] args)
|
|
{
|
|
InitializeFromArgs(args);
|
|
|
|
if (args.Count() == 3)
|
|
{
|
|
DeleteSongs();
|
|
}
|
|
else
|
|
{
|
|
CrawlSongs();
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Initialization and Setup
|
|
|
|
/// <summary>
|
|
/// Initializes application properties from command line arguments.
|
|
/// Sets up Controller, SongsFolderPath, Args, and FirebaseClient.
|
|
/// </summary>
|
|
/// <param name="args">Command line arguments array</param>
|
|
private static void InitializeFromArgs(string[] args)
|
|
{
|
|
Args = args;
|
|
if (args.Length > 0)
|
|
{
|
|
Controller = args[0];
|
|
}
|
|
if (args.Length > 1)
|
|
{
|
|
SongsFolderPath = args[1];
|
|
}
|
|
FirebaseClient = CreateFirebaseClient();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates and configures a Firebase client using settings from App.config.
|
|
/// </summary>
|
|
/// <returns>Configured Firebase client instance</returns>
|
|
private static FireSharp.FirebaseClient CreateFirebaseClient()
|
|
{
|
|
IFirebaseConfig config = new FirebaseConfig
|
|
{
|
|
AuthSecret = ConfigurationManager.AppSettings["Firebase.Secret"],
|
|
BasePath = ConfigurationManager.AppSettings["Firebase.Path"]
|
|
};
|
|
return new FireSharp.FirebaseClient(config);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Validates command line arguments against expected length.
|
|
/// </summary>
|
|
/// <param name="args">Arguments array to validate</param>
|
|
/// <param name="expectedLength">Expected number of arguments</param>
|
|
/// <param name="usage">Usage message to display if validation fails</param>
|
|
private static void ValidateArgs(string[] args, int expectedLength, string usage)
|
|
{
|
|
if (args.Length != expectedLength)
|
|
{
|
|
Console.WriteLine(usage);
|
|
return;
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Firebase Utilities
|
|
|
|
/// <summary>
|
|
/// Generates Firebase path for a specific controller and data type.
|
|
/// </summary>
|
|
/// <param name="controller">Controller identifier</param>
|
|
/// <param name="pathType">Type of data (songs, favorites, disabled, etc.)</param>
|
|
/// <returns>Formatted Firebase path</returns>
|
|
private static string GetControllerPath(string controller, string pathType)
|
|
{
|
|
return string.Format("controllers/{0}/{1}", controller, pathType);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Loads data from Firebase with fallback handling for different response formats.
|
|
/// </summary>
|
|
/// <typeparam name="T">Type of data to load</typeparam>
|
|
/// <param name="client">Firebase client instance</param>
|
|
/// <param name="path">Firebase path to load from</param>
|
|
/// <returns>List of loaded objects</returns>
|
|
private static List<T> LoadFirebaseData<T>(FireSharp.FirebaseClient client, string path) where T : class
|
|
{
|
|
try
|
|
{
|
|
return client.Get(path).ResultAs<List<T>>();
|
|
}
|
|
catch
|
|
{
|
|
return convertToList<T>(client.Get(path).Body);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Converts dynamic JSON response to strongly-typed list.
|
|
/// Handles Firebase responses that don't deserialize directly to List<T>.
|
|
/// </summary>
|
|
/// <typeparam name="T">Type to convert to</typeparam>
|
|
/// <param name="json">Dynamic JSON response</param>
|
|
/// <returns>List of converted objects</returns>
|
|
private static List<T> convertToList<T>(dynamic json) where T : class
|
|
{
|
|
dynamic data = JsonConvert.DeserializeObject<dynamic>(json);
|
|
var list = new List<T>();
|
|
foreach (var itemDynamic in data)
|
|
{
|
|
var fjson = itemDynamic.Value.ToString();
|
|
var f = JsonConvert.DeserializeObject<T>(fjson);
|
|
list.Add(f);
|
|
}
|
|
return list;
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region File System Operations
|
|
|
|
/// <summary>
|
|
/// Scans directory for all supported music file types (mp3, mp4, zip).
|
|
/// </summary>
|
|
/// <param name="songpath">Directory path to scan</param>
|
|
/// <returns>List of all music file paths found</returns>
|
|
private static List<string> GetAllMusicFiles(string songpath)
|
|
{
|
|
List<string> files = new List<string>();
|
|
files.AddRange(FindFiles("mp4", songpath));
|
|
files.AddRange(FindFiles("mp3", songpath));
|
|
files.AddRange(FindFiles("zip", songpath));
|
|
return files;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Finds all files with specified extension in directory and subdirectories.
|
|
/// </summary>
|
|
/// <param name="ext">File extension to search for</param>
|
|
/// <param name="path">Directory path to search</param>
|
|
/// <returns>Array of file paths found</returns>
|
|
private static string[] FindFiles(string ext, string path)
|
|
{
|
|
Console.Write(string.Format("\rscanning {0} for {1} - ", path, ext));
|
|
string[] files = Directory.GetFiles(path, "*." + ext, SearchOption.AllDirectories);
|
|
Console.WriteLine(string.Format("{0} found", files.Length));
|
|
return files;
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Song Processing
|
|
|
|
/// <summary>
|
|
/// Processes a list of files and converts them to Song objects.
|
|
/// Handles duplicate checking, debug limits, and error handling.
|
|
/// </summary>
|
|
/// <param name="files">List of file paths to process</param>
|
|
/// <param name="songs">List to add processed songs to</param>
|
|
/// <param name="checkDuplicates">Whether to check for duplicate songs</param>
|
|
/// <param name="debug">Whether to run in debug mode (limited processing)</param>
|
|
/// <param name="debugLimit">Maximum number of files to process in debug mode</param>
|
|
private static void ProcessFilesToSongs(List<string> files, List<Song> songs, bool checkDuplicates = false, bool debug = false, int debugLimit = 1000)
|
|
{
|
|
Song song = null;
|
|
int i = 0;
|
|
foreach (string filepath in files)
|
|
{
|
|
i++;
|
|
try
|
|
{
|
|
song = MakeSong(filepath);
|
|
Console.WriteLine(string.Format("{0:000000}/{1} - {2} - {3}", i, files.Count, song.Artist, song.Title));
|
|
|
|
if (checkDuplicates)
|
|
{
|
|
if (!songs.Any(s => s.Title.ToLower() == song.Title.ToLower() && s.Artist.ToLower() == song.Artist.ToLower()))
|
|
{
|
|
songs.Add(song);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
songs.Add(song);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
if (debug && i > debugLimit)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a Song object from a file path.
|
|
/// Handles different file types (mp3, mp4, zip) and extracts metadata.
|
|
/// </summary>
|
|
/// <param name="filepath">Path to the music file</param>
|
|
/// <returns>Song object with extracted metadata</returns>
|
|
private static Song MakeSong(string filepath)
|
|
{
|
|
Song song = null;
|
|
var ext = Path.GetExtension(filepath).ToLower();
|
|
switch (ext)
|
|
{
|
|
case ".mp3":
|
|
case ".mp4":
|
|
song = ReadId3(filepath);
|
|
break;
|
|
case ".zip":
|
|
song = ReadId3FromZip(filepath);
|
|
break;
|
|
}
|
|
CheckTitle(song);
|
|
song.Path = filepath;
|
|
return song;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads ID3 tags from audio files (mp3, mp4).
|
|
/// </summary>
|
|
/// <param name="path">Path to audio file</param>
|
|
/// <returns>Song object with extracted metadata</returns>
|
|
static Song ReadId3(string path)
|
|
{
|
|
Song song = new Song();
|
|
TagLib.File tagFile;
|
|
try
|
|
{
|
|
tagFile = TagLib.File.Create(path);
|
|
song.Title = tagFile.Tag.Title.Trim();
|
|
song.Artist = tagFile.Tag.FirstPerformer.Trim();
|
|
}
|
|
catch
|
|
{
|
|
// do nothing;
|
|
}
|
|
song.Path = path;
|
|
return song;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads ID3 tags from mp3 files inside zip archives.
|
|
/// </summary>
|
|
/// <param name="zippath">Path to zip file</param>
|
|
/// <returns>Song object with extracted metadata</returns>
|
|
private static Song ReadId3FromZip(string zippath)
|
|
{
|
|
ZipFile.ExtractToDirectory(zippath, "c:\\temp");
|
|
string filepath = Directory.GetFiles("c:\\temp", "*.mp3")[0];
|
|
Song song = ReadId3(filepath);
|
|
foreach (string file in Directory.GetFiles("c:\\temp")) File.Delete(file);
|
|
return song;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Extracts title and artist from filename if ID3 tags are missing.
|
|
/// Assumes format: "Artist - Title" or just "Title".
|
|
/// </summary>
|
|
/// <param name="song">Song object to update</param>
|
|
private static void CheckTitle(Song song)
|
|
{
|
|
if(string.IsNullOrEmpty(song.Title))
|
|
{
|
|
string file = Path.GetFileNameWithoutExtension(song.Path);
|
|
string[] parts = file.Split('-');
|
|
if (parts.Length == 1)
|
|
{
|
|
song.Title = parts[0].Trim();
|
|
}
|
|
else if (parts.Length > 1)
|
|
{
|
|
song.Artist = parts[parts.Length - 2].Trim();
|
|
song.Title = parts[parts.Length - 1].Trim();
|
|
}
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Song Management
|
|
|
|
/// <summary>
|
|
/// Synchronizes song properties from source lists using path matching.
|
|
/// Used to restore favorites and disabled states after re-indexing.
|
|
/// </summary>
|
|
/// <param name="songs">Target song list to update</param>
|
|
/// <param name="sourceList">Source list containing property values</param>
|
|
/// <param name="propertySetter">Action to set the property on matching songs</param>
|
|
private static void SyncSongProperties(List<Song> songs, List<Song> sourceList, Action<Song> propertySetter)
|
|
{
|
|
if (sourceList != null && sourceList.Count > 0)
|
|
{
|
|
sourceList.ForEach(s =>
|
|
{
|
|
if (s != null)
|
|
{
|
|
var found = songs.Find(ls => s.Path.ToLower() == ls.Path.ToLower());
|
|
if (found != null)
|
|
{
|
|
propertySetter(found);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Finds duplicate songs based on artist and title matching.
|
|
/// </summary>
|
|
/// <param name="songs">List of songs to check for duplicates</param>
|
|
/// <param name="filter">Optional filter to apply before duplicate checking</param>
|
|
/// <returns>Dictionary of duplicate groups keyed by "Artist - Title"</returns>
|
|
private static Dictionary<string, List<Song>> FindDuplicateSongs(List<Song> songs, Func<Song, bool> filter = null)
|
|
{
|
|
Dictionary<string, List<Song>> dupes = new Dictionary<string, List<Song>>();
|
|
int i = 0;
|
|
|
|
foreach (var localsong in songs)
|
|
{
|
|
i++;
|
|
Console.WriteLine(string.Format("Checking for {0:000000}/{1}) {2} - {3}", i, songs.Count, localsong.Artist, localsong.Title));
|
|
|
|
if (!string.IsNullOrEmpty(localsong.Artist) && !string.IsNullOrEmpty(localsong.Title))
|
|
{
|
|
if (filter != null && !filter(localsong))
|
|
continue;
|
|
|
|
string key = localsong.Artist + " - " + localsong.Title;
|
|
if (!dupes.ContainsKey(key))
|
|
{
|
|
var dsongs = songs.Where(s => s.Title.ToLower() == localsong.Title.ToLower() && s.Artist.ToLower() == localsong.Artist.ToLower());
|
|
if (dsongs.Count() > 1)
|
|
{
|
|
List<Song> d = new List<Song>();
|
|
d.AddRange(dsongs.ToList());
|
|
dupes.Add(key, d);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return dupes;
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Core Operations
|
|
|
|
/// <summary>
|
|
/// Main crawling operation. Scans music files, creates song objects, and uploads to Firebase.
|
|
/// Maintains favorites and disabled states from previous runs.
|
|
/// </summary>
|
|
private static void CrawlSongs()
|
|
{
|
|
var debug = false;
|
|
ValidateArgs(Args, 2, "usage: songcrawler partyid songspath");
|
|
if (Args.Length != 2) return;
|
|
|
|
string songsPath = GetControllerPath(Controller, "songs");
|
|
string favoritesPath = GetControllerPath(Controller, "favorites");
|
|
string disabledPath = GetControllerPath(Controller, "disabled");
|
|
Console.WriteLine("Loading current library");
|
|
|
|
List<Song> songs = new List<Song>();
|
|
List<Song> disabled = LoadFirebaseData<Song>(FirebaseClient, disabledPath);
|
|
List<Song> favorited = LoadFirebaseData<Song>(FirebaseClient, favoritesPath);
|
|
|
|
FirebaseClient.Set(songsPath, songs);
|
|
|
|
List<string> files = GetAllMusicFiles(SongsFolderPath);
|
|
|
|
ProcessFilesToSongs(files, songs, debug: debug, debugLimit: 1000);
|
|
|
|
//sync all favorite, history, disabled
|
|
SyncSongProperties(songs, favorited, song => song.Favorite = true);
|
|
SyncSongProperties(songs, disabled, song => song.Disabled = true);
|
|
FirebaseClient.Set(songsPath, songs);
|
|
|
|
// Create newSongs list from most recently created files
|
|
var created = songs.Select(s => new CreatedSong(File.GetCreationTime(s.Path), s)).ToList();
|
|
var first200 = created.Where(s => s.created != null).OrderByDescending(s => s.created).Take(200);
|
|
var added = first200.Select(s => new PathOnly(path: s.song.Path)).ToList();
|
|
string newSongs = GetControllerPath(Controller, "newSongs");
|
|
FirebaseClient.Set(newSongs, added);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Crawls songs without allowing duplicates based on artist and title.
|
|
/// </summary>
|
|
private static void CrawlNoDupeSongs()
|
|
{
|
|
ValidateArgs(Args, 2, "usage: songcrawler partyid songspath");
|
|
if (Args.Length != 2) return;
|
|
|
|
string firepath = GetControllerPath(Controller, "songs");
|
|
Console.WriteLine("Loading current library");
|
|
List<Song> songs = new List<Song>();
|
|
|
|
List<string> files = GetAllMusicFiles(SongsFolderPath);
|
|
|
|
ProcessFilesToSongs(files, songs, checkDuplicates: true);
|
|
Console.WriteLine(string.Format("{0:000000}/{1} unique songs", songs.Count(), files.Count()));
|
|
FirebaseClient.Set(firepath, songs);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Deletes all songs for the specified controller.
|
|
/// </summary>
|
|
private static void DeleteSongs()
|
|
{
|
|
ValidateArgs(Args, 3, "usage: songcrawler partyid songspath delete");
|
|
if (Args.Length != 3) return;
|
|
|
|
string firepath = GetControllerPath(Controller, "songs");
|
|
Console.WriteLine("Deleting Songs ...");
|
|
List<Song> songs = new List<Song>();
|
|
FirebaseClient.Set(firepath, songs);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Maintenance Operations
|
|
|
|
/// <summary>
|
|
/// Fixes newSongs list by replacing path-only entries with full song objects.
|
|
/// </summary>
|
|
private static void fixNewSongs()
|
|
{
|
|
string songsPath = GetControllerPath(Controller, "songs");
|
|
string newSongsPath = GetControllerPath(Controller, "newSongs");
|
|
|
|
List<Song> songs = FirebaseClient.Get(songsPath).ResultAs<List<Song>>();
|
|
List<Song> newSongs = FirebaseClient.Get(newSongsPath).ResultAs<List<Song>>();
|
|
|
|
List<Song> updated = new List<Song>();
|
|
foreach (Song n in newSongs){
|
|
var found = songs.First(s => s.Path == n.Path);
|
|
if(found != null)
|
|
{
|
|
updated.Add(found);
|
|
}
|
|
}
|
|
FirebaseClient.Set(newSongsPath, updated);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Fixes history list by ensuring proper JSON structure.
|
|
/// </summary>
|
|
private static void fixHistory()
|
|
{
|
|
string historyPath = GetControllerPath(Controller, "history");
|
|
List<History> history = null;
|
|
try
|
|
{
|
|
history = FirebaseClient.Get(historyPath).ResultAs<List<History>>();
|
|
}
|
|
catch
|
|
{
|
|
dynamic data = JsonConvert.DeserializeObject<dynamic>(FirebaseClient.Get(historyPath).Body);
|
|
history = new List<History>();
|
|
foreach (var itemDynamic in data)
|
|
{
|
|
var fjson = itemDynamic.Value.ToString();
|
|
var f = JsonConvert.DeserializeObject<History>(fjson);
|
|
history.Add(f);
|
|
}
|
|
}
|
|
FirebaseClient.Set(historyPath, history);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Finds and disables duplicate songs, keeping only the first occurrence.
|
|
/// Only processes mp4 files that are not already disabled.
|
|
/// </summary>
|
|
private static void DisableDuplicates()
|
|
{
|
|
ValidateArgs(Args, 1, "usage: songcrawler partyid songspath");
|
|
if (Args.Length != 1) return;
|
|
|
|
string firepath = GetControllerPath(Controller, "songs");
|
|
Console.WriteLine("Loading current library");
|
|
List<Song> songs = FirebaseClient.Get(firepath).ResultAs<List<Song>>();
|
|
|
|
var dupes = FindDuplicateSongs(songs, song => !song.Disabled && song.Path.Contains(".mp4"));
|
|
|
|
// Disable duplicate songs (keep the first one, disable the rest)
|
|
foreach (var duplicateGroup in dupes.Values)
|
|
{
|
|
for (int i = 1; i < duplicateGroup.Count; i++) // Skip first one, disable the rest
|
|
{
|
|
duplicateGroup[i].Disabled = true;
|
|
}
|
|
}
|
|
FirebaseClient.Set(firepath, songs);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Finds duplicates in a local directory and exports to JSON file.
|
|
/// </summary>
|
|
/// <param name="args">Command line arguments (not used, hardcoded path)</param>
|
|
private static void FindDuplicates(string[] args)
|
|
{
|
|
string songpath = @"D:\KaraokeData\Karaoke"; // args[0];
|
|
List<Song> songs = new List<Song>();
|
|
|
|
List<string> files = GetAllMusicFiles(songpath);
|
|
|
|
ProcessFilesToSongs(files, songs);
|
|
|
|
var dupes = FindDuplicateSongs(songs);
|
|
File.WriteAllText(@"D:\dupliates.json", JsonConvert.SerializeObject(dupes.OrderBy(o => o.Key)));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Utility Classes
|
|
|
|
/// <summary>
|
|
/// Represents a song with its creation timestamp for sorting.
|
|
/// </summary>
|
|
public class CreatedSong
|
|
{
|
|
public DateTime created { get; set; }
|
|
public Song song { get; set; }
|
|
public CreatedSong(DateTime created, Song song)
|
|
{
|
|
this.song = song;
|
|
this.created = created;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lightweight class for storing only file paths in newSongs list.
|
|
/// </summary>
|
|
private class PathOnly
|
|
{
|
|
[JsonProperty("path")]
|
|
public String Path { get; set; }
|
|
public PathOnly(string path)
|
|
{
|
|
this.Path = path;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Generates a consistent GUID from a string input using MD5 hashing.
|
|
/// </summary>
|
|
/// <param name="input">String to generate GUID from</param>
|
|
/// <returns>Consistent GUID for the input string</returns>
|
|
private static Guid GuidFromString(string input)
|
|
{
|
|
using (MD5 md5 = MD5.Create())
|
|
{
|
|
byte[] hash = md5.ComputeHash(Encoding.Default.GetBytes(input));
|
|
return new Guid(hash);
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
}
|