diff --git a/README.md b/README.md index 8247b21..06f45c2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ Inspired by [Last.fm](https://last.fm), [Maloja](https://github.com/krateng/maloja), and [Lastfmstats.com](https://www.lastfmstats.com). - **Z**uletzt is written with [**Z**ig](https://github.com/ziglang/zig) and [Jetzig](https://github.com/jetzig-framework/jetzig) as a means of learning the language, reintroducing myself to programming, and combining the functionality of the aforementioned inspirations. @@ -12,6 +11,35 @@ Zuletzt means "last" in German. Licensed under MIT. +## Usage +Zuletzt allows uploads of Scrobbles at the `/upload` page, where you +can import Scrobbles from a Spotify data export, Last.FM data export (a `.json` +file from lastfmstats.com), or by providing a Last.FM username and connecting +to Last.FM directly. + +Zuletzt will not make any assumptions about the data, and only change metadata when asked to by a rule. Two albums will be considered the same if: +- They share the same title (case/diacritic sensitive) +- The album artist(s) are the same + +Zuletzt allows you to list multiple artists under an album using rules, but +does not try to automatically split artists along common delimiters. For +example, there's no way to know that "Mermaid Avenue" by "Billy Bragg, Wilco" +is performed by two artists, while "Ants From Up There" by "Black Country, New +Road" is performed by one artist. Thus, a rule needs to be made to tell Zuletzt +"Mermaid Avenue" is performed by "Billy Bragg" and "Wilco". + +Two songs will only be considered the same if: +- They share the same title (case/diacritic sensitive) +- They appear on the same album + +If two or more songs with the same spelling appear on an album, they are +necessarily grouped under the same name, as there is no way to differentiate +them (see "Once In Royal David's City" on Sufjan Stevens's "Songs For +Christmas", for example). Every artist that performs on those songs with +receive attribution for the combined song. + +If two artists have the same name, they are necessarily listed as the same artist, but can be separated with a rule, or after the fact, with a disambiguation string. + ## To-Do List: - [ ] Entity statistics - [x] See all artists under "/artists" @@ -60,6 +88,7 @@ Licensed under MIT. - [ ] Rank songs - [ ] Custom statistics[^7] - [ ] "Playlists"[^8] +- [ ] First launch setup [^1]: I do not intend to exactly replicate all the statistics Lastfmstats.com provides, but I would at least like to give the user the option to see those kinds of statistics, or generate them themselves (see 7). diff --git a/src/app/database/Schema.zig b/src/app/database/Schema.zig index eeec984..0a30a19 100644 --- a/src/app/database/Schema.zig +++ b/src/app/database/Schema.zig @@ -4,7 +4,7 @@ pub const Album = jetquery.Model( @This(), "albums", struct { - id: i32, + id: i64, name: []const u8, length: ?f32, created_at: jetquery.DateTime, @@ -22,9 +22,9 @@ pub const Albumsong = jetquery.Model( @This(), "albumsongs", struct { - id: i32, - song_id: i32, - album_id: i32, + id: i64, + song_id: i64, + album_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -32,9 +32,7 @@ pub const Albumsong = jetquery.Model( .relations = .{ .song = jetquery.belongsTo(.Song, .{}), .album = jetquery.belongsTo(.Album, .{}), - .scrobbles = jetquery.hasMany(.Scrobble, .{ - .foreign_key = "albumsong", - }), + .scrobbles = jetquery.hasMany(.Scrobble, .{ .foreign_key = "albumsong" }), .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), }, }, @@ -44,9 +42,9 @@ pub const Albumsongsartist = jetquery.Model( @This(), "albumsongsartists", struct { - id: i32, - albumsong_id: i32, - artist_id: i32, + id: i64, + albumsong_id: i64, + artist_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -62,9 +60,9 @@ pub const Artistalbum = jetquery.Model( @This(), "artistalbums", struct { - id: i32, - album_id: i32, - artist_id: i32, + id: i64, + album_id: i64, + artist_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -80,7 +78,7 @@ pub const Artist = jetquery.Model( @This(), "artists", struct { - id: i32, + id: i64, name: []const u8, disambiguation: ?[]const u8, created_at: jetquery.DateTime, @@ -90,6 +88,7 @@ pub const Artist = jetquery.Model( .relations = .{ .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), .artistalbums = jetquery.hasMany(.Artistalbum, .{}), + .artistsongs = jetquery.hasMany(.Artistsong, .{}), }, }, ); @@ -98,17 +97,15 @@ pub const Scrobble = jetquery.Model( @This(), "scrobbles", struct { - id: i32, - albumsong: i32, + id: i64, + albumsong: i64, datetime: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .albumsong = jetquery.belongsTo(.Albumsong, .{ - .foreign_key = "albumsong", - }), + .albumsong = jetquery.belongsTo(.Albumsong, .{ .foreign_key = "albumsong" }), }, }, ); @@ -117,7 +114,7 @@ pub const Song = jetquery.Model( @This(), "songs", struct { - id: i32, + id: i64, name: []const u8, length: ?f32, hidden: bool, @@ -127,6 +124,25 @@ pub const Song = jetquery.Model( .{ .relations = .{ .albumsongs = jetquery.hasMany(.Albumsong, .{}), + .artistsongs = jetquery.hasMany(.Artistsong, .{}), + }, + }, +); + +pub const Artistsong = jetquery.Model( + @This(), + "artistsongs", + struct { + id: i64, + artist_id: i64, + song_id: i64, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .artist = jetquery.belongsTo(.Artist, .{}), + .song = jetquery.belongsTo(.Song, .{}), }, }, ); diff --git a/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig index 9a52b6b..e8ae1d6 100644 --- a/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig +++ b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "songs", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("length", .float, .{ .optional = true }), t.column("hidden", .boolean, .{}), diff --git a/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig index d706cfe..86b6184 100644 --- a/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig +++ b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albums", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("length", .float, .{ .optional = true }), t.timestamps(.{}), diff --git a/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig index 4381a5b..96c3063 100644 --- a/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig +++ b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albumsongs", &.{ - t.primaryKey("id", .{}), - t.column("song_id", .integer, .{ .reference = .{ "songs", "id" } }), - t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("song_id", .bigint, .{ .reference = .{ "songs", "id" } }), + t.column("album_id", .bigint, .{ .reference = .{ "albums", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index 088a43c..c429e19 100644 --- a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -6,8 +6,8 @@ pub fn up(repo: anytype) !void { try repo.createTable( "scrobbles", &.{ - t.primaryKey("id", .{}), - t.column("albumsong", .integer, .{ .reference = .{ "albumsongs", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("albumsong", .bigint, .{ .reference = .{ "albumsongs", "id" } }), t.column("datetime", .datetime, .{}), t.timestamps(.{}), }, diff --git a/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig index ed0d3d4..97c5bfe 100644 --- a/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig +++ b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "artists", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("disambiguation", .string, .{ .optional = true }), t.timestamps(.{}), diff --git a/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig index 55c7d84..3355196 100644 --- a/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig +++ b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albumsongsartists", &.{ - t.primaryKey("id", .{}), - t.column("albumsong_id", .integer, .{ .reference = .{ "albumsongs", "id" } }), - t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("albumsong_id", .bigint, .{ .reference = .{ "albumsongs", "id" } }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig index 76bac1e..3c3ea7f 100644 --- a/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig +++ b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "artistalbums", &.{ - t.primaryKey("id", .{}), - t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), - t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("album_id", .bigint, .{ .reference = .{ "albums", "id" } }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig b/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig new file mode 100644 index 0000000..7d0a6c1 --- /dev/null +++ b/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig @@ -0,0 +1,20 @@ +const std = @import("std"); +const jetquery = @import("jetquery"); +const t = jetquery.schema.table; + +pub fn up(repo: anytype) !void { + try repo.createTable( + "artistsongs", + &.{ + t.primaryKey("id", .{ .type = .bigint }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), + t.column("song_id", .bigint, .{ .reference = .{ "songs", "id" } }), + t.timestamps(.{}), + }, + .{}, + ); +} + +pub fn down(repo: anytype) !void { + try repo.dropTable("artistsongs", .{}); +} diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index dcf7f1f..ea743b9 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -20,12 +20,13 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig // Probably want to include artist name here, but not sure how to yet - const track_artist_count = item.getT(.array, "artists_track").?.count(); - const album_artist_count = item.getT(.array, "artists_album").?.count(); - var track_artist_name_buffer = try allocator.alloc([]const u8, track_artist_count); - var album_artist_name_buffer = try allocator.alloc([]const u8, album_artist_count); - var track_artist_id_buffer = try allocator.alloc(i32, track_artist_count); - var album_artist_id_buffer = try allocator.alloc(i32, album_artist_count); + const track_artists = item.getT(.array, "artists_track").?.items(); + const album_artists = item.getT(.array, "artists_album").?.items(); + + var track_artist_name_buffer = try allocator.alloc([]const u8, track_artists.len); + var album_artist_name_buffer = try allocator.alloc([]const u8, album_artists.len); + var track_artist_id_buffer = try allocator.alloc(i64, track_artists.len); + var album_artist_id_buffer = try allocator.alloc(i64, album_artists.len); const scrobble: Data.Scrobble = .{ .track = item.getT(.string, "track").?, @@ -35,95 +36,99 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig .date = @as(i64, @truncate(item.getT(.integer, "date").?)), }; - var id_prehash = std.ArrayList(u8).init(allocator); + var album_hash_string = std.ArrayList(u8).init(allocator); + var track_hash_string = std.ArrayList(u8).init(allocator); - for (item.getT(.array, "artists_track").?.items(), 0..track_artist_count) |artist, i| { + // I theoretically don't need this for loop + for (track_artists, 0..track_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); track_artist_name_buffer[i] = artist_name; - track_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + track_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); } - for (item.getT(.array, "artists_album").?.items(), 0..album_artist_count) |artist, i| { + for (album_artists, 0..album_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; - album_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); - try id_prehash.appendSlice(artist_name); + album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + try album_hash_string.appendSlice(artist_name); } - try id_prehash.appendSlice(scrobble.album); - const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); - try id_prehash.appendSlice(scrobble.track); - const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); + try album_hash_string.appendSlice(scrobble.album); + try track_hash_string.appendSlice(scrobble.album); + const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(album_hash_string.items))); + try track_hash_string.appendSlice(scrobble.track); + const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(track_hash_string.items))); - var albumsong = try jetzig.database.Query(.Albumsong) - .findBy(.{ .album_id = album_id, .song_id = song_id }) + var albumsong_id = try jetzig.database.Query(.Albumsong) + .find(album_hash ^ track_hash) .select(.{.id}).execute(env.repo); - var ins_album = try jetzig.database.Query(.Album) - .find(album_id) + var album_id = try jetzig.database.Query(.Album) + .find(album_hash) .select(.{.id}).execute(env.repo); - for (track_artist_name_buffer, track_artist_id_buffer) |artist_name, artist_id| { - var ins_artist = try jetzig.database.Query(.Artist) - .find(artist_id) + for (track_artist_name_buffer, track_artist_id_buffer) |scrobble_track_artist, track_artist_hash| { + var artist_id = try jetzig.database.Query(.Artist) + .find(track_artist_hash) .select(.{.id}).execute(env.repo); - if (ins_artist == null) - ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + if (artist_id == null) + artist_id = try jetzig.database.Query(.Artist) + .insert(.{ .id = track_artist_hash, .name = scrobble_track_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); - if (albumsong == null) { - var ins_song = try jetzig.database.Query(.Song) - .find(song_id) + if (albumsong_id == null) { + var track_id = try jetzig.database.Query(.Song) + .find(track_hash) .select(.{.id}).execute(env.repo); - if (ins_song == null) - ins_song = try jetzig.database.Query(.Song) - .insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }) + if (track_id == null) + track_id = try jetzig.database.Query(.Song) + .insert(.{ .id = track_hash, .name = scrobble.track, .length = null, .hidden = false }) .returning(.{.id}).execute(env.repo); - if (ins_album == null) - ins_album = try jetzig.database.Query(.Album) - .insert(.{ .id = album_id, .name = scrobble.album, .length = null }) + if (album_id == null) + album_id = try jetzig.database.Query(.Album) + .insert(.{ .id = album_hash, .name = scrobble.album, .length = null }) .returning(.{.id}).execute(env.repo); - albumsong = try jetzig.database.Query(.Albumsong) - .insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }) + albumsong_id = try jetzig.database.Query(.Albumsong) + .insert(.{ .song_id = track_id.?.id, .album_id = album_id.?.id }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } else { const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) - .findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }) + .findBy(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }) .select(.{.id}).execute(env.repo); if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } } - for (album_artist_name_buffer, album_artist_id_buffer) |artist_name, artist_id| { - const ins_artistalbum = try jetzig.database.Query(.Artistalbum) - .findBy(.{ .album_id = ins_album.?.id, .artist_id = artist_id }) + + for (album_artist_name_buffer, album_artist_id_buffer) |scrobble_album_artist, album_artist_hash| { + const artistalbum_id = try jetzig.database.Query(.Artistalbum) + .findBy(.{ .album_id = album_id.?.id, .artist_id = album_artist_hash }) .select(.{.id}).execute(env.repo); - if (ins_artistalbum == null) { - var ins_artist = try jetzig.database.Query(.Artist) - .find(artist_id) + if (artistalbum_id == null) { + var artist_id = try jetzig.database.Query(.Artist) + .find(album_artist_hash) .select(.{.id}).execute(env.repo); - if (ins_artist == null) - ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + if (artist_id == null) + artist_id = try jetzig.database.Query(.Artist) + .insert(.{ .id = artist_id, .name = scrobble_album_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Artistalbum) - .insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .album_id = album_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } } try jetzig.database.Query(.Scrobble) - .insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); + .insert(.{ .albumsong = albumsong_id.?.id, .datetime = scrobble.date }).execute(env.repo); } } } diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 713e6d6..03046b3 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -20,7 +20,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id}); try root.put("album", album); - const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_items), .{id}); + const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .get_songs), .{id}); try root.put("songs", songs); const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id}); diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 1f7ff85..3ec8eba 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -21,7 +21,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id}); try root.put("artist", artist); - const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_items), .{id}); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .get_albums), .{id}); try root.put("albums", albums); const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id}); diff --git a/src/app/views/partials/_newtable.zmpl b/src/app/views/partials/_newtable.zmpl index 6746a2a..f9dc618 100644 --- a/src/app/views/partials/_newtable.zmpl +++ b/src/app/views/partials/_newtable.zmpl @@ -1,5 +1,6 @@ @args T: type, table_data: *ZmplValue, columns: T +