From 0537ef7db211ec3c7b693d65e1cd549cb3f5e5ee Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Apr 2025 10:44:28 -0400 Subject: [PATCH 001/103] she QUERY on my DATA so she's BASEd --- config/database.zig | 2 +- .../2025-02-17_22-38-45_create_ratings.zig | 21 ------------------- ...025-02-19_18-03-51_create_albumartists.zig | 20 ------------------ ...025-02-19_18-04-22_create_songsartists.zig | 20 ------------------ .../2025-02-19_18-46-48_create_albumsongs.zig | 20 ------------------ ...-02-21_14-24-31_create_scrobbleartists.zig | 20 ------------------ ...g => 2025-04-07_14-31-14_create_songs.zig} | 0 ... => 2025-04-07_14-31-45_create_albums.zig} | 0 ...2025-04-07_14-34-39_create_albumsongs.zig} | 8 +++---- ... 2025-04-07_14-35-53_create_scrobbles.zig} | 5 ++--- ...=> 2025-04-07_14-38-02_create_artists.zig} | 2 +- ...-07_14-39-09_create_albumsongsartists.zig} | 7 ++++--- ...25-04-07_14-40-17_create_artistalbums.zig} | 7 ++++--- 13 files changed, 16 insertions(+), 116 deletions(-) delete mode 100644 src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig delete mode 100644 src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig delete mode 100644 src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig delete mode 100644 src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig delete mode 100644 src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig rename src/app/database/migrations/{2025-02-17_22-38-47_create_songs.zig => 2025-04-07_14-31-14_create_songs.zig} (100%) rename src/app/database/migrations/{2025-02-17_22-38-40_create_albums.zig => 2025-04-07_14-31-45_create_albums.zig} (100%) rename src/app/database/migrations/{2025-02-17_22-38-41_create_aliases.zig => 2025-04-07_14-34-39_create_albumsongs.zig} (57%) rename src/app/database/migrations/{2025-02-17_22-38-46_create_scrobbles.zig => 2025-04-07_14-35-53_create_scrobbles.zig} (72%) rename src/app/database/migrations/{2025-02-17_22-38-42_create_artists.zig => 2025-04-07_14-38-02_create_artists.zig} (85%) rename src/app/database/migrations/{2025-02-17_22-38-44_create_mastersongs.zig => 2025-04-07_14-39-09_create_albumsongsartists.zig} (54%) rename src/app/database/migrations/{2025-02-17_22-38-43_create_masteralbums.zig => 2025-04-07_14-40-17_create_artistalbums.zig} (56%) diff --git a/config/database.zig b/config/database.zig index 361129d..4042ff0 100644 --- a/config/database.zig +++ b/config/database.zig @@ -15,7 +15,7 @@ pub const database = .{ .port = 5432, .username = "postgres", .password = "postgres", - .database = "zuletzt_dev", + .database = "zuletzt_rsql", .pool_size = 16, }, diff --git a/src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig b/src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig deleted file mode 100644 index d7ac939..0000000 --- a/src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig +++ /dev/null @@ -1,21 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "ratings", - &.{ - t.primaryKey("id", .{}), - t.column("reference_id", .integer, .{}), - t.column("score", .float, .{}), - t.column("date", .datetime, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("ratings", .{}); -} diff --git a/src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig b/src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig deleted file mode 100644 index b0e4f54..0000000 --- a/src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Albumartists", - &.{ - t.primaryKey("id", .{}), - t.column("album_id", .integer, .{}), - t.column("artist_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Albumartists", .{}); -} diff --git a/src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig b/src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig deleted file mode 100644 index a509d7a..0000000 --- a/src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Songartists", - &.{ - t.primaryKey("id", .{}), - t.column("song_id", .integer, .{}), - t.column("artist_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Songartists", .{}); -} diff --git a/src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig b/src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig deleted file mode 100644 index 1865c2e..0000000 --- a/src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Albumsongs", - &.{ - t.primaryKey("id", .{}), - t.column("album_id", .integer, .{}), - t.column("song_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Albumsongs", .{}); -} diff --git a/src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig b/src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig deleted file mode 100644 index 2125a87..0000000 --- a/src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Scrobbleartists", - &.{ - t.primaryKey("id", .{}), - t.column("scrobble_id", .integer, .{}), - t.column("artist_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Scrobbleartists", .{}); -} diff --git a/src/app/database/migrations/2025-02-17_22-38-47_create_songs.zig b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig similarity index 100% rename from src/app/database/migrations/2025-02-17_22-38-47_create_songs.zig rename to src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig diff --git a/src/app/database/migrations/2025-02-17_22-38-40_create_albums.zig b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig similarity index 100% rename from src/app/database/migrations/2025-02-17_22-38-40_create_albums.zig rename to src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig diff --git a/src/app/database/migrations/2025-02-17_22-38-41_create_aliases.zig b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig similarity index 57% rename from src/app/database/migrations/2025-02-17_22-38-41_create_aliases.zig rename to src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig index 11cbb70..4381a5b 100644 --- a/src/app/database/migrations/2025-02-17_22-38-41_create_aliases.zig +++ b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig @@ -4,11 +4,11 @@ const t = jetquery.schema.table; pub fn up(repo: anytype) !void { try repo.createTable( - "aliases", + "albumsongs", &.{ t.primaryKey("id", .{}), - t.column("reference_id", .integer, .{}), - t.column("alias", .string, .{}), + t.column("song_id", .integer, .{ .reference = .{ "songs", "id" } }), + t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), t.timestamps(.{}), }, .{}, @@ -16,5 +16,5 @@ pub fn up(repo: anytype) !void { } pub fn down(repo: anytype) !void { - try repo.dropTable("aliases", .{}); + try repo.dropTable("albumsongs", .{}); } diff --git a/src/app/database/migrations/2025-02-17_22-38-46_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig similarity index 72% rename from src/app/database/migrations/2025-02-17_22-38-46_create_scrobbles.zig rename to src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index 9764d99..088a43c 100644 --- a/src/app/database/migrations/2025-02-17_22-38-46_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -7,9 +7,8 @@ pub fn up(repo: anytype) !void { "scrobbles", &.{ t.primaryKey("id", .{}), - t.column("song_id", .integer, .{}), - t.column("album_id", .integer, .{}), - t.column("date", .datetime, .{}), + t.column("albumsong", .integer, .{ .reference = .{ "albumsongs", "id" } }), + t.column("datetime", .datetime, .{}), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-02-17_22-38-42_create_artists.zig b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig similarity index 85% rename from src/app/database/migrations/2025-02-17_22-38-42_create_artists.zig rename to src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig index 2c92de4..ed0d3d4 100644 --- a/src/app/database/migrations/2025-02-17_22-38-42_create_artists.zig +++ b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig @@ -8,7 +8,7 @@ pub fn up(repo: anytype) !void { &.{ t.primaryKey("id", .{}), t.column("name", .string, .{}), - t.column("descriptive_string", .string, .{}), + t.column("disambiguation", .string, .{ .optional = true }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-02-17_22-38-44_create_mastersongs.zig b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig similarity index 54% rename from src/app/database/migrations/2025-02-17_22-38-44_create_mastersongs.zig rename to src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig index 050a467..55c7d84 100644 --- a/src/app/database/migrations/2025-02-17_22-38-44_create_mastersongs.zig +++ b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig @@ -4,10 +4,11 @@ const t = jetquery.schema.table; pub fn up(repo: anytype) !void { try repo.createTable( - "mastersongs", + "albumsongsartists", &.{ t.primaryKey("id", .{}), - t.column("name", .string, .{}), + t.column("albumsong_id", .integer, .{ .reference = .{ "albumsongs", "id" } }), + t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, @@ -15,5 +16,5 @@ pub fn up(repo: anytype) !void { } pub fn down(repo: anytype) !void { - try repo.dropTable("mastersongs", .{}); + try repo.dropTable("albumsongsartists", .{}); } diff --git a/src/app/database/migrations/2025-02-17_22-38-43_create_masteralbums.zig b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig similarity index 56% rename from src/app/database/migrations/2025-02-17_22-38-43_create_masteralbums.zig rename to src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig index 69e82de..76bac1e 100644 --- a/src/app/database/migrations/2025-02-17_22-38-43_create_masteralbums.zig +++ b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig @@ -4,10 +4,11 @@ const t = jetquery.schema.table; pub fn up(repo: anytype) !void { try repo.createTable( - "masteralbums", + "artistalbums", &.{ t.primaryKey("id", .{}), - t.column("name", .string, .{}), + t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), + t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, @@ -15,5 +16,5 @@ pub fn up(repo: anytype) !void { } pub fn down(repo: anytype) !void { - try repo.dropTable("masteralbums", .{}); + try repo.dropTable("artistalbums", .{}); } From 64038079d8d49f2354c3bac3603a1f3c3dfc2b0b Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Apr 2025 15:44:52 -0400 Subject: [PATCH 002/103] Update process_scrobbles.zig to fit new db --- src/app/jobs/process_scrobbles.zig | 65 +++++++++++++----------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 1314465..6481d2e 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -54,55 +54,48 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig // artists have an album of the same name, then // the IDs also depend on the hash of the artist // name. As far as I can tell, this is only an - // issue for Weezer. + // issue for Weezer and Peter Gabriel, but their + // albums go by unique names anyways. // Artist: Artist hash. If two artists have the same name, - // then a descriptive string can be provided to + // then a disambiguating string can be provided to // differentiate after the fact, or in a rule. //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); //const song_id = (song_hash ^ artist_hash ^ album_hash); - // Inserts - const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }); - const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .descriptive_string = "" }); - const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }); + var albumsong_id = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); + var ins_album_id = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); - // Checks - const album_check = try jetzig.database.Query(.Album).find(album_id).execute(env.repo); - const artist_check = try jetzig.database.Query(.Artist).find(artist_id).execute(env.repo); - const song_check = try jetzig.database.Query(.Song).find(song_id).execute(env.repo); + var ins_artist_id = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); + if (ins_artist_id == null) ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); - // I think there must be a better way to do this next part - // There are very few situations where artist_check is null - // but song_check/album is not. Also yes, the order of these - // checks is weird, I didn't put a lot of thought into it - var associative_table_flags: [3]bool = [3]bool{ true, true, true }; + if (albumsong_id == null) { + const ins_song_id: []const u8 = + jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo) catch + jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); - if (album_check == null) { - try env.repo.execute(album_insert); - try jetzig.database.Query(.Albumartist).insert(.{ .album_id = album_id, .artist_id = artist_id }).execute(env.repo); - associative_table_flags[0] = false; - try jetzig.database.Query(.Albumsong).insert(.{ .album_id = album_id, .song_id = song_id }).execute(env.repo); - associative_table_flags[1] = false; + if (ins_album_id == null) ins_album_id = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); + + albumsong_id = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song_id, .album_id = ins_album_id }).execute(env.repo); + + try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); + try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); + } else { + const ins_albumsongartist = try jetzig.database.Query(.Albumsongartist).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); + if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); + + const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); + if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); } - if (artist_check == null) { - try env.repo.execute(artist_insert); - if (associative_table_flags[0]) try jetzig.database.Query(.Albumartist).insert(.{ .album_id = album_id, .artist_id = artist_id }).execute(env.repo); - try jetzig.database.Query(.Songartist).insert(.{ .song_id = song_id, .artist_id = artist_id }).execute(env.repo); - associative_table_flags[2] = false; - } + //if (ins_artist_id == null) { + // ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); + // try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); + // try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); + //} - if (song_check == null) { - try env.repo.execute(song_insert); - if (associative_table_flags[1]) try jetzig.database.Query(.Albumsong).insert(.{ .album_id = album_id, .song_id = song_id }).execute(env.repo); - if (associative_table_flags[2]) try jetzig.database.Query(.Songartist).insert(.{ .song_id = song_id, .artist_id = artist_id }).execute(env.repo); - } - - const scr_id = try jetzig.database.Query(.Scrobble).insert(.{ .song_id = song_id, .album_id = album_id, .date = scrobble.date }).returning(.{.id}).execute(env.repo); - defer env.repo.free(scr_id); - try jetzig.database.Query(.Scrobbleartist).insert(.{ .scrobble_id = scr_id.?.id, .artist_id = artist_id }).execute(env.repo); + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong_id = albumsong_id, .date = scrobble.date }).execute(env.repo); } } From 3f69183b6f082ade92a17b58ee779e98d8d218c8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:23:12 -0400 Subject: [PATCH 003/103] Create new Schema from migrations --- src/app/database/Schema.zig | 211 ++++++++++-------------------------- 1 file changed, 59 insertions(+), 152 deletions(-) diff --git a/src/app/database/Schema.zig b/src/app/database/Schema.zig index 231c283..eeec984 100644 --- a/src/app/database/Schema.zig +++ b/src/app/database/Schema.zig @@ -12,27 +12,68 @@ pub const Album = jetquery.Model( }, .{ .relations = .{ - .masteralbum = jetquery.belongsTo(.Masteralbum, .{}), - .scrobbles = jetquery.hasMany(.Scrobble, .{}), - .ratings = jetquery.hasMany(.Rating, .{}), - .aliases = jetquery.hasMany(.Alias, .{}), .albumsongs = jetquery.hasMany(.Albumsong, .{}), - .albumartists = jetquery.hasMany(.Albumartist, .{}), + .artistalbums = jetquery.hasMany(.Artistalbum, .{}), }, }, ); -pub const Alias = jetquery.Model( +pub const Albumsong = jetquery.Model( @This(), - "aliases", + "albumsongs", struct { id: i32, - reference_id: i32, - alias: []const u8, + song_id: i32, + album_id: i32, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, - .{}, + .{ + .relations = .{ + .song = jetquery.belongsTo(.Song, .{}), + .album = jetquery.belongsTo(.Album, .{}), + .scrobbles = jetquery.hasMany(.Scrobble, .{ + .foreign_key = "albumsong", + }), + .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), + }, + }, +); + +pub const Albumsongsartist = jetquery.Model( + @This(), + "albumsongsartists", + struct { + id: i32, + albumsong_id: i32, + artist_id: i32, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .albumsong = jetquery.belongsTo(.Albumsong, .{}), + .artist = jetquery.belongsTo(.Artist, .{}), + }, + }, +); + +pub const Artistalbum = jetquery.Model( + @This(), + "artistalbums", + struct { + id: i32, + album_id: i32, + artist_id: i32, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .album = jetquery.belongsTo(.Album, .{}), + .artist = jetquery.belongsTo(.Artist, .{}), + }, + }, ); pub const Artist = jetquery.Model( @@ -41,70 +82,14 @@ pub const Artist = jetquery.Model( struct { id: i32, name: []const u8, - descriptive_string: []const u8, + disambiguation: ?[]const u8, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .scrobbleartists = jetquery.hasMany(.Scrobbleartist, .{}), - .aliases = jetquery.hasMany(.Alias, .{}), - .artistsongs = jetquery.hasMany(.Songartist, .{}), - .mastersongs = jetquery.hasMany(.Mastersong, .{}), - .artistalbums = jetquery.hasMany(.Albumartist, .{}), - .masteralbums = jetquery.hasMany(.Masteralbum, .{}), - }, - }, -); - -pub const Masteralbum = jetquery.Model( - @This(), - "masteralbums", - struct { - id: i32, - name: []const u8, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .albums = jetquery.hasMany(.Album, .{}), - }, - }, -); - -pub const Mastersong = jetquery.Model( - @This(), - "mastersongs", - struct { - id: i32, - name: []const u8, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .songs = jetquery.hasMany(.Song, .{}), - }, - }, -); - -pub const Rating = jetquery.Model( - @This(), - "ratings", - struct { - id: i32, - reference_id: i32, - score: f32, - date: jetquery.DateTime, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .song = jetquery.belongsTo(.Song, .{}), - .album = jetquery.belongsTo(.Album, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), + .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), + .artistalbums = jetquery.hasMany(.Artistalbum, .{}), }, }, ); @@ -114,17 +99,16 @@ pub const Scrobble = jetquery.Model( "scrobbles", struct { id: i32, - song_id: i32, - album_id: i32, - date: jetquery.DateTime, + albumsong: i32, + datetime: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .song = jetquery.belongsTo(.Song, .{}), - .album = jetquery.belongsTo(.Album, .{}), - .scrobbleartists = jetquery.hasMany(.Scrobbleartist, .{}), + .albumsong = jetquery.belongsTo(.Albumsong, .{ + .foreign_key = "albumsong", + }), }, }, ); @@ -142,84 +126,7 @@ pub const Song = jetquery.Model( }, .{ .relations = .{ - .mastersong = jetquery.belongsTo(.Mastersong, .{}), - .scrobbles = jetquery.hasMany(.Scrobble, .{}), - .ratings = jetquery.hasMany(.Rating, .{}), - .aliases = jetquery.hasMany(.Alias, .{}), - .songartists = jetquery.hasMany(.Songartist, .{}), .albumsongs = jetquery.hasMany(.Albumsong, .{}), }, }, ); - -pub const Albumartist = jetquery.Model( - @This(), - "Albumartists", - struct { - id: i32, - album_id: i32, - artist_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .album = jetquery.belongsTo(.Album, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), - }, - }, -); - -pub const Songartist = jetquery.Model( - @This(), - "Songartists", - struct { - id: i32, - song_id: i32, - artist_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .song = jetquery.belongsTo(.Song, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), - }, - }, -); - -pub const Albumsong = jetquery.Model( - @This(), - "Albumsongs", - struct { - id: i32, - album_id: i32, - song_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .album = jetquery.belongsTo(.Album, .{}), - .song = jetquery.belongsTo(.Song, .{}), - }, - }, -); - -pub const Scrobbleartist = jetquery.Model( - @This(), - "Scrobbleartists", - struct { - id: i32, - scrobble_id: i32, - artist_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .scrobble = jetquery.belongsTo(.Scrobble, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), - }, - }, -); From 09d4453665b654f5be6d428ed798b72113a9442a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:24:16 -0400 Subject: [PATCH 004/103] Fix various issues with process_scrobbles I use the ins_ variables an unnecessary amount I think, I need to take a closer look at it, and give them better names --- src/app/jobs/process_scrobbles.zig | 36 ++++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 6481d2e..70c2976 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -64,29 +64,31 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); //const song_id = (song_hash ^ artist_hash ^ album_hash); - var albumsong_id = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); - var ins_album_id = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); + var albumsong = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); + var ins_album = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); - var ins_artist_id = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); - if (ins_artist_id == null) ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); + var ins_artist = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); + if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); - if (albumsong_id == null) { - const ins_song_id: []const u8 = - jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo) catch - jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); + if (albumsong == null) { + var ins_song = try jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); + if (ins_song == null) ins_song = try jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo); - if (ins_album_id == null) ins_album_id = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); + if (ins_album == null) { + ins_album = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); + // I think there's still technically a bug here when you have a different artist but I'm not sure + try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist.?.id, .album_id = ins_album.?.id }).execute(env.repo); + } - albumsong_id = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song_id, .album_id = ins_album_id }).execute(env.repo); + albumsong = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }).returning(.{.id}).execute(env.repo); - try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); - try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); + try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } else { - const ins_albumsongartist = try jetzig.database.Query(.Albumsongartist).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); - if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); + const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist).findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); + if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); - const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); - if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); + const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); + if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } //if (ins_artist_id == null) { @@ -95,7 +97,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig // try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); //} - try jetzig.database.Query(.Scrobble).insert(.{ .albumsong_id = albumsong_id, .date = scrobble.date }).execute(env.repo); + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); } } From 27358fe2179e6b8ad04e9ae3423d5b6380f2a6ed Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:24:48 -0400 Subject: [PATCH 005/103] Implement db searches using raw sql --- src/app/views/albums.zig | 111 +++++++++++++++++++++++------------- src/app/views/artists.zig | 91 ++++++++++++++++++++--------- src/app/views/scrobbles.zig | 73 ++++++++++++++++-------- src/app/views/songs.zig | 60 +++++++++++++------ 4 files changed, 226 insertions(+), 109 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 634f414..9f363f5 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -5,60 +5,91 @@ const jetquery = @import("jetzig").jetquery; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var albums_view = try root.put("albums", .array); - const albums = try jetzig.database.Query(.Album) - .select(.{ .id, .name }) - .include(.albumartists, .{ .select = .{.artist_id} }) - .include(.scrobbles, .{ .select = .{.id} }) - .orderBy(.{ .name = .asc }) - .all(request.repo); - //const albums = try request.repo.all(query); + //const albums = try jetzig.database.Query(.Album) + // .select(.{ .id, .name }) + // .include(.albumartists, .{ .select = .{.artist_id} }) + // .include(.scrobbles, .{ .select = .{.id} }) + // .orderBy(.{ .name = .asc }) + // .all(request.repo); + ////const albums = try request.repo.all(query); - for (albums) |album| { + const query = + \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN albums ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + ; + + var albums_jq_result = try request.repo.executeSql(query, .{}); + defer albums_jq_result.deinit(); + + const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try albums_jq_result.postgresql.result.next()) |album_row| { + const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); var album_view = try albums_view.append(.object); - - var artist_infos = try album_view.put("artist_info", .array); - for (album.albumartists) |artist| { - var artist_info = try artist_infos.append(.object); - const artist_data = try jetzig.database.Query(.Artist) - .find(artist.artist_id) - .select(.{ .id, .name }) - .execute(request.repo); - try artist_info.put("name", artist_data.?.name); - try artist_info.put("id", artist_data.?.id); - } - try album_view.put("name", album.name); try album_view.put("url", album.id); - try album_view.put("scrobbles", (album.scrobbles).len); + try album_view.put("scrobbles", album.scrobbles); } return request.render(.ok); } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - const album = try jetzig.database.Query(.Album) - .find(id) - .select(.{ .id, .name }) - .execute(request.repo); var root = try request.data(.object); - try root.put("album", album.?.name); var songs_view = try root.put("songs", .array); - const query = jetzig.database.Query(.Albumsong) - .select(.{.id}) - .include(.song, .{ .select = .{ .name, .id } }) - .join(.inner, .album) - .where(.{ .album = .{ .id = id } }); + const album_name = try jetzig.database.Query(.Album).find(id).select(.{ .id, .name }).execute(request.repo); + _ = try root.put("album", album_name.?.name); - const songs = try request.repo.all(query); - for (songs) |song| { - const scrobbles = try jetzig.database.Query(.Scrobble) - .where(.{ .song_id = song.song.id }) - .count() - .execute(request.repo); + const query = + \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albumsongs.album_id = $1 + \\GROUP BY songs.id + \\ORDER BY scrobbles DESC + ; + + var songs_js_result = try request.repo.executeSql(query, .{id}); + defer songs_js_result.deinit(); + + const Song = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try songs_js_result.postgresql.result.next()) |song_row| { + const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); var song_view = try songs_view.append(.object); - try song_view.put("name", song.song.name); - try song_view.put("url", song.song.id); - try song_view.put("scrobbles", scrobbles); + try song_view.put("name", song.name); + try song_view.put("url", song.id); + try song_view.put("scrobbles", song.scrobbles); } + + //const album = try jetzig.database.Query(.Album) + // .find(id) + // .select(.{ .id, .name }) + // .execute(request.repo); + //var root = try request.data(.object); + //try root.put("album", album.?.name); + //var songs_view = try root.put("songs", .array); + //const query = jetzig.database.Query(.Albumsong) + // .select(.{.id}) + // .include(.song, .{ .select = .{ .name, .id } }) + // .join(.inner, .album) + // .where(.{ .album = .{ .id = id } }); + + //const songs = try request.repo.all(query); + //for (songs) |song| { + // const scrobbles = try jetzig.database.Query(.Scrobble) + // .where(.{ .song_id = song.song.id }) + // .count() + // .execute(request.repo); + // var song_view = try songs_view.append(.object); + // try song_view.put("name", song.song.name); + // try song_view.put("url", song.song.id); + // try song_view.put("scrobbles", scrobbles); + //} return request.render(.ok); } diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 78058a7..b8d6bc3 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -5,46 +5,85 @@ const jetquery = @import("jetzig").jetquery; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var artists_view = try root.put("artists", .array); - const artists = try jetzig.database.Query(.Artist) - .select(.{ .id, .name }) - .include(.scrobbleartists, .{ .select = .{.id} }) - .orderBy(.{ .name = .asc }) - .all(request.repo); - for (artists) |artist| { + + const query = + \\SELECT artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongsartists + \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id + \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY artists.id + \\ORDER BY scrobbles DESC; + ; + + var artists_jq_result = try request.repo.executeSql(query, .{}); + defer artists_jq_result.deinit(); + + const Artist = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try artists_jq_result.postgresql.result.next()) |artist_row| { + const artist = try artist_row.to(Artist, .{ .dupe = true, .allocator = request.allocator }); var artist_view = try artists_view.append(.object); try artist_view.put("name", artist.name); try artist_view.put("url", artist.id); - try artist_view.put("scrobbles", (artist.scrobbleartists).len); + try artist_view.put("scrobbles", artist.scrobbles); } return request.render(.ok); } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - const artist = try jetzig.database.Query(.Artist) - .find(id) - .select(.{ .id, .name }) - .execute(request.repo); var root = try request.data(.object); - try root.put("artist", artist.?.name); var albums_view = try root.put("albums", .array); - const query = jetzig.database.Query(.Albumartist) - .select(.{.id}) - .include(.album, .{ .select = .{ .name, .id } }) - .join(.inner, .artist) - .where(.{ .artist = .{ .id = id } }); + const artist_name = try jetzig.database.Query(.Artist).find(id).select(.{ .id, .name }).execute(request.repo); + _ = try root.put("artist", artist_name.?.name); + const query = + \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE artistalbums.artist_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + ; - const albums = try request.repo.all(query); - for (albums) |album| { - const scrobbles = try jetzig.database.Query(.Scrobble) - .where(.{ .album_id = album.album.id }) - .count() - .execute(request.repo); + var albums_jq_result = try request.repo.executeSql(query, .{id}); + defer albums_jq_result.deinit(); + + const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try albums_jq_result.postgresql.result.next()) |album_row| { + const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); var album_view = try albums_view.append(.object); - try album_view.put("name", album.album.name); - try album_view.put("url", album.album.id); - try album_view.put("scrobbles", scrobbles); + try album_view.put("name", album.name); + try album_view.put("url", album.id); + try album_view.put("scrobbles", album.scrobbles); } + //const artist = try jetzig.database.Query(.Artist) + // .find(id) + // .select(.{ .id, .name }) + // .execute(request.repo); + //var root = try request.data(.object); + //try root.put("artist", artist.?.name); + //var albums_view = try root.put("albums", .array); + //const query = jetzig.database.Query(.Albumartist) + // .select(.{.id}) + // .include(.album, .{ .select = .{ .name, .id } }) + // .join(.inner, .artist) + // .where(.{ .artist = .{ .id = id } }); + + //const albums = try request.repo.all(query); + //for (albums) |album| { + // const scrobbles = try jetzig.database.Query(.Scrobble) + // .where(.{ .album_id = album.album.id }) + // .count() + // .execute(request.repo); + // var album_view = try albums_view.append(.object); + // try album_view.put("name", album.album.name); + // try album_view.put("url", album.album.id); + // try album_view.put("scrobbles", scrobbles); + //} return request.render(.ok); } diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index ebda828..48e5f2b 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -4,33 +4,58 @@ const jetzig = @import("jetzig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var scrobbles_view = try root.put("scrobbles", .array); - const query = jetzig.database.Query(.Scrobble) - .select(.{ .id, .date }) - .include(.song, .{ .select = .{ .id, .name } }) - .include(.album, .{ .select = .{ .id, .name } }) - .include(.scrobbleartists, .{ .select = .{.artist_id} }) - .orderBy(.{ .date = .desc }); - const scrobbles = try request.repo.all(query); - for (scrobbles) |scrobble| { + //const query = jetzig.database.Query(.Scrobble) + // .select(.{ .id, .date }) + // .include(.song, .{ .select = .{ .id, .name } }) + // .include(.album, .{ .select = .{ .id, .name } }) + // .include(.scrobbleartists, .{ .select = .{.artist_id} }) + // .orderBy(.{ .date = .desc }); + //const scrobbles = try request.repo.all(query); + + const query = + \\SELECT songs.name, songs.id, albums.name, albums.id, scrobbles.datetime + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\ORDER BY scrobbles.datetime ASC + ; + + var scrobbles_js_result = try request.repo.executeSql(query, .{}); + defer scrobbles_js_result.deinit(); + + const Scrobble = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, date: i64 }; + + while (try scrobbles_js_result.postgresql.result.next()) |scrobble_row| { + const scrobble = try scrobble_row.to(Scrobble, .{ .dupe = true, .allocator = request.allocator }); var scrobble_view = try scrobbles_view.append(.object); - - var artist_infos = try scrobble_view.put("artist_info", .array); - for (scrobble.scrobbleartists) |artist| { - var artist_info = try artist_infos.append(.object); - const artist_data = try jetzig.database.Query(.Artist) - .find(artist.artist_id) - .select(.{ .id, .name }) - .execute(request.repo); - try artist_info.put("name", artist_data.?.name); - try artist_info.put("id", artist_data.?.id); - } - - try scrobble_view.put("song_name", scrobble.song.name); - try scrobble_view.put("song_id", scrobble.song.id); - try scrobble_view.put("album_name", scrobble.album.name); - try scrobble_view.put("album_id", scrobble.album.id); + try scrobble_view.put("song_name", scrobble.song_name); + try scrobble_view.put("song_id", scrobble.song_id); + try scrobble_view.put("album_name", scrobble.album_name); + try scrobble_view.put("album_id", scrobble.album_id); try scrobble_view.put("date", scrobble.date); } + + //for (scrobbles) |scrobble| { + // var scrobble_view = try scrobbles_view.append(.object); + + // var artist_infos = try scrobble_view.put("artist_info", .array); + // for (scrobble.scrobbleartists) |artist| { + // var artist_info = try artist_infos.append(.object); + // const artist_data = try jetzig.database.Query(.Artist) + // .find(artist.artist_id) + // .select(.{ .id, .name }) + // .execute(request.repo); + // try artist_info.put("name", artist_data.?.name); + // try artist_info.put("id", artist_data.?.id); + // } + + // try scrobble_view.put("song_name", scrobble.song.name); + // try scrobble_view.put("song_id", scrobble.song.id); + // try scrobble_view.put("album_name", scrobble.album.name); + // try scrobble_view.put("album_id", scrobble.album.id); + // try scrobble_view.put("date", scrobble.date); + //} return request.render(.ok); } diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 6c134b1..fe2bec4 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -4,30 +4,52 @@ const jetzig = @import("jetzig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var songs_view = try root.put("songs", .array); - const songs = try jetzig.database.Query(.Song) - .select(.{ .id, .name }) - .include(.songartists, .{ .select = .{.artist_id} }) - .include(.scrobbles, .{ .select = .{.id} }) - .orderBy(.{ .name = .asc }) - .all(request.repo); + //const songs = try jetzig.database.Query(.Song) + // .select(.{ .id, .name }) + // .include(.songartists, .{ .select = .{.artist_id} }) + // .include(.scrobbles, .{ .select = .{.id} }) + // .orderBy(.{ .name = .asc }) + // .all(request.repo); - for (songs) |song| { + const query = + \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY songs.id + \\ORDER BY scrobbles DESC + ; + + var songs_js_result = try request.repo.executeSql(query, .{}); + defer songs_js_result.deinit(); + + const Song = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try songs_js_result.postgresql.result.next()) |song_row| { + const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); var song_view = try songs_view.append(.object); - - var artist_infos = try song_view.put("artist_info", .array); - for (song.songartists) |artist| { - var artist_info = try artist_infos.append(.object); - const artist_data = try jetzig.database.Query(.Artist) - .find(artist.artist_id) - .select(.{ .id, .name }) - .execute(request.repo); - try artist_info.put("name", artist_data.?.name); - try artist_info.put("id", artist_data.?.id); - } try song_view.put("name", song.name); try song_view.put("url", song.id); - try song_view.put("scrobbles", (song.scrobbles).len); + try song_view.put("scrobbles", song.scrobbles); } + + //for (songs) |song| { + // var song_view = try songs_view.append(.object); + + // var artist_infos = try song_view.put("artist_info", .array); + // for (song.songartists) |artist| { + // var artist_info = try artist_infos.append(.object); + // const artist_data = try jetzig.database.Query(.Artist) + // .find(artist.artist_id) + // .select(.{ .id, .name }) + // .execute(request.repo); + // try artist_info.put("name", artist_data.?.name); + // try artist_info.put("id", artist_data.?.id); + // } + // try song_view.put("name", song.name); + // try song_view.put("url", song.id); + // try song_view.put("scrobbles", (song.scrobbles).len); + //} return request.render(.ok); } From 41ab0dc88844acf0b36b0f7e5f45c938878ec062 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:26:56 -0400 Subject: [PATCH 006/103] Remove artists column from views I kinda just didn't want to deal with it while implementing the raw sql. Bringing it back is my next priority, but I want to do the searching in a nice way, and I'm not sure how to do that yet --- src/app/views/albums/index.zmpl | 6 ------ src/app/views/scrobbles/index.zmpl | 6 ------ src/app/views/songs/index.zmpl | 6 ------ 3 files changed, 18 deletions(-) diff --git a/src/app/views/albums/index.zmpl b/src/app/views/albums/index.zmpl index 2c259a6..7de3b87 100644 --- a/src/app/views/albums/index.zmpl +++ b/src/app/views/albums/index.zmpl @@ -10,7 +10,6 @@ Name -Artist(s) Scrobbles @@ -18,11 +17,6 @@ @for (.albums) |album| { {{album.name}} - - @for (album.get("artist_info").?) |ai| { - {{ai.name}} - } - {{album.scrobbles}} } diff --git a/src/app/views/scrobbles/index.zmpl b/src/app/views/scrobbles/index.zmpl index 377f50f..26657ff 100644 --- a/src/app/views/scrobbles/index.zmpl +++ b/src/app/views/scrobbles/index.zmpl @@ -10,7 +10,6 @@ Song -Artist(s) Album Date @@ -19,11 +18,6 @@ @for (.scrobbles) |scrobble| { {{scrobble.song_name}} - - @for (scrobble.get("artist_info").?) |ai| { - {{ai.name}} - } - {{scrobble.album_name}} {{scrobble.date}} diff --git a/src/app/views/songs/index.zmpl b/src/app/views/songs/index.zmpl index a0a1128..85eaa25 100644 --- a/src/app/views/songs/index.zmpl +++ b/src/app/views/songs/index.zmpl @@ -10,7 +10,6 @@ Name -Artists(s) Scrobbles @@ -18,11 +17,6 @@ @for (.songs) |song| { {{song.name}} - - @for (song.get("artist_info").?) |ai| { - {{ai.name}} - } - {{song.scrobbles}} } From 2c4af0b37893d84cfd0061cc42f7d3d22be7d355 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 14:05:15 -0400 Subject: [PATCH 007/103] Include artist column for albums I'm convinced there's a better way of doing this, but this is all I can think of right now --- src/app/views/albums.zig | 18 +++++++++++++++++- src/app/views/albums/index.zmpl | 6 ++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 9f363f5..3fbb8ab 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -22,7 +22,10 @@ pub fn index(request: *jetzig.Request) !jetzig.View { \\ORDER BY scrobbles DESC ; - var albums_jq_result = try request.repo.executeSql(query, .{}); + var inter_conn = try request.repo.connect(); + defer inter_conn.release(); + + var albums_jq_result = try inter_conn.executeSql(query, .{}, null, request.repo); defer albums_jq_result.deinit(); const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; @@ -30,6 +33,19 @@ pub fn index(request: *jetzig.Request) !jetzig.View { while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); var album_view = try albums_view.append(.object); + var artist_infos = try album_view.put("artist_info", .array); + const artist_data = try jetzig.database.Query(.Artistalbum) + .select(.{.id}) + .where(.{ .album_id = album.id }) + .include(.artist, .{ .select = .{ .name, .id } }) + .all(request.repo); + + for (artist_data) |artist| { + var artist_info = try artist_infos.append(.object); + try artist_info.put("name", artist.artist.name); + try artist_info.put("url", artist.artist.id); + } + try album_view.put("name", album.name); try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); diff --git a/src/app/views/albums/index.zmpl b/src/app/views/albums/index.zmpl index 7de3b87..e5536eb 100644 --- a/src/app/views/albums/index.zmpl +++ b/src/app/views/albums/index.zmpl @@ -10,6 +10,7 @@ Name +Artist(s) Scrobbles @@ -17,6 +18,11 @@ @for (.albums) |album| { {{album.name}} + + @for (album.get("artist_info").?) |ai| { + {{ai.name}} + } + {{album.scrobbles}} } From 4d63844def269fc78f2d50bcac9b31e694c16257 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 15:05:44 -0400 Subject: [PATCH 008/103] Make artist retrieval apart of main query This feels bad or wrong somehow, but it do be working tho --- src/app/views/albums.zig | 45 +++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 3fbb8ab..2100051 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -14,41 +14,52 @@ pub fn index(request: *jetzig.Request) !jetzig.View { ////const albums = try request.repo.all(query); const query = - \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\GROUP BY albums.id + \\INNER JOIN artistalbums ON artistalbums.album_id = albums.id + \\INNER JOIN artists ON artists.id = artistalbums.artist_id + \\GROUP BY albums.id, artists.id \\ORDER BY scrobbles DESC ; - var inter_conn = try request.repo.connect(); - defer inter_conn.release(); + //var inter_conn = try request.repo.connect(); + //defer inter_conn.release(); - var albums_jq_result = try inter_conn.executeSql(query, .{}, null, request.repo); + var albums_jq_result = try request.repo.executeSql(query, .{}); defer albums_jq_result.deinit(); - const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; + const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; - while (try albums_jq_result.postgresql.result.next()) |album_row| { + var prev_album_id: ?i32 = null; + var prev_artist_infos = try root.put("test", .array); + + blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); + if (album.id == prev_album_id) { + var artist_info = try prev_artist_infos.append(.object); + try artist_info.put("name", album.artist_name); + try artist_info.put("url", album.artist_id); + continue :blk; + } var album_view = try albums_view.append(.object); var artist_infos = try album_view.put("artist_info", .array); - const artist_data = try jetzig.database.Query(.Artistalbum) - .select(.{.id}) - .where(.{ .album_id = album.id }) - .include(.artist, .{ .select = .{ .name, .id } }) - .all(request.repo); + //const artist_data = try jetzig.database.Query(.Artistalbum) + // .select(.{.id}) + // .where(.{ .album_id = album.id }) + // .include(.artist, .{ .select = .{ .name, .id } }) + // .all(request.repo); - for (artist_data) |artist| { - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", artist.artist.name); - try artist_info.put("url", artist.artist.id); - } + var artist_info = try artist_infos.append(.object); + try artist_info.put("name", album.artist_name); + try artist_info.put("url", album.artist_id); try album_view.put("name", album.name); try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); + prev_artist_infos = artist_infos; + prev_album_id = album.id; } return request.render(.ok); } From 387493d3c0c339613c2cf232620aaaeba4ec918d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 15:17:10 -0400 Subject: [PATCH 009/103] Change typedef of prev_artist_infos Feeling much better about my choices this time around --- src/app/views/albums.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 2100051..00b2338 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -33,12 +33,12 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; var prev_album_id: ?i32 = null; - var prev_artist_infos = try root.put("test", .array); + var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); if (album.id == prev_album_id) { - var artist_info = try prev_artist_infos.append(.object); + var artist_info = try prev_artist_infos.?.append(.object); try artist_info.put("name", album.artist_name); try artist_info.put("url", album.artist_id); continue :blk; From ff8cdabbf1e2b3b8706d71fb43a89e1b73f315b9 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 15:28:00 -0400 Subject: [PATCH 010/103] Cleanup --- src/app/views/albums.zig | 43 +--------------------------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 00b2338..57d2bb1 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -5,14 +5,6 @@ const jetquery = @import("jetzig").jetquery; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var albums_view = try root.put("albums", .array); - //const albums = try jetzig.database.Query(.Album) - // .select(.{ .id, .name }) - // .include(.albumartists, .{ .select = .{.artist_id} }) - // .include(.scrobbles, .{ .select = .{.id} }) - // .orderBy(.{ .name = .asc }) - // .all(request.repo); - ////const albums = try request.repo.all(query); - const query = \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs @@ -24,9 +16,6 @@ pub fn index(request: *jetzig.Request) !jetzig.View { \\ORDER BY scrobbles DESC ; - //var inter_conn = try request.repo.connect(); - //defer inter_conn.release(); - var albums_jq_result = try request.repo.executeSql(query, .{}); defer albums_jq_result.deinit(); @@ -45,12 +34,6 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } var album_view = try albums_view.append(.object); var artist_infos = try album_view.put("artist_info", .array); - //const artist_data = try jetzig.database.Query(.Artistalbum) - // .select(.{.id}) - // .where(.{ .album_id = album.id }) - // .include(.artist, .{ .select = .{ .name, .id } }) - // .all(request.repo); - var artist_info = try artist_infos.append(.object); try artist_info.put("name", album.artist_name); try artist_info.put("url", album.artist_id); @@ -58,6 +41,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { try album_view.put("name", album.name); try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); + prev_artist_infos = artist_infos; prev_album_id = album.id; } @@ -92,31 +76,6 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { try song_view.put("url", song.id); try song_view.put("scrobbles", song.scrobbles); } - - //const album = try jetzig.database.Query(.Album) - // .find(id) - // .select(.{ .id, .name }) - // .execute(request.repo); - //var root = try request.data(.object); - //try root.put("album", album.?.name); - //var songs_view = try root.put("songs", .array); - //const query = jetzig.database.Query(.Albumsong) - // .select(.{.id}) - // .include(.song, .{ .select = .{ .name, .id } }) - // .join(.inner, .album) - // .where(.{ .album = .{ .id = id } }); - - //const songs = try request.repo.all(query); - //for (songs) |song| { - // const scrobbles = try jetzig.database.Query(.Scrobble) - // .where(.{ .song_id = song.song.id }) - // .count() - // .execute(request.repo); - // var song_view = try songs_view.append(.object); - // try song_view.put("name", song.song.name); - // try song_view.put("url", song.song.id); - // try song_view.put("scrobbles", scrobbles); - //} return request.render(.ok); } From 18cdb48b5326b819783dcfbe73dcfab67bd6ab58 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 18 Apr 2025 21:29:00 -0400 Subject: [PATCH 011/103] Begin rules --- src/app/jobs/process_rule.zig | 32 +++++++++++ src/app/views/rules.zig | 21 ++++++- src/app/views/rules/index.zmpl | 102 ++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 src/app/jobs/process_rule.zig diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig new file mode 100644 index 0000000..fb0d78b --- /dev/null +++ b/src/app/jobs/process_rule.zig @@ -0,0 +1,32 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + _ = env; + //_ = params; + + const rule = try params.toJson(); + + //const rule = struct { + // name: []const u8 = params, + // conditions: []struct { + // match_on: []const u8, + // match_cond: []const u8, + // match_text: []const u8, + // }, + // actions: []struct { + // action: []const u8, + // action_cond: []const u8, + // action_text: []const u8, + // }, + //}; + + //var file = try std.fs.cwd().openFile("rules.json", .{}); + + //_ = try file.write(rule); + try std.fs.cwd().writeFile(.{ "rules.json", rule, .{} }); + + // Job execution code goes here. Add any code that you would like to run in the background. + //try env.logger.INFO("Running a job.", .{}); +} diff --git a/src/app/views/rules.zig b/src/app/views/rules.zig index a2a222e..bb8ea4b 100644 --- a/src/app/views/rules.zig +++ b/src/app/views/rules.zig @@ -20,6 +20,26 @@ pub fn edit(id: []const u8, request: *jetzig.Request) !jetzig.View { } pub fn post(request: *jetzig.Request) !jetzig.View { + const params = try request.params(); + + var job = try request.job("process_rule"); + + _ = try job.params.put("name", params.get("rule-title")); + + var conditionals = try job.params.put("conditionals", .array); + var cond0 = try conditionals.append(.object); + try cond0.put("match_on", params.get("match-on")); + try cond0.put("match_cond", params.get("match-cond")); + try cond0.put("match_txt", params.get("match-txt")); + + var actions = try job.params.put("actions", .array); + var act0 = try actions.append(.object); + try act0.put("action", params.get("action")); + try act0.put("action_on", params.get("action-on")); + try act0.put("action_txt", params.get("action-txt")); + + try job.schedule(); + return request.render(.created); } @@ -38,7 +58,6 @@ pub fn delete(id: []const u8, request: *jetzig.Request) !jetzig.View { return request.render(.ok); } - test "index" { var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); defer app.deinit(); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index 76457d0..4d15188 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -1,3 +1,99 @@ -
- Content goes here -
+ + + + + +@partial partials/header +

Rules

+Rules allow you change the default Scrobble import behavior based on provided criteria. +Add a rule below. +

+
+ + +
+If + + + + + + + +
+then + + +with + + +
+ +Current rules: + + \ No newline at end of file From 5383b69eb66c72aaf31a6ee6e1d057a708843d26 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 19 Apr 2025 15:01:30 -0400 Subject: [PATCH 012/103] Allow reading and writing rules.json I like the idea of letting the user write to a file themselves for rules, but I think this is going to significantly slow things down. Will probably switch to SQL table at some point. Also very hardcoded for my purposes. ALSO the code looks bad, I think there must be a better way... --- .gitignore | 3 +- src/app/jobs/process_rule.zig | 63 ++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 70b0239..76f5b26 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ static/ src/app/database/data.db-journal src/app/database/old_migrations/ src/lib -src/app/scripts/ \ No newline at end of file +src/app/scripts/ +rules.json \ No newline at end of file diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index fb0d78b..93d1279 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -2,31 +2,54 @@ const std = @import("std"); const jetzig = @import("jetzig"); pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { - _ = allocator; _ = env; //_ = params; - const rule = try params.toJson(); + const Rule = struct { + name: []const u8, + conditionals: []struct { + match_on: []const u8, + match_cond: []const u8, + match_txt: []const u8, + }, + actions: []struct { + action: []const u8, + action_on: []const u8, + action_txt: []const u8, + }, + }; - //const rule = struct { - // name: []const u8 = params, - // conditions: []struct { - // match_on: []const u8, - // match_cond: []const u8, - // match_text: []const u8, - // }, - // actions: []struct { - // action: []const u8, - // action_cond: []const u8, - // action_text: []const u8, - // }, - //}; + const Rules = struct { + rules: []Rule, + }; - //var file = try std.fs.cwd().openFile("rules.json", .{}); + const rule = try std.json.parseFromSliceLeaky(Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); - //_ = try file.write(rule); - try std.fs.cwd().writeFile(.{ "rules.json", rule, .{} }); + const file_read: std.fs.File = std.fs.cwd().openFile("rules.json", .{}) catch |read_err| switch (read_err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true, .exclusive = true }) catch |write_err| switch (write_err) { + error.PathAlreadyExists => unreachable, + else => { + std.log.debug("{any} while writing file", .{write_err}); + return; + }, + }, + else => { + std.log.debug("{any} while reading file", .{read_err}); + return; + }, + }; - // Job execution code goes here. Add any code that you would like to run in the background. - //try env.logger.INFO("Running a job.", .{}); + var rules = std.ArrayList(Rule).init(allocator); + const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); + if (file_content.len != 0) { + const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + try rules.appendSlice(content.rules); + } + try rules.append(rule); + file_read.close(); + + const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); + const out_rules = Rules{ .rules = rules.items }; + const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file_write.writeAll(out); } From baf9ef38a4c83fb4588d7c12f883cec294f1bd95 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 19 Apr 2025 15:36:51 -0400 Subject: [PATCH 013/103] Simplify file creation branch of process_rule.zig Still not quite where I want it, but definitely better than what it was --- src/app/jobs/process_rule.zig | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 93d1279..7cb1a71 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -20,18 +20,25 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }; const Rules = struct { - rules: []Rule, + rules: []const Rule, }; const rule = try std.json.parseFromSliceLeaky(Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); const file_read: std.fs.File = std.fs.cwd().openFile("rules.json", .{}) catch |read_err| switch (read_err) { - error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true, .exclusive = true }) catch |write_err| switch (write_err) { - error.PathAlreadyExists => unreachable, - else => { - std.log.debug("{any} while writing file", .{write_err}); - return; - }, + error.FileNotFound => { + const file = std.fs.cwd().createFile("rules.json", .{ .read = true, .exclusive = true }) catch |write_err| switch (write_err) { + error.PathAlreadyExists => unreachable, + else => { + std.log.debug("{any} while writing file", .{write_err}); + return; + }, + }; + const out_rules = Rules{ .rules = &[_]Rule{rule} }; + const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file.writeAll(out); + file.close(); + return; }, else => { std.log.debug("{any} while reading file", .{read_err}); @@ -40,16 +47,19 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }; var rules = std.ArrayList(Rule).init(allocator); + defer rules.deinit(); + const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); - if (file_content.len != 0) { - const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); - try rules.appendSlice(content.rules); - } + const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + try rules.appendSlice(content.rules); try rules.append(rule); file_read.close(); const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); const out_rules = Rules{ .rules = rules.items }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file_write.writeAll(out); + file_write.close(); + return; } From 445ca45fa96576fe4e8ada1729cb40fa570e074f Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 21 Apr 2025 00:17:16 -0400 Subject: [PATCH 014/103] Begin rule application The more I think about this, the more I think it's gonna be super slow and bad. There must bve a good way of doing this, but I'm not sure how... --- src/app/jobs/process_scrobbles.zig | 7 ++++++- src/apply_rule.zig | 11 +++++++++++ src/types.zig | 26 ++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 src/apply_rule.zig diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 70c2976..bbfc9d4 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -14,14 +14,19 @@ const lastfm = @import("../../types.zig").LastFM; // - logger: Logger attached to the same stream as the Jetzig server. // - environment: Enum of `{ production, development }`. pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { - _ = allocator; //_ = env; + const file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); + const file_content = try file.readToEndAlloc(allocator, 16_000_000); + + const rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { //const fixed_date: u32 = @as(u32, item.getT(.integer, "date").?); const scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; + for (rules) |rule| {} + // Make hashes //const album_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.album))); //const artist_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.artist))); diff --git a/src/apply_rule.zig b/src/apply_rule.zig new file mode 100644 index 0000000..871c996 --- /dev/null +++ b/src/apply_rule.zig @@ -0,0 +1,11 @@ +const Scrobble = @import("types").LastFMScrobble; +const Rules = @import("types").Rules; + +pub fn applyRule(scrobble: Scrobble, rules: Rules) !Scrobble { + var output_scrobble: Scrobble = scrobble; + for (rules) |rule| { + for (rule.conditionals) |cond| { + switch (cond.match_cond) {} + } + } +} diff --git a/src/types.zig b/src/types.zig index 55bca1c..00e063c 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,5 +1,3 @@ -const zeit = @import("zeit"); - pub const LastFMScrobble = struct { track: []const u8, artist: []const u8, @@ -36,3 +34,27 @@ pub const SpotifyScrobble = struct { offline_timestamp: u64, incognito_mode: ?bool, }; + +const Rule = struct { + name: []const u8, + conditionals: []struct { + match_on: MatchOn, + match_cond: enum { is, contains }, + match_txt: []const u8, + }, + actions: []struct { + action: []const u8, + action_on: enum { is, contains }, + action_txt: []const u8, + }, +}; + +const Rules = struct { + rules: []const Rule, +}; + +const MatchOn = enum { + artist, + album, + song, +}; From 87a2fe2d3479ace74d3d11439bd6ad5e9f184e80 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 21 Apr 2025 12:23:20 -0400 Subject: [PATCH 015/103] Complete preliminary find and replace rules Tested by replacing AJR with John Van Derwood. Need to test on albums and artists, as well as matching on one piece of metadata, and replacing another --- src/app/jobs/process_scrobbles.zig | 9 +++++--- src/app/views/rules/index.zmpl | 2 +- src/apply_rule.zig | 33 +++++++++++++++++++++++++----- src/types.zig | 12 +++++------ 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index bbfc9d4..06d7365 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -3,6 +3,9 @@ const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; const Scrobble = @import("../../types.zig").LastFMScrobble; const lastfm = @import("../../types.zig").LastFM; +//const Rules = @import("../../types.zig").Rules; +const Data = @import("../../types.zig"); +const rules = @import("../../apply_rule.zig"); // The `run` function for a job is invoked every time the job is processed by a queue worker // (or by the Jetzig server if the job is processed in-line). @@ -18,14 +21,14 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig const file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); const file_content = try file.readToEndAlloc(allocator, 16_000_000); - const rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + const rule_list = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { //const fixed_date: u32 = @as(u32, item.getT(.integer, "date").?); - const scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; + const pre_scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; - for (rules) |rule| {} + const scrobble = rules.applyScrobbleRule(pre_scrobble, rule_list); // Make hashes //const album_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.album))); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index 4d15188..b9a1b17 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -21,7 +21,7 @@ If @@ -87,7 +91,7 @@ then with diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 53f3b27..032da9d 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -3,6 +3,8 @@ const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; const ScrobbleTypes = @import("../../types.zig"); const zeit = @import("zeit"); +const rules = @import("../../apply_rule.zig"); +const Data = @import("../../types.zig"); pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { _ = data; @@ -31,6 +33,11 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; + const rule_file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); + defer rule_file.close(); + const file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); + const rule_list = try std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}); + // The only difference between a LastFM scrobble and a Spotify scrobble is the format. // I've made a branches for each, because doing it all in one made the readability terrible, // and formatting the date in particular was challenging. I could probably pull out the @@ -46,12 +53,14 @@ pub fn post(request: *jetzig.Request) !jetzig.View { if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; var value = try scrobbles_data.append(.object); + const formatted_scrobble = rules.applyScrobbleRule(scrobble, rule_list); + // This is so unnecessary, probably useful once I start doing Spotify integration though inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| { - try value.put(f.name, @as(f.type, @field(scrobble, f.name))); + try value.put(f.name, @as(f.type, @field(formatted_scrobble, f.name))); } // Note sure why this works for ZMPL, but not for jobs. - try scrobbles_view.append(scrobble); + try scrobbles_view.append(formatted_scrobble); } }, 1 => { @@ -88,7 +97,9 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } // Turn SpotifyScrobble into a LastFM scrobble - const formatted_scrobble: ScrobbleTypes.LastFMScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const pre_formatted_scrobble: ScrobbleTypes.LastFMScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + + const formatted_scrobble = rules.applyScrobbleRule(pre_formatted_scrobble, rule_list); var value = try scrobbles_data.append(.object); diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 769d062..92d7598 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -12,7 +12,7 @@ pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { inline else => |on| match_found = match_found and std.mem.eql(u8, @field(scrobble, @tagName(on)), cond.match_txt), }, .contains => switch (cond.match_on) { - inline else => |on| match_found = match_found and (std.mem.count(u8, @field(scrobble, @tagName(on)), cond.match_txt) > 0), + inline else => |on| match_found = match_found and std.mem.containsAtLeast(u8, @field(scrobble, @tagName(on)), 1, cond.match_txt), }, } } From e9c72041a530cf283178d9fcefe3eee693daff03 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 22 Apr 2025 13:50:39 -0400 Subject: [PATCH 017/103] Allow multiple conditions in rules. Scrobble processing appears noticeably slower (according to the logs), so I think rules are going to be something to optimize later. Fortunately, they shouldn't need to be applied too often --- build.zig.zon | 4 +- src/app/jobs/process_rule.zig | 29 +++--------- src/app/views/rules.zig | 13 +++-- src/app/views/rules/index.zmpl | 87 ++++++++++------------------------ src/apply_rule.zig | 28 ++++++++--- src/types.zig | 3 +- 6 files changed, 67 insertions(+), 97 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 823b42c..0839430 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -17,8 +17,8 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/2c52792217b9441ed5e91d67e7ec5a8959285307.tar.gz", - .hash = "jetzig-0.0.0-IpAgLbMeDwDYRqWu0OJixGbcDUoUbfAN0xGe1xsYRHTj", + .url = "https://github.com/jetzig-framework/jetzig/archive/86d82026ab574d4e5c3c6cc3817dda84b510001a.tar.gz", + .hash = "jetzig-0.0.0-IpAgLTkzDwDKmsY9MqM41EHDXWGkViiECa0lzV8xl17x", }, .zeit = .{ .url = "https://github.com/rockorager/zeit/archive/refs/heads/main.tar.gz", diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 7cb1a71..f211dbe 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -1,29 +1,14 @@ const std = @import("std"); const jetzig = @import("jetzig"); +const Data = @import("../../types.zig"); pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { _ = env; //_ = params; - const Rule = struct { - name: []const u8, - conditionals: []struct { - match_on: []const u8, - match_cond: []const u8, - match_txt: []const u8, - }, - actions: []struct { - action: []const u8, - action_on: []const u8, - action_txt: []const u8, - }, - }; + std.log.debug("{s}", .{try params.toJson()}); - const Rules = struct { - rules: []const Rule, - }; - - const rule = try std.json.parseFromSliceLeaky(Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); + const rule = try std.json.parseFromSliceLeaky(Data.Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); const file_read: std.fs.File = std.fs.cwd().openFile("rules.json", .{}) catch |read_err| switch (read_err) { error.FileNotFound => { @@ -34,7 +19,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig return; }, }; - const out_rules = Rules{ .rules = &[_]Rule{rule} }; + const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file.writeAll(out); file.close(); @@ -46,17 +31,17 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }, }; - var rules = std.ArrayList(Rule).init(allocator); + var rules = std.ArrayList(Data.Rule).init(allocator); defer rules.deinit(); const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); - const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); try rules.appendSlice(content.rules); try rules.append(rule); file_read.close(); const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); - const out_rules = Rules{ .rules = rules.items }; + const out_rules = Data.Rules{ .rules = rules.items }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file_write.writeAll(out); diff --git a/src/app/views/rules.zig b/src/app/views/rules.zig index bb8ea4b..957f5d2 100644 --- a/src/app/views/rules.zig +++ b/src/app/views/rules.zig @@ -25,12 +25,17 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var job = try request.job("process_rule"); _ = try job.params.put("name", params.get("rule-title")); + _ = try job.params.put("cond_req", params.get("cond-req")); var conditionals = try job.params.put("conditionals", .array); - var cond0 = try conditionals.append(.object); - try cond0.put("match_on", params.get("match-on")); - try cond0.put("match_cond", params.get("match-cond")); - try cond0.put("match_txt", params.get("match-txt")); + inline for (0..5) |i| { + if (!std.mem.eql(u8, "", params.getT(.string, comptime std.fmt.comptimePrint("match-txt{}", .{i})).?)) { + var cond = try conditionals.append(.object); + try cond.put("match_on", params.get(comptime std.fmt.comptimePrint("match-on{}", .{i}))); + try cond.put("match_cond", params.get(comptime std.fmt.comptimePrint("match-cond{}", .{i}))); + try cond.put("match_txt", params.get(comptime std.fmt.comptimePrint("match-txt{}", .{i}))); + } + } var actions = try job.params.put("actions", .array); var act0 = try actions.append(.object); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index ab55c96..038ce71 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -12,74 +12,37 @@ Add a rule below.
+Match + +conditonals. +
If - - - - - - - +@for (0..5) |i| { + + + + + + + +
+}
diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 92d7598..db6eb42 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -2,17 +2,33 @@ const std = @import("std"); const Scrobble = @import("./types.zig").LastFMScrobble; const Rules = @import("./types.zig").Rules; +// Wrapper for containsAtLeast to make the switch below to work +fn containsAtLeastOne(haystack: []const u8, needle: []const u8) bool { + return std.mem.containsAtLeast(u8, haystack, 1, needle); +} + +fn eqlDecomped(haystack: []const u8, needle: []const u8) bool { + return std.mem.eql(u8, haystack, needle); +} + pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { - var match_found: bool = true; var output_scrobble: Scrobble = scrobble; for (rules.rules) |rule| { + var match_found: bool = switch (rule.cond_req) { + .any => false, + .all => true, + }; for (rule.conditionals) |cond| { - switch (cond.match_cond) { - .is => switch (cond.match_on) { - inline else => |on| match_found = match_found and std.mem.eql(u8, @field(scrobble, @tagName(on)), cond.match_txt), + const match_fn: *const fn ([]const u8, []const u8) bool = switch (cond.match_cond) { + .is => eqlDecomped, + .contains => containsAtLeastOne, + }; + switch (rule.cond_req) { + .any => switch (cond.match_on) { + inline else => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), }, - .contains => switch (cond.match_on) { - inline else => |on| match_found = match_found and std.mem.containsAtLeast(u8, @field(scrobble, @tagName(on)), 1, cond.match_txt), + .all => switch (cond.match_on) { + inline else => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), }, } } diff --git a/src/types.zig b/src/types.zig index 61b0e88..0e385a5 100644 --- a/src/types.zig +++ b/src/types.zig @@ -35,8 +35,9 @@ pub const SpotifyScrobble = struct { incognito_mode: ?bool, }; -const Rule = struct { +pub const Rule = struct { name: []const u8, + cond_req: enum { any, all }, conditionals: []struct { match_on: ScrobbleFields, match_cond: enum { is, contains }, From 0631ded1155a98df046617f08b3c5a5c7333f0ea Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 23 Apr 2025 19:32:32 -0400 Subject: [PATCH 018/103] Work on add artist action in rules Really close to having it work, but there seems to be an error when uploading files, which causes particularly annoying problems on WSL when testing, so I'm commiting and trying on my desktop. --- src/app/jobs/process_rule.zig | 14 +- src/app/jobs/process_scrobbles.zig | 217 ++++++++++++++++++++--------- src/app/views/partials/_table.zmpl | 6 +- src/app/views/rules/index.zmpl | 3 +- src/app/views/upload.zig | 114 +++++++++------ src/apply_rule.zig | 45 ++++-- src/types.zig | 46 +++--- 7 files changed, 300 insertions(+), 145 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index f211dbe..217826a 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -35,12 +35,20 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig defer rules.deinit(); const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); - const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); - try rules.appendSlice(content.rules); - try rules.append(rule); file_read.close(); const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); + if (file_content.len == 0) { + const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; + const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file_write.writeAll(out); + file_write.close(); + return; + } + const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); + try rules.appendSlice(content.rules); + try rules.append(rule); + const out_rules = Data.Rules{ .rules = rules.items }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 4262fe5..87871c7 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -1,9 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const Scrobble = @import("../../types.zig").LastFMScrobble; const lastfm = @import("../../types.zig").LastFM; -//const Rules = @import("../../types.zig").Rules; const Data = @import("../../types.zig"); const rules = @import("../../apply_rule.zig"); @@ -18,26 +16,50 @@ const rules = @import("../../apply_rule.zig"); // - environment: Enum of `{ production, development }`. pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { //_ = env; - _ = allocator; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { - //const fixed_date: u32 = @as(u32, item.getT(.integer, "date").?); - const scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; + //var buffer: [256**4]u8 = undefined; + //var fba = std.heap.FixedBufferAllocator.init(&buffer); + //const alloc = fba.allocator(); - // Make hashes - //const album_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.album))); - //const artist_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.artist))); - //const song_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.track))); + var alssu8 = std.ArrayList([]const u8).init(allocator); + defer alssu8.deinit(); - // Create a buffer to hold the metadata to hash. Numbers based on the title of a - // particularly long Sufjan Stevens song title, and we're gonna pray the metadata - // does not exceed three times it's length. - var buffer = [_]u8{undefined} ** (288 * 3); - const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.artist))); - const album_prehash = try std.fmt.bufPrint(&buffer, "{s}{s}", .{ scrobble.artist, scrobble.album }); - const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(album_prehash))); - const song_prehash = try std.fmt.bufPrint(&buffer, "{s}{s}{s}", .{ scrobble.artist, scrobble.album, scrobble.track }); - const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(song_prehash))); + for (item.getT(.array, "artists_track").?.items()) |artist| { + try alssu8.append(try artist.coerce([]const u8)); + } + + const track_artists = try alssu8.toOwnedSlice(); + + for (item.getT(.array, "artists_album").?.items()) |artist| { + try alssu8.append(try artist.coerce([]const u8)); + } + + const album_artists = try alssu8.toOwnedSlice(); + + const scrobble: Data.Scrobble = .{ + .track = item.getT(.string, "track").?, + .artists_track = track_artists, + .album = item.getT(.string, "album") orelse "", + .artists_album = album_artists, + .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), + }; + + var id_prehash = std.ArrayList(u8).init(allocator); + defer id_prehash.deinit(); + + var alartist = std.ArrayList(struct { name: []const u8, id: i32 }).init(allocator); + defer alartist.deinit(); + + for (scrobble.artists_track) |artist| { + //try id_prehash.appendSlice(artist); + try alartist.append(.{ .name = artist, .id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist))) }); + } + //const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); + try id_prehash.appendSlice(scrobble.album); + const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); + try id_prehash.appendSlice(scrobble.track); + const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); // Make IDs // Song: Song hash XOR artist hash XOR album hash @@ -65,64 +87,121 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); //const song_id = (song_hash ^ artist_hash ^ album_hash); + var albumsong = try jetzig.database.Query(.Albumsong) + .findBy(.{ + .album_id = album_id, + .song_id = song_id, + }) + .select(.{.id}) + .execute(env.repo); - var albumsong = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); - var ins_album = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); + var ins_album = try jetzig.database.Query(.Album) + .find(album_id) + .select(.{.id}) + .execute(env.repo); - var ins_artist = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); - if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); + for (alartist.items) |artist| { + var ins_artist = try jetzig.database.Query(.Artist) + .find(artist.id) + .select(.{.id}) + .execute(env.repo); - if (albumsong == null) { - var ins_song = try jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); - if (ins_song == null) ins_song = try jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo); + if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist) + .insert(.{ + .id = artist.id, + .name = artist.name, + .disambiguation = null, + }) + .returning(.{.id}) + .execute(env.repo); - if (ins_album == null) { - ins_album = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); - // I think there's still technically a bug here when you have a different artist but I'm not sure - try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist.?.id, .album_id = ins_album.?.id }).execute(env.repo); + if (albumsong == null) { + var ins_song = try jetzig.database.Query(.Song) + .find(song_id) + .select(.{.id}) + .execute(env.repo); + + if (ins_song == null) ins_song = try jetzig.database.Query(.Song) + .insert(.{ + .id = song_id, + .name = scrobble.track, + .length = null, + .hidden = false, + }) + .returning(.{.id}) + .execute(env.repo); + + if (ins_album == null) { + ins_album = try jetzig.database.Query(.Album) + .insert(.{ + .id = album_id, + .name = scrobble.album, + .length = null, + }) + .returning(.{.id}) + .execute(env.repo); + // I think there's still technically a bug here when you have a different artist but I'm not sure + try jetzig.database.Query(.Artistalbum) + .insert(.{ + .artist_id = ins_artist.?.id, + .album_id = ins_album.?.id, + }) + .execute(env.repo); + } + + albumsong = try jetzig.database.Query(.Albumsong) + .insert(.{ + .song_id = ins_song.?.id, + .album_id = ins_album.?.id, + }) + .returning(.{.id}) + .execute(env.repo); + + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ + .albumsong_id = albumsong.?.id, + .artist_id = ins_artist.?.id, + }) + .execute(env.repo); + } else { + const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) + .findBy(.{ + .albumsong_id = albumsong.?.id, + .artist_id = ins_artist.?.id, + }) + .select(.{.id}) + .execute(env.repo); + + if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist) + .insert(.{ + .albumsong_id = albumsong.?.id, + .artist_id = ins_artist.?.id, + }) + .execute(env.repo); + + const ins_artistalbum = try jetzig.database.Query(.Artistalbum) + .findBy(.{ + .album_id = ins_album.?.id, + .artist_id = ins_artist.?.id, + }) + .select(.{.id}) + .execute(env.repo); + + if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum) + .insert(.{ + .album_id = ins_album.?.id, + .artist_id = ins_artist.?.id, + }) + .execute(env.repo); } - - albumsong = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }).returning(.{.id}).execute(env.repo); - - try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); - } else { - const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist).findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); - if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); - - const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); - if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } - //if (ins_artist_id == null) { - // ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); - // try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); - // try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); - //} - - try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); + try jetzig.database.Query(.Scrobble) + .insert(.{ + .albumsong = albumsong.?.id, + .datetime = scrobble.date, + }) + .execute(env.repo); } } - - // I would like to replicate this kind of functionality for several kinds of queries - // This one gives me all albums by Dream Theater (it also returns Dream Theater for - // each entry, but removing artists.name from the SELECT would remove that) - // - // SELECT - // artists.name, albums.name - // FROM - // "Albumartists" - // INNER JOIN artists - // ON "Albumartists".artist_id = artists.id - // INNER JOIN albums - // ON "Albumartists".album_id = albums.id - // WHERE artists.name = 'Dream Theater'; - - //const query = jetzig.database.Query(.Artist).include(.artistalbums, .{}); - //const results = try env.repo.all(query); - //defer env.repo.free(results); - //for (results) |result| { - // for (result.artistalbums) |artistalbum| { - // std.log.debug("{s}: {any}", .{ result.name, artistalbum.album_id }); - // } - //} } diff --git a/src/app/views/partials/_table.zmpl b/src/app/views/partials/_table.zmpl index 8ea394d..83c1cd4 100644 --- a/src/app/views/partials/_table.zmpl +++ b/src/app/views/partials/_table.zmpl @@ -10,7 +10,11 @@ @for (table_data) |value| { {{value.track}} - {{value.artist}} + + @for (value.get("artists").?) |artist| { + {{artist}} + } + {{value.album}} {{value.date}} diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index 038ce71..f97ef35 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -52,7 +52,8 @@ then diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 032da9d..98f0009 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -1,7 +1,6 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const ScrobbleTypes = @import("../../types.zig"); const zeit = @import("zeit"); const rules = @import("../../apply_rule.zig"); const Data = @import("../../types.zig"); @@ -33,58 +32,70 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - const rule_file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); + const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), + else => err, + }); + defer rule_file.close(); const file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = try std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}); + const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}) catch null; - // The only difference between a LastFM scrobble and a Spotify scrobble is the format. - // I've made a branches for each, because doing it all in one made the readability terrible, - // and formatting the date in particular was challenging. I could probably pull out the - // actual appending at some point, since that's the same process for each, but I'm not - // sure how to do that yet. switch (source) { 0 => { - const content: ScrobbleTypes.LastFM = try std.json.parseFromSliceLeaky(ScrobbleTypes.LastFM, request.allocator, file.content, .{}); + const content: Data.LastFM = try std.json.parseFromSliceLeaky(Data.LastFM, request.allocator, file.content, .{}); const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; appends: for (content.scrobbles) |scrobble| { // We can short-circuit on the limiter bools if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; - var value = try scrobbles_data.append(.object); - const formatted_scrobble = rules.applyScrobbleRule(scrobble, rule_list); + const formatted_scrobble = if (rule_list) |rl| + rules.applyScrobbleRule(request.allocator, scrobble, rl) + else + Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date, + }; - // This is so unnecessary, probably useful once I start doing Spotify integration though - inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| { - try value.put(f.name, @as(f.type, @field(formatted_scrobble, f.name))); + var scrobble_view = try scrobbles_view.append(.object); + var artists = try scrobble_view.put("artists", .array); + + try scrobble_view.put("track", formatted_scrobble.track); + try scrobble_view.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_track) |artist| { + try artists.append(artist); } - // Note sure why this works for ZMPL, but not for jobs. - try scrobbles_view.append(formatted_scrobble); + try scrobble_view.put("date", formatted_scrobble.date); + + var scrobble_data = try scrobbles_data.append(.object); + var artists_album = try scrobble_data.put("artists_album", .array); + var artists_track = try scrobble_data.put("artists_track", .array); + + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_album) |artist| { + try artists_album.append(artist); + } + + for (formatted_scrobble.artists_track) |artist| { + try artists_track.append(artist); + } + try scrobble_data.put("date", formatted_scrobble.date); } }, 1 => { - const content: []ScrobbleTypes.SpotifyScrobble = try std.json.parseFromSliceLeaky([]ScrobbleTypes.SpotifyScrobble, request.allocator, file.content, .{}); + const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else (try zeit.instant(.{})).time(); const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); appends: for (content) |scrobble| { - // A LastFM Scrobble occurs when half a song has been played - // or the song plays for 4 minutes, whichever happens first. - // Spotify considers a song played if it plays for 30 seconds. - // Ideally, I would go with the LastFM convention, but Spotify - // history data only gives us so much information. I'm okay - // with the 30 second convention, but eventually I would prefer - // to get the song length from MusicBrainz and check if it meets - // the requirement. Until then, if it goes 30 seconds, or the - // reason_end field reads "trackdone", then it counts as a Scrobble. - // May consider giving user control to the minimum millisecond requirement. if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { skipped_tracks += 1; continue :appends; } - // In the case where the artist is null, but there's other metadata, I could - // probably let the user edit it in themselves, although I'm not sure if that - // situation happens. if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { skipped_tracks += 1; continue :appends; @@ -96,19 +107,42 @@ pub fn post(request: *jetzig.Request) !jetzig.View { continue :appends; } - // Turn SpotifyScrobble into a LastFM scrobble - const pre_formatted_scrobble: ScrobbleTypes.LastFMScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const pre_formatted_scrobble: Data.ImportedScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const formatted_scrobble = if (rule_list) |rl| + rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; - const formatted_scrobble = rules.applyScrobbleRule(pre_formatted_scrobble, rule_list); + var scrobble_view = try scrobbles_view.append(.object); + var artists = try scrobble_view.put("artists", .array); - var value = try scrobbles_data.append(.object); - - // This is so unnecessary, probably useful once I start doing Spotify integration though - inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| { - try value.put(f.name, @as(f.type, @field(formatted_scrobble, f.name))); + try scrobble_view.put("track", formatted_scrobble.track); + try scrobble_view.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_track) |artist| { + try artists.append(artist); } - // Note sure why this works for ZMPL, but not for jobs. - try scrobbles_view.append(formatted_scrobble); + try scrobble_view.put("date", formatted_scrobble.date); + + var scrobble_data = try scrobbles_data.append(.object); + var artists_album = try scrobble_data.put("artists_album", .array); + var artists_track = try scrobble_data.put("artists_track", .array); + + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_album) |artist| { + try artists_album.append(artist); + } + + for (formatted_scrobble.artists_track) |artist| { + try artists_track.append(artist); + } + try scrobble_data.put("date", formatted_scrobble.date); } }, else => unreachable, diff --git a/src/apply_rule.zig b/src/apply_rule.zig index db6eb42..dd9f8bb 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -1,18 +1,32 @@ const std = @import("std"); -const Scrobble = @import("./types.zig").LastFMScrobble; const Rules = @import("./types.zig").Rules; +const Data = @import("./types.zig"); // Wrapper for containsAtLeast to make the switch below to work -fn containsAtLeastOne(haystack: []const u8, needle: []const u8) bool { +fn containsWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.containsAtLeast(u8, haystack, 1, needle); } -fn eqlDecomped(haystack: []const u8, needle: []const u8) bool { +fn eqlWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.eql(u8, haystack, needle); } -pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { - var output_scrobble: Scrobble = scrobble; +pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) Data.Scrobble { + var output_scrobble = Data.Scrobble{ + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .date = scrobble.date, + }; + + //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + //const gpalloc = gpa.allocator(); + + //var arena = std.heap.ArenaAllocator.init(gpalloc); + //defer arena.deinit(); + //const allocator = arena.allocator(); + for (rules.rules) |rule| { var match_found: bool = switch (rule.cond_req) { .any => false, @@ -20,8 +34,8 @@ pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { }; for (rule.conditionals) |cond| { const match_fn: *const fn ([]const u8, []const u8) bool = switch (cond.match_cond) { - .is => eqlDecomped, - .contains => containsAtLeastOne, + .is => eqlWrapper, + .contains => containsWrapper, }; switch (rule.cond_req) { .any => switch (cond.match_on) { @@ -35,9 +49,22 @@ pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { if (match_found) { for (rule.actions) |act| { switch (act.action) { - .add => {}, + .add => { + var al = std.ArrayList([]const u8).init(allocator); + switch (act.action_on) { + .album, .track => unreachable, + inline else => |on| { + // I have decided an error won't happen :) + al.appendSlice(@field(output_scrobble, @tagName(on))) catch unreachable; + al.append(act.action_txt) catch unreachable; + const artists = al.toOwnedSlice() catch unreachable; + @field(output_scrobble, @tagName(on)) = artists; + }, + } + }, .replace => switch (act.action_on) { - inline else => |on| @field(output_scrobble, @tagName(on)) = act.action_txt, + inline .album, .track => |on| @field(output_scrobble, @tagName(on)) = act.action_txt, + inline .artists_album, .artists_track => |on| @field(output_scrobble, @tagName(on)) = &[_][]const u8{act.action_txt}, }, } } diff --git a/src/types.zig b/src/types.zig index 0e385a5..f0268ed 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,51 +1,59 @@ -pub const LastFMScrobble = struct { +pub const ImportedScrobble = struct { track: []const u8, artist: []const u8, album: []const u8 = "", date: i128, }; +pub const Scrobble = struct { + track: []const u8, + artists_track: []const []const u8, + album: []const u8 = "", + artists_album: []const []const u8, + date: i128, +}; + // From lastfmstats.com -pub const LastFM = struct { username: []const u8, scrobbles: []LastFMScrobble }; +pub const LastFM = struct { username: []const u8, scrobbles: []ImportedScrobble }; // I derived whether or not these values were optional from searching // the respective fields for null in Vim, so there may be some fields // that can be optional that I haven't run into yet pub const SpotifyScrobble = struct { ts: []const u8, - username: []const u8, - platform: []const u8, + //username: []const u8, + //platform: []const u8, ms_played: u64, - conn_country: []const u8, - ip_addr_decrypted: ?[]const u8, - user_agent_decrypted: ?[]const u8, + //conn_country: []const u8, + //ip_addr_decrypted: ?[]const u8, + //user_agent_decrypted: ?[]const u8, master_metadata_track_name: ?[]const u8, master_metadata_album_artist_name: ?[]const u8, master_metadata_album_album_name: ?[]const u8, - spotify_track_uri: ?[]const u8, - episode_name: ?[]const u8, - episode_show_name: ?[]const u8, - spotify_episode_uri: ?[]const u8, + //spotify_track_uri: ?[]const u8, + //episode_name: ?[]const u8, + //episode_show_name: ?[]const u8, + //spotify_episode_uri: ?[]const u8, reason_start: []const u8, reason_end: ?[]const u8, - shuffle: bool, + //shuffle: bool, skipped: ?bool, - offline: bool, + //offline: bool, offline_timestamp: u64, - incognito_mode: ?bool, + //incognito_mode: ?bool, }; pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, conditionals: []struct { - match_on: ScrobbleFields, + match_on: enum { artist, album, track }, match_cond: enum { is, contains }, match_txt: []const u8, }, actions: []struct { action: enum { replace, add }, - action_on: ScrobbleFields, + action_on: enum { artists_album, album, artists_track, track }, action_txt: []const u8, }, }; @@ -53,9 +61,3 @@ pub const Rule = struct { pub const Rules = struct { rules: []const Rule, }; - -pub const ScrobbleFields = enum { - artist, - album, - track, -}; From be8c1191b05e2fb8b941fc383c1b7f66b06425f4 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 24 Apr 2025 09:34:34 -0400 Subject: [PATCH 019/103] Clean --- src/app/jobs/process_scrobbles.zig | 23 ++++++----------------- src/apply_rule.zig | 15 ++++----------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 87871c7..fc30e58 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -18,41 +18,30 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //_ = env; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { - //var buffer: [256**4]u8 = undefined; - //var fba = std.heap.FixedBufferAllocator.init(&buffer); - //const alloc = fba.allocator(); - - var alssu8 = std.ArrayList([]const u8).init(allocator); - defer alssu8.deinit(); + var track_artists_al = std.ArrayList([]const u8).init(allocator); + var album_artists_al = std.ArrayList([]const u8).init(allocator); for (item.getT(.array, "artists_track").?.items()) |artist| { - try alssu8.append(try artist.coerce([]const u8)); + try track_artists_al.append(try artist.coerce([]const u8)); } - const track_artists = try alssu8.toOwnedSlice(); - for (item.getT(.array, "artists_album").?.items()) |artist| { - try alssu8.append(try artist.coerce([]const u8)); + try album_artists_al.append(try artist.coerce([]const u8)); } - const album_artists = try alssu8.toOwnedSlice(); - const scrobble: Data.Scrobble = .{ .track = item.getT(.string, "track").?, - .artists_track = track_artists, + .artists_track = track_artists_al.items, .album = item.getT(.string, "album") orelse "", - .artists_album = album_artists, + .artists_album = album_artists_al.items, .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), }; var id_prehash = std.ArrayList(u8).init(allocator); - defer id_prehash.deinit(); var alartist = std.ArrayList(struct { name: []const u8, id: i32 }).init(allocator); - defer alartist.deinit(); for (scrobble.artists_track) |artist| { - //try id_prehash.appendSlice(artist); try alartist.append(.{ .name = artist, .id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist))) }); } //const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); diff --git a/src/apply_rule.zig b/src/apply_rule.zig index dd9f8bb..c63ad4a 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -20,13 +20,6 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc .date = scrobble.date, }; - //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - //const gpalloc = gpa.allocator(); - - //var arena = std.heap.ArenaAllocator.init(gpalloc); - //defer arena.deinit(); - //const allocator = arena.allocator(); - for (rules.rules) |rule| { var match_found: bool = switch (rule.cond_req) { .any => false, @@ -57,9 +50,11 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc // I have decided an error won't happen :) al.appendSlice(@field(output_scrobble, @tagName(on))) catch unreachable; al.append(act.action_txt) catch unreachable; - const artists = al.toOwnedSlice() catch unreachable; - @field(output_scrobble, @tagName(on)) = artists; + @field(output_scrobble, @tagName(on)) = al.items; }, + //else => { + // std.log.debug("Adding artists doesn't work yet", .{}); + //}, } }, .replace => switch (act.action_on) { @@ -73,5 +68,3 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc return output_scrobble; } - -//pub fn applyAlbumRule() !Album {} From 9df8f9ea12f14bced8fc220fb6f4827e6cce39e4 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 10:41:42 -0400 Subject: [PATCH 020/103] Fix segfault in applyScrobbleRule Thanks bob :) --- src/app/jobs/process_rule.zig | 1 - src/app/views/upload.zig | 16 +++++++++++----- src/apply_rule.zig | 16 +++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 217826a..ef0fc09 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -32,7 +32,6 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }; var rules = std.ArrayList(Data.Rule).init(allocator); - defer rules.deinit(); const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); file_read.close(); diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 98f0009..59442a8 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -38,8 +38,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { }); defer rule_file.close(); - const file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}) catch null; + const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); + const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; switch (source) { 0 => { @@ -51,7 +51,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; const formatted_scrobble = if (rule_list) |rl| - rules.applyScrobbleRule(request.allocator, scrobble, rl) + try rules.applyScrobbleRule(request.allocator, scrobble, rl) else Data.Scrobble{ .album = scrobble.album, @@ -107,9 +107,15 @@ pub fn post(request: *jetzig.Request) !jetzig.View { continue :appends; } - const pre_formatted_scrobble: Data.ImportedScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const pre_formatted_scrobble: Data.ImportedScrobble = .{ + .track = scrobble.master_metadata_track_name.?, + .album = scrobble.master_metadata_album_album_name.?, + .artist = scrobble.master_metadata_album_artist_name.?, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000, + }; + const formatted_scrobble = if (rule_list) |rl| - rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) else Data.Scrobble{ .album = pre_formatted_scrobble.album, diff --git a/src/apply_rule.zig b/src/apply_rule.zig index c63ad4a..6c03b38 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -11,12 +11,14 @@ fn eqlWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.eql(u8, haystack, needle); } -pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) Data.Scrobble { +pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) !Data.Scrobble { + const artists = try allocator.alloc([]const u8, 1); + artists[0] = scrobble.artist; var output_scrobble = Data.Scrobble{ .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, + .artists_track = artists, .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, + .artists_album = artists, .date = scrobble.date, }; @@ -47,10 +49,10 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc switch (act.action_on) { .album, .track => unreachable, inline else => |on| { - // I have decided an error won't happen :) - al.appendSlice(@field(output_scrobble, @tagName(on))) catch unreachable; - al.append(act.action_txt) catch unreachable; - @field(output_scrobble, @tagName(on)) = al.items; + try al.appendSlice(@field(output_scrobble, @tagName(on))); + try al.append(act.action_txt); + const list = try al.toOwnedSlice(); + @field(output_scrobble, @tagName(on)) = list; }, //else => { // std.log.debug("Adding artists doesn't work yet", .{}); From 5e58e81ca7ef162c1af4ceed412bd124a3948809 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 14:28:39 -0400 Subject: [PATCH 021/103] Fix album artist parsing in process_scrobbles --- src/app/jobs/process_rule.zig | 3 - src/app/jobs/process_scrobbles.zig | 189 +++++++++-------------------- 2 files changed, 60 insertions(+), 132 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index ef0fc09..7b60d54 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -4,9 +4,6 @@ const Data = @import("../../types.zig"); pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { _ = env; - //_ = params; - - std.log.debug("{s}", .{try params.toJson()}); const rule = try std.json.parseFromSliceLeaky(Data.Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index fc30e58..ae3cfa8 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -18,179 +18,110 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //_ = env; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { - var track_artists_al = std.ArrayList([]const u8).init(allocator); - var album_artists_al = std.ArrayList([]const u8).init(allocator); + const track_artist_count = item.getT(.array, "artists_track").?.count(); + const album_artist_count = item.getT(.array, "artists_album").?.count(); + var track_artist_name_buffer = try allocator.alloc([]const u8, track_artist_count); + var album_artist_name_buffer = try allocator.alloc([]const u8, album_artist_count); + var track_artist_id_buffer = try allocator.alloc(i32, track_artist_count); + var album_artist_id_buffer = try allocator.alloc(i32, album_artist_count); - for (item.getT(.array, "artists_track").?.items()) |artist| { - try track_artists_al.append(try artist.coerce([]const u8)); + for (item.getT(.array, "artists_track").?.items(), 0..track_artist_count) |artist, i| { + const artist_name = try artist.coerce([]const u8); + track_artist_name_buffer[i] = artist_name; + track_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); } - for (item.getT(.array, "artists_album").?.items()) |artist| { - try album_artists_al.append(try artist.coerce([]const u8)); + for (item.getT(.array, "artists_album").?.items(), 0..album_artist_count) |artist, i| { + const artist_name = try artist.coerce([]const u8); + album_artist_name_buffer[i] = artist_name; + album_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); } const scrobble: Data.Scrobble = .{ .track = item.getT(.string, "track").?, - .artists_track = track_artists_al.items, + .artists_track = track_artist_name_buffer, .album = item.getT(.string, "album") orelse "", - .artists_album = album_artists_al.items, + .artists_album = album_artist_name_buffer, .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), }; + // Probably want to include artist name here, but not sure how to yet var id_prehash = std.ArrayList(u8).init(allocator); - var alartist = std.ArrayList(struct { name: []const u8, id: i32 }).init(allocator); - - for (scrobble.artists_track) |artist| { - try alartist.append(.{ .name = artist, .id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist))) }); - } - //const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); try id_prehash.appendSlice(scrobble.album); const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); try id_prehash.appendSlice(scrobble.track); const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); - // Make IDs - // Song: Song hash XOR artist hash XOR album hash - // This way, if two songs share a name, then - // the IDs also depend on the hash of the album - // they're on, as well as the artist name. As far - // as I can tell, this is only as issue for Sufjan - // Steven's `Songs for Christmas`. (In practice. - // In reality, there are albums with several untitled - // songs (Selected Ambient Works Vol. II by Aphex Twin, - // ( ) by Sigur Ros, ...) that have working titles - // in their place.) - - // Album: If the album is not self-titled, then - // album hash XOR artist hash. This way, if two - // artists have an album of the same name, then - // the IDs also depend on the hash of the artist - // name. As far as I can tell, this is only an - // issue for Weezer and Peter Gabriel, but their - // albums go by unique names anyways. - - // Artist: Artist hash. If two artists have the same name, - // then a disambiguating string can be provided to - // differentiate after the fact, or in a rule. - - //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); - //const song_id = (song_hash ^ artist_hash ^ album_hash); var albumsong = try jetzig.database.Query(.Albumsong) - .findBy(.{ - .album_id = album_id, - .song_id = song_id, - }) - .select(.{.id}) - .execute(env.repo); + .findBy(.{ .album_id = album_id, .song_id = song_id }) + .select(.{.id}).execute(env.repo); var ins_album = try jetzig.database.Query(.Album) .find(album_id) - .select(.{.id}) - .execute(env.repo); + .select(.{.id}).execute(env.repo); - for (alartist.items) |artist| { + for (track_artist_name_buffer, track_artist_id_buffer) |artist_name, artist_id| { var ins_artist = try jetzig.database.Query(.Artist) - .find(artist.id) - .select(.{.id}) - .execute(env.repo); + .find(artist_id) + .select(.{.id}).execute(env.repo); - if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ - .id = artist.id, - .name = artist.name, - .disambiguation = null, - }) - .returning(.{.id}) - .execute(env.repo); + if (ins_artist == null) + ins_artist = try jetzig.database.Query(.Artist) + .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + .returning(.{.id}).execute(env.repo); if (albumsong == null) { var ins_song = try jetzig.database.Query(.Song) .find(song_id) - .select(.{.id}) - .execute(env.repo); + .select(.{.id}).execute(env.repo); - if (ins_song == null) ins_song = try jetzig.database.Query(.Song) - .insert(.{ - .id = song_id, - .name = scrobble.track, - .length = null, - .hidden = false, - }) - .returning(.{.id}) - .execute(env.repo); + if (ins_song == null) + ins_song = try jetzig.database.Query(.Song) + .insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }) + .returning(.{.id}).execute(env.repo); - if (ins_album == null) { + if (ins_album == null) ins_album = try jetzig.database.Query(.Album) - .insert(.{ - .id = album_id, - .name = scrobble.album, - .length = null, - }) - .returning(.{.id}) - .execute(env.repo); - // I think there's still technically a bug here when you have a different artist but I'm not sure - try jetzig.database.Query(.Artistalbum) - .insert(.{ - .artist_id = ins_artist.?.id, - .album_id = ins_album.?.id, - }) - .execute(env.repo); - } + .insert(.{ .id = album_id, .name = scrobble.album, .length = null }) + .returning(.{.id}).execute(env.repo); albumsong = try jetzig.database.Query(.Albumsong) - .insert(.{ - .song_id = ins_song.?.id, - .album_id = ins_album.?.id, - }) - .returning(.{.id}) - .execute(env.repo); + .insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }) + .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Albumsongsartist) - .insert(.{ - .albumsong_id = albumsong.?.id, - .artist_id = ins_artist.?.id, - }) - .execute(env.repo); + .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } else { const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) - .findBy(.{ - .albumsong_id = albumsong.?.id, - .artist_id = ins_artist.?.id, - }) - .select(.{.id}) - .execute(env.repo); + .findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }) + .select(.{.id}).execute(env.repo); - if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist) - .insert(.{ - .albumsong_id = albumsong.?.id, - .artist_id = ins_artist.?.id, - }) - .execute(env.repo); + if (ins_albumsongartist == null) + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + } + } + for (album_artist_name_buffer, album_artist_id_buffer) |artist_name, artist_id| { + const ins_artistalbum = try jetzig.database.Query(.Artistalbum) + .findBy(.{ .album_id = ins_album.?.id, .artist_id = artist_id }) + .select(.{.id}).execute(env.repo); - const ins_artistalbum = try jetzig.database.Query(.Artistalbum) - .findBy(.{ - .album_id = ins_album.?.id, - .artist_id = ins_artist.?.id, - }) - .select(.{.id}) - .execute(env.repo); - - if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum) - .insert(.{ - .album_id = ins_album.?.id, - .artist_id = ins_artist.?.id, - }) - .execute(env.repo); + if (ins_artistalbum == null) { + var ins_artist = try jetzig.database.Query(.Artist) + .find(artist_id) + .select(.{.id}).execute(env.repo); + if (ins_artist == null) + ins_artist = try jetzig.database.Query(.Artist) + .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + .returning(.{.id}).execute(env.repo); + try jetzig.database.Query(.Artistalbum) + .insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } } try jetzig.database.Query(.Scrobble) - .insert(.{ - .albumsong = albumsong.?.id, - .datetime = scrobble.date, - }) - .execute(env.repo); + .insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); } } } From 18d4df0a5c2d404e9ffe2752e9b74aff5948c6be Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 15:48:47 -0400 Subject: [PATCH 022/103] Fix albums not being hashed correctly Also provides more actions for rules, but they don't seem to work... --- src/app/jobs/process_scrobbles.zig | 25 ++++++++++--------- src/app/views/rules.zig | 17 +++++++++---- src/app/views/rules/index.zmpl | 39 +++++++++++++----------------- 3 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index ae3cfa8..b2ae22e 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -18,6 +18,9 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //_ = env; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { + + // Probably want to include artist name here, but not sure how to yet + const track_artist_count = item.getT(.array, "artists_track").?.count(); const album_artist_count = item.getT(.array, "artists_album").?.count(); var track_artist_name_buffer = try allocator.alloc([]const u8, track_artist_count); @@ -25,6 +28,16 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig var track_artist_id_buffer = try allocator.alloc(i32, track_artist_count); var album_artist_id_buffer = try allocator.alloc(i32, album_artist_count); + const scrobble: Data.Scrobble = .{ + .track = item.getT(.string, "track").?, + .artists_track = track_artist_name_buffer, + .album = item.getT(.string, "album") orelse "", + .artists_album = album_artist_name_buffer, + .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), + }; + + var id_prehash = std.ArrayList(u8).init(allocator); + for (item.getT(.array, "artists_track").?.items(), 0..track_artist_count) |artist, i| { const artist_name = try artist.coerce([]const u8); track_artist_name_buffer[i] = artist_name; @@ -35,19 +48,9 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; album_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + try id_prehash.appendSlice(artist_name); } - const scrobble: Data.Scrobble = .{ - .track = item.getT(.string, "track").?, - .artists_track = track_artist_name_buffer, - .album = item.getT(.string, "album") orelse "", - .artists_album = album_artist_name_buffer, - .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), - }; - - // Probably want to include artist name here, but not sure how to yet - var id_prehash = std.ArrayList(u8).init(allocator); - try id_prehash.appendSlice(scrobble.album); const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); try id_prehash.appendSlice(scrobble.track); diff --git a/src/app/views/rules.zig b/src/app/views/rules.zig index 957f5d2..7a2df91 100644 --- a/src/app/views/rules.zig +++ b/src/app/views/rules.zig @@ -22,6 +22,8 @@ pub fn edit(id: []const u8, request: *jetzig.Request) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View { const params = try request.params(); + std.log.debug("{s}", .{try params.toJson()}); + var job = try request.job("process_rule"); _ = try job.params.put("name", params.get("rule-title")); @@ -30,6 +32,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var conditionals = try job.params.put("conditionals", .array); inline for (0..5) |i| { if (!std.mem.eql(u8, "", params.getT(.string, comptime std.fmt.comptimePrint("match-txt{}", .{i})).?)) { + //if (params.getT(.string, comptime std.fmt.comptimePrint("match-txt{}", .{i})) != null) { var cond = try conditionals.append(.object); try cond.put("match_on", params.get(comptime std.fmt.comptimePrint("match-on{}", .{i}))); try cond.put("match_cond", params.get(comptime std.fmt.comptimePrint("match-cond{}", .{i}))); @@ -38,11 +41,15 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } var actions = try job.params.put("actions", .array); - var act0 = try actions.append(.object); - try act0.put("action", params.get("action")); - try act0.put("action_on", params.get("action-on")); - try act0.put("action_txt", params.get("action-txt")); - + inline for (0..5) |i| { + if (!std.mem.eql(u8, "", params.getT(.string, comptime std.fmt.comptimePrint("action-txt{}", .{i})).?)) { + //if (params.getT(.string, comptime std.fmt.comptimePrint("action-txt{}", .{i})) != null) { + var act = try actions.append(.object); + try act.put("action", params.get(comptime std.fmt.comptimePrint("action{}", .{i}))); + try act.put("action_on", params.get(comptime std.fmt.comptimePrint("action-on{}", .{i}))); + try act.put("action_txt", params.get(comptime std.fmt.comptimePrint("action-txt{}", .{i}))); + } + } try job.schedule(); return request.render(.created); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index f97ef35..5661f97 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -14,8 +14,8 @@ Add a rule below.
Match conditonals.
@@ -38,27 +38,22 @@ If
} - - - - -
then - - -with - +@for (0..5) |i| { + + + with + +
+} From 01fe10f045909597e892b828dec06135bba3b8e1 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 16:27:03 -0400 Subject: [PATCH 023/103] Fix limit on rule parameters and fix segfault in applyScrobbleRule For sure this time --- src/apply_rule.zig | 6 +++++- src/main.zig | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 6c03b38..4f99ca9 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -61,7 +61,11 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc }, .replace => switch (act.action_on) { inline .album, .track => |on| @field(output_scrobble, @tagName(on)) = act.action_txt, - inline .artists_album, .artists_track => |on| @field(output_scrobble, @tagName(on)) = &[_][]const u8{act.action_txt}, + inline .artists_album, .artists_track => |on| { + const artist = try allocator.alloc([]const u8, 1); + artist[0] = act.action_txt; + @field(output_scrobble, @tagName(on)) = artist; + }, }, } } diff --git a/src/main.zig b/src/main.zig index 1cab920..cdc9bca 100644 --- a/src/main.zig +++ b/src/main.zig @@ -14,10 +14,13 @@ pub const jetzig_options = struct { // htmx middleware skips layouts when `HX-Target` header is present and issues // `HX-Redirect` instead of a regular HTTP redirect when `request.redirect` is called. jetzig.middleware.HtmxMiddleware, - // Demo middleware included with new projects. Remove once you are familiar with Jetzig's - // middleware system. + // Demo middleware included with new projects. Remove once you are familiar with Jetzig's + // middleware system. }; + // This is currently the largest number of parameters one can have in a rule + pub const max_multipart_form_fields = 42; + // Maximum bytes to allow in request body. pub const max_bytes_request_body: usize = std.math.pow(usize, 2, 24); From 65136a44d652dbda40b76597ddc59ee34dc83a1a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 23:58:50 -0400 Subject: [PATCH 024/103] Add more information to artists view, songs view, and format dates correctly in scrobbles view --- README.md | 5 +++- src/app/views/albums.zig | 1 + src/app/views/artists.zig | 47 +++++++++++++++--------------- src/app/views/artists/get.zmpl | 17 +++++++++++ src/app/views/scrobbles.zig | 5 +++- src/app/views/scrobbles/index.zmpl | 2 +- src/app/views/songs.zig | 30 +++++++++++++++---- src/app/views/songs/index.zmpl | 6 ++++ src/apply_rule.zig | 3 -- 9 files changed, 82 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index b62b576..f18e7e0 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,14 @@ Licensed under MIT. - [ ] List all songs on artist page, with respective album - [x] List all albums on artist page - [x] Include number of plays for each + - [x] List albums features on - [x] See all albums under "/albums" - [x] See all songs from album - [x] Include number of plays + - [x] Include name of artist(s) + - [ ] Include artists features on each song - [x] See all songs under "/songs" - - [ ] Include respective artist(s) + - [x] Include respective artist(s) - [ ] Include respective album[^10] - [x] Include number of plays - [ ] Create disambiguation pages diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 57d2bb1..cc6d765 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -33,6 +33,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { continue :blk; } var album_view = try albums_view.append(.object); + var artist_infos = try album_view.put("artist_info", .array); var artist_info = try artist_infos.append(.object); try artist_info.put("name", album.artist_name); diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index b8d6bc3..9f707f0 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -35,6 +35,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var albums_view = try root.put("albums", .array); + var appears = try root.put("appears", .array); const artist_name = try jetzig.database.Query(.Artist).find(id).select(.{ .id, .name }).execute(request.repo); _ = try root.put("artist", artist_name.?.name); const query = @@ -48,6 +49,18 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { \\ORDER BY scrobbles DESC ; + const appears_query = + \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 AND artistalbums.artist_id != $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC; + ; + var albums_jq_result = try request.repo.executeSql(query, .{id}); defer albums_jq_result.deinit(); @@ -60,30 +73,18 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); } - //const artist = try jetzig.database.Query(.Artist) - // .find(id) - // .select(.{ .id, .name }) - // .execute(request.repo); - //var root = try request.data(.object); - //try root.put("artist", artist.?.name); - //var albums_view = try root.put("albums", .array); - //const query = jetzig.database.Query(.Albumartist) - // .select(.{.id}) - // .include(.album, .{ .select = .{ .name, .id } }) - // .join(.inner, .artist) - // .where(.{ .artist = .{ .id = id } }); - //const albums = try request.repo.all(query); - //for (albums) |album| { - // const scrobbles = try jetzig.database.Query(.Scrobble) - // .where(.{ .album_id = album.album.id }) - // .count() - // .execute(request.repo); - // var album_view = try albums_view.append(.object); - // try album_view.put("name", album.album.name); - // try album_view.put("url", album.album.id); - // try album_view.put("scrobbles", scrobbles); - //} + var appears_jq_result = try request.repo.executeSql(appears_query, .{id}); + defer appears_jq_result.deinit(); + + while (try appears_jq_result.postgresql.result.next()) |album_row| { + const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); + var album_view = try appears.append(.object); + try album_view.put("name", album.name); + try album_view.put("url", album.id); + try album_view.put("scrobbles", album.scrobbles); + } + return request.render(.ok); } diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 911d2c3..2cc50cd 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -6,6 +6,7 @@ @partial partials/header

{{.artist}}

+

Albums

@@ -21,6 +22,22 @@ }
+

Albums Featured On

+ + + + + + + + @for (.appears) |album| { + + + + + } + +
NameScrobbles
{{album.name}}{{album.scrobbles}}
- +@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) \ No newline at end of file diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 4164fc1..34cea3e 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -138,19 +138,23 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { defer first_last_songs_jq_result.deinit(); // These look backwards, but it's correct this way + var last_song_view = try root.put("last", .object); + if (try first_last_songs_jq_result.postgresql.result.next()) |last_song_row| { const last_song = try last_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try root.put("last_song_name", last_song.name); - try root.put("last_song_id", last_song.id); - try root.put("last_song_date", (try dateFmt(request.allocator, last_song.date))); - } + try last_song_view.put("name", last_song.name); + try last_song_view.put("id", last_song.id); + try last_song_view.put("date", (try dateFmt(request.allocator, last_song.date))); + } else unreachable; + + var first_song_view = try root.put("first", .object); if (try first_last_songs_jq_result.postgresql.result.next()) |first_song_row| { const first_song = try first_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try root.put("first_song_name", first_song.name); - try root.put("first_song_id", first_song.id); - try root.put("first_song_date", (try dateFmt(request.allocator, first_song.date))); - } + try first_song_view.put("name", first_song.name); + try first_song_view.put("id", first_song.id); + try first_song_view.put("date", (try dateFmt(request.allocator, first_song.date))); + } else unreachable; try first_last_songs_jq_result.drain(); diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index adf6158..48c1c25 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -1,53 +1,22 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.album, .scrobbles}; +} + - @partial partials/header

{{.artist.name}}

-{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place) -
-First listen: {{.first_song_name}} ({{.first_song_date}}) -
-Most recent listen: {{.last_song_name}} ({{.last_song_date}}) +@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, last_song: .last, first_song: .first) +

Albums

-
- - - - - - -@for (.albums) |album| { - - - - -} - -
NameScrobbles
{{album.name}}{{album.scrobbles}}
+@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) +

Albums Featured On

- - - - - - - - @for (.appears) |album| { - - - - - } - -
NameScrobbles
{{album.name}}{{album.scrobbles}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .appears, columns: columns) + \ No newline at end of file diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl new file mode 100644 index 0000000..322a0e8 --- /dev/null +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -0,0 +1,9 @@ +@args scrobbles: i64, rank: []const u8, last_song: *ZmplValue, first_song: *ZmplValue + +
+{{scrobbles}} scrobbles ({{rank}} place) +
+First listen: {{first_song.name}} ({{first_song.date}}) +
+Most recent listen: {{last_song.name}} ({{last_song.date}}) +
\ No newline at end of file diff --git a/src/app/views/partials/_newtable.zmpl b/src/app/views/partials/_newtable.zmpl index bee43c1..8960e21 100644 --- a/src/app/views/partials/_newtable.zmpl +++ b/src/app/views/partials/_newtable.zmpl @@ -1,18 +1,29 @@ -@args table_data *ZmplValue, table_headers: []enum{Song, Album, Artist, Scrobbles, Date} +@args T: type, table_data: *ZmplValue, columns: T
@zig { - for (table_headers) |header| { + for (columns) |header| { switch (header) { - .Artist => { + .song => { + + }, + .album => { + + }, + .artist => { + + }, + .artistlist => { }, - inline else => |other| { - const h = @tagName(other); - + .scrobbles => { + }, + .date => { + + } } } } @@ -20,35 +31,38 @@ @zig { - for (table_data) |data| { + const array = table_data.items(.array); + for (array) |ent| { - for (table_header) |header| { + for (columns) |header| { switch (header) { - .Song => { + .song, .album, .artist => { + const path = switch (header) { + .song => "songs", + .album => "albums", + .artist => "artists", + else => unreachable + }; }, - .Album => { + .artistlist => { - }, - .Artist => { - }, - .Scrobbles => { - + .scrobbles => { + }, - .Date =>{ - + .date =>{ + } - }; + } } + } } diff --git a/src/types.zig b/src/types.zig index 2574610..455f282 100644 --- a/src/types.zig +++ b/src/types.zig @@ -62,10 +62,13 @@ pub const Rules = struct { rules: []const Rule, }; -pub const Headers = []enum { - Song, - Album, - Artist, - Scrobbles, - Date, -}; +// Can't import types in .zmpl files, so defining this here +// doesn't really do much (except maybe in the .zig file for views?) +//pub const HeaderTypes = []enum { +// song, +// album, +// artist, +// artistlist, +// scrobbles, +// date, +//}; From 365b9dbf11345af987960b92d58148769ccdbe43 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 13 May 2025 14:24:14 -0400 Subject: [PATCH 036/103] Switch to using newtable partial for all tables Will be renamed eventually, don't care right now. Also cleans up a lot of code I wasn't particularly happy about --- build.zig.zon | 4 +-- src/app/views/albums.zig | 42 ++++++++++++++------------- src/app/views/artists.zig | 25 ++++++++++++---- src/app/views/artists/index.zmpl | 31 ++++---------------- src/app/views/partials/_newtable.zmpl | 28 ++++++++++-------- src/app/views/scrobbles.zig | 41 +++++++++++++------------- src/app/views/scrobbles/index.zmpl | 39 ++++--------------------- src/app/views/songs.zig | 32 ++++++++++---------- src/app/views/songs/index.zmpl | 37 ++++------------------- src/types.zig | 15 ++++++++++ 10 files changed, 128 insertions(+), 166 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index f2e5778..d7fb785 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -17,8 +17,8 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/a298192bb0cddf9c45d7d0d976a9852804457de2.tar.gz", - .hash = "jetzig-0.0.0-IpAgLf5aDwB4UOKMhIjtK22zBsPfbWEwCYgHT0hayyT-", + .url = "https://github.com/jetzig-framework/jetzig/archive/7be1d137fcab5c422e05f12092f6e04a02900d6f.tar.gz", + .hash = "jetzig-0.0.0-IpAgLURbDwB1NlywUH7lnQ3zptNvSQWVosaA1k7l1cNz", }, .zeit = .{ .url = "https://github.com/rockorager/zeit/archive/refs/tags/v0.6.0.tar.gz", diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 1264dac..95f5b86 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -1,6 +1,8 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; +const TableRow = @import("../../types.zig").TableRows; +const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -22,28 +24,26 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; var prev_album_id: ?i32 = null; - var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; + + var row: ?TableRow = null; + var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); if (album.id == prev_album_id) { - var artist_info = try prev_artist_infos.?.append(.object); - try artist_info.put("name", album.artist_name); - try artist_info.put("id", album.artist_id); + try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); continue :blk; + } else { + try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); + + row = TableRow{ + .album = .{ .name = album.name, .id = album.id }, + .artistlist = try artistlist.toOwnedSlice(), + .scrobbles = album.scrobbles, + }; + + try albums_view.append(row); } - var album_view = try albums_view.append(.object); - - try album_view.put("name", album.name); - try album_view.put("id", album.id); - try album_view.put("scrobbles", album.scrobbles); - - var artist_infos = try album_view.put("artist_info", .array); - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", album.artist_name); - try artist_info.put("id", album.artist_id); - - prev_artist_infos = artist_infos; prev_album_id = album.id; } return request.render(.ok); @@ -72,10 +72,12 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { while (try songs_js_result.postgresql.result.next()) |song_row| { const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); - var song_view = try songs_view.append(.object); - try song_view.put("name", song.name); - try song_view.put("id", song.id); - try song_view.put("scrobbles", song.scrobbles); + const row = TableRow{ + .song = .{ .name = song.name, .id = song.id }, + .scrobbles = song.scrobbles, + }; + + try songs_view.append(row); } return request.render(.ok); } diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 34cea3e..461235a 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -1,6 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; +const TableRow = @import("../../types.zig").TableRows; const dateFmt = @import("../../date_fmt.zig").dateFmt; const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; @@ -25,10 +26,12 @@ pub fn index(request: *jetzig.Request) !jetzig.View { while (try artists_jq_result.postgresql.result.next()) |artist_row| { const artist = try artist_row.to(Artist, .{ .dupe = true, .allocator = request.allocator }); - var artist_view = try artists_view.append(.object); - try artist_view.put("name", artist.name); - try artist_view.put("url", artist.id); - try artist_view.put("scrobbles", artist.scrobbles); + const row = TableRow{ + .artist = .{ .name = artist.name, .id = artist.id }, + .scrobbles = artist.scrobbles, + }; + + try artists_view.append(row); } return request.render(.ok); @@ -84,7 +87,12 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - try albums_view.append(album); + const album_table_row = TableRow{ + .album = .{ .name = album.name, .id = album.id }, + .scrobbles = album.scrobbles, + }; + + try albums_view.append(album_table_row); } //albums_jq_result.drain(); @@ -107,7 +115,12 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { while (try appears_jq_result.postgresql.result.next()) |appears_row| { const appears = try appears_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - try appears_view.append(appears); + const appears_table_row = TableRow{ + .album = .{ .name = appears.name, .id = appears.id }, + .scrobbles = appears.scrobbles, + }; + + try appears_view.append(appears_table_row); } //appears_jq_result.drain(); diff --git a/src/app/views/artists/index.zmpl b/src/app/views/artists/index.zmpl index 6854e07..0648c91 100644 --- a/src/app/views/artists/index.zmpl +++ b/src/app/views/artists/index.zmpl @@ -1,34 +1,15 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.artist, .scrobbles}; +} + - @partial partials/header

Artists

-
SongAlbumArtistArtist(s){{h}}ScrobblesDate
- {{data.name}} + {{ent.name}} - {{data.name}} - - @for (data.get("artist_info").?) |ai| { - {{ai.name}} + @for (ent.get("artist_info").?) |artist| { + {{artist.name}} } {{data.scrobbles}}{{ent.scrobbles}}{{data.date}}{{ent.date}}
- - - - - - - -@for (.artists) |artist| { - - - - -} - -
NameScrobbles
{{artist.name}}{{artist.scrobbles}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .artists, columns: columns) \ No newline at end of file diff --git a/src/app/views/partials/_newtable.zmpl b/src/app/views/partials/_newtable.zmpl index 8960e21..6746a2a 100644 --- a/src/app/views/partials/_newtable.zmpl +++ b/src/app/views/partials/_newtable.zmpl @@ -32,33 +32,37 @@ @zig { const array = table_data.items(.array); - for (array) |ent| { + for (array) |row| { for (columns) |header| { switch (header) { - .song, .album, .artist => { - const path = switch (header) { - .song => "songs", - .album => "albums", - .artist => "artists", - else => unreachable - }; + .song => { - {{ent.name}} + {{row.song.name}} + + }, + .album => { + + {{row.album.name}} + + }, + .artist => { + + {{row.artist.name}} }, .artistlist => { - @for (ent.get("artist_info").?) |artist| { + @for (row.get("artistlist").?) |artist| { {{artist.name}} } }, .scrobbles => { - {{ent.scrobbles}} + {{row.scrobbles}} }, .date =>{ - {{ent.date}} + {{row.date}} } } } diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 1386a83..9403fa9 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -1,6 +1,8 @@ const std = @import("std"); const jetzig = @import("jetzig"); const zeit = @import("zeit"); +const TableRow = @import("../../types.zig").TableRows; +const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -23,33 +25,30 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Scrobble = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; var prev_s_id: ?i32 = null; - var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; + + var row: ?TableRow = null; + var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); blk: while (try scrobbles_jq_result.postgresql.result.next()) |scrobble_row| { const scrobble = try scrobble_row.to(Scrobble, .{ .dupe = true, .allocator = request.allocator }); if (scrobble.s_id == prev_s_id) { - var artist_info = try prev_artist_infos.?.append(.object); - try artist_info.put("name", scrobble.artist_name); - try artist_info.put("url", scrobble.artist_id); + try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); continue :blk; + } else { + var date = std.ArrayList(u8).init(request.allocator); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(scrobble.date, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + + try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); + + row = TableRow{ + .song = .{ .name = scrobble.song_name, .id = scrobble.song_id }, + .album = .{ .name = scrobble.album_name, .id = scrobble.album_id }, + .artistlist = try artistlist.toOwnedSlice(), + .date = date.items, + }; + + try scrobbles_view.append(row); } - // Not appending the scrobble directly because we don't want the unix timestamp or scrobble id - var scrobble_view = try scrobbles_view.append(.object); - - var artist_infos = try scrobble_view.put("artist_info", .array); - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", scrobble.artist_name); - try artist_info.put("url", scrobble.artist_id); - - try scrobble_view.put("song_name", scrobble.song_name); - try scrobble_view.put("song_id", scrobble.song_id); - try scrobble_view.put("album_name", scrobble.album_name); - try scrobble_view.put("album_id", scrobble.album_id); - var date = std.ArrayList(u8).init(request.allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(scrobble.date, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); - try scrobble_view.put("date", date.items); - - prev_artist_infos = artist_infos; prev_s_id = scrobble.s_id; } diff --git a/src/app/views/scrobbles/index.zmpl b/src/app/views/scrobbles/index.zmpl index b298cb6..c3d6759 100644 --- a/src/app/views/scrobbles/index.zmpl +++ b/src/app/views/scrobbles/index.zmpl @@ -1,42 +1,15 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; +} + - @partial partials/header

Scrobbles

- - - - - - - - - - -@for (.scrobbles) |scrobble| { - - - - - - -} - -
SongArtist(s)AlbumDate
{{scrobble.song_name}} - @for (scrobble.get("artist_info").?) |ai| { - {{ai.name}} - } - {{scrobble.album_name}}{{scrobble.date}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) \ No newline at end of file diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index cd220f8..5302906 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -1,5 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); +const TableRow = @import("../../types.zig").TableRows; +const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -22,28 +24,26 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Song = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; var prev_song_id: ?i32 = null; - var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; + + var row: ?TableRow = null; + var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); blk: while (try songs_js_result.postgresql.result.next()) |song_row| { const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); if (song.id == prev_song_id) { - var artist_info = try prev_artist_infos.?.append(.object); - try artist_info.put("name", song.artist_name); - try artist_info.put("url", song.artist_id); + try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); continue :blk; + } else { + try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); + + row = TableRow{ + .song = .{ .name = song.name, .id = song.id }, + .artistlist = try artistlist.toOwnedSlice(), + .scrobbles = song.scrobbles, + }; + + try songs_view.append(row); } - var song_view = try songs_view.append(.object); - - var artist_infos = try song_view.put("artist_info", .array); - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", song.artist_name); - try artist_info.put("url", song.artist_id); - - try song_view.put("name", song.name); - try song_view.put("url", song.id); - try song_view.put("scrobbles", song.scrobbles); - - prev_artist_infos = artist_infos; prev_song_id = song.id; } return request.render(.ok); diff --git a/src/app/views/songs/index.zmpl b/src/app/views/songs/index.zmpl index 181da87..6964630 100644 --- a/src/app/views/songs/index.zmpl +++ b/src/app/views/songs/index.zmpl @@ -1,40 +1,15 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .scrobbles}; +} + - @partial partials/header

Songs

- - - - - - - - - -@for (.songs) |song| { - - - - - -} - -
NameArtist(s)Scrobbles
{{song.name}} - @for (song.get("artist_info").?) |ai| { - {{ai.name}} - } - {{song.scrobbles}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) \ No newline at end of file diff --git a/src/types.zig b/src/types.zig index 455f282..2a82886 100644 --- a/src/types.zig +++ b/src/types.zig @@ -72,3 +72,18 @@ pub const Rules = struct { // scrobbles, // date, //}; +// + +pub const TableRows = struct { + song: ?HyperlinkData = null, + album: ?HyperlinkData = null, + artist: ?HyperlinkData = null, + artistlist: ?[]HyperlinkData = null, + scrobbles: ?i64 = null, + date: ?[]const u8 = null, +}; + +pub const HyperlinkData = struct { + name: []const u8, + id: i32, +}; From c42b8d24dd0c71afe5b52c7fe0c554331da08dd3 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 15:37:21 -0400 Subject: [PATCH 037/103] Fix typo --- src/app/views/albums.zig | 2 +- src/app/views/artists.zig | 2 +- src/app/views/scrobbles.zig | 2 +- src/app/views/songs.zig | 2 +- src/types.zig | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 95f5b86..9c6063e 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -1,7 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 461235a..9fd9dcd 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -1,7 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const dateFmt = @import("../../date_fmt.zig").dateFmt; const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 9403fa9..321af90 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -1,7 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const zeit = @import("zeit"); -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 5302906..f91b5ed 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -1,6 +1,6 @@ const std = @import("std"); const jetzig = @import("jetzig"); -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { diff --git a/src/types.zig b/src/types.zig index 2a82886..e71018c 100644 --- a/src/types.zig +++ b/src/types.zig @@ -74,7 +74,7 @@ pub const Rules = struct { //}; // -pub const TableRows = struct { +pub const TableRow = struct { song: ?HyperlinkData = null, album: ?HyperlinkData = null, artist: ?HyperlinkData = null, From 4991bac9a452a6f07cefbc844d1bc070adf54800 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 15:39:21 -0400 Subject: [PATCH 038/103] Add LastFM scrobble type In preparation for importing via LastFM api --- src/app/views/upload.zig | 2 +- src/types.zig | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 5f47cc4..25a9128 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -37,7 +37,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { switch (source) { 0 => { - const content: Data.LastFM = try std.json.parseFromSliceLeaky(Data.LastFM, request.allocator, file.content, .{}); + const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; appends: for (content.scrobbles) |scrobble| { diff --git a/src/types.zig b/src/types.zig index e71018c..e8ac141 100644 --- a/src/types.zig +++ b/src/types.zig @@ -14,7 +14,7 @@ pub const Scrobble = struct { }; // From lastfmstats.com -pub const LastFM = struct { username: []const u8, scrobbles: []ImportedScrobble }; +pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble }; // I derived whether or not these values were optional from searching // the respective fields for null in Vim, so there may be some fields @@ -43,6 +43,37 @@ pub const SpotifyScrobble = struct { //incognito_mode: ?bool, }; +pub const LastFMWeb = struct { + track: []struct { + artist: LastFMWebHyperlinkData, + album: LastFMWebHyperlinkData, + name: []const u8, + mbid: []const u8, + image: struct { + size: []const u8, + @"#text": []const u8, + }, + date: struct { + uts: []const u8, + @"#text": []const u8, + }, + }, + @"@attr": LastFMWebAttr, +}; + +pub const LastFMWebAttr = struct { + perPage: u32, + totalPages: u32, + page: u32, + user: []const u8, + total: u32, +}; + +pub const LastFMWebHyperlinkData = struct { + mbid: []const u8, + @"#text": []const u8, +}; + pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, From 52fefc9ba5b28649c0bfeb926cef4e8d0b77fa27 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:22:09 -0400 Subject: [PATCH 039/103] Create dateCompare function Will eventually try to move away from zeit. Don't need all of it's functionality as long as SQL can format dates --- src/date_fmt.zig | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 603a209..00fe93b 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -1,8 +1,35 @@ const std = @import("std"); const zeit = @import("zeit"); +const Data = @import("types.zig"); pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } + +pub fn dateCompare(self: *[]const u8, earliest: []const u8, latest: []const u8) bool { + const range = if (self.len == 19) 4 else 2; + const time = std.fmt.parseInt(u32, self[0..range], 10) catch 0; + const etime = std.fmt.parseInt(u32, earliest[0..range], 10) catch 0; + const ltime = std.fmt.parseInt(u32, latest[0..range], 10) catch 0; + + if (time != etime and time != ltime) { + return (time > etime and time < ltime); + } else { + return dateCompare(self[range + 1 ..], earliest[range + 1 ..], latest[range + 1 ..]); + } +} + +pub fn scrobbleToRow(allocator: std.mem.Allocator, scrobble: Data.Scrobble) !Data.TableRow { + var artistlist = std.ArrayList(Data.HyperlinkData).init(allocator); + for (scrobble.artists_track) |a| { + try artistlist.append(Data.HyperlinkData{ .name = a, .id = 0 }); + } + return Data.TableRow{ + .song = .{ .name = scrobble.track, .id = 0 }, + .artistlist = artistlist.items, + .album = .{ .name = scrobble.album, .id = 0 }, + .date = try dateFmt(allocator, scrobble.date), + }; +} From f69ffb2b373f6d21647b7550147bed5dc66ba0db Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:22:34 -0400 Subject: [PATCH 040/103] Move upload.zig to the new table partial --- src/app/views/upload/post.zmpl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/app/views/upload/post.zmpl b/src/app/views/upload/post.zmpl index 176f094..91c5347 100644 --- a/src/app/views/upload/post.zmpl +++ b/src/app/views/upload/post.zmpl @@ -1,15 +1,16 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; +} + - @partial partials/header

File Uploaded Successfully

-

Scrobbles Added

- -@partial partials/table(table_data: .scrobbles, table_headers: .upload_table, table_context: .context) - +@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) \ No newline at end of file From 89e98c7a47f907c507df88cd74d429090b1de26a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:23:12 -0400 Subject: [PATCH 041/103] Allow uploads from LastFM API Very slow at the moment. Look into ways to speed this up --- src/app/views/upload.zig | 313 ++++++++++++++++++++------------ src/app/views/upload/index.zmpl | 7 +- 2 files changed, 205 insertions(+), 115 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 25a9128..220e2c9 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -4,6 +4,8 @@ const jetquery = @import("jetzig").jetquery; const zeit = @import("zeit"); const rules = @import("../../apply_rule.zig"); const Data = @import("../../types.zig"); +const Utils = @import("../../date_fmt.zig"); +const Client = std.http.Client; pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { _ = data; @@ -13,99 +15,152 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - if (try request.file("upload")) |file| { - const params = try request.params(); - const source = try std.fmt.parseInt(u8, params.get("t").?.string.value, 10); // This param is required in HTML - const before_limiter: bool = if (params.get("bbool")) |_| true else false; - const after_limiter: bool = if (params.get("abool")) |_| true else false; + const params = try request.params(); + const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), + else => err, + }); - var scrobbles_view = try root.put("scrobbles", .array); - var job = try request.job("process_scrobbles"); - var scrobbles_data = try job.params.put("scrobbles", .array); + defer rule_file.close(); + const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); + const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + var job = try request.job("process_scrobbles"); + const source = params.getT(.integer, "t").?; // This param is required in HTML + const before_limiter: bool = if (params.get("bbool")) |_| true else false; + const after_limiter: bool = if (params.get("abool")) |_| true else false; - var skipped_tracks: u64 = 0; - var limited_tracks: u64 = 0; + var scrobbles_view = try root.put("scrobbles", .array); + var scrobbles_data = try job.params.put("scrobbles", .array); - const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), - else => err, - }); + var skipped_tracks: u64 = 0; + var limited_tracks: u64 = 0; - defer rule_file.close(); - const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + switch (source) { + 0, 1 => { + if (try request.file("upload")) |file| { + std.log.debug("{s}", .{file.filename}); + switch (source) { + 0 => { + const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); + const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; + const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; + appends: for (content.scrobbles) |scrobble| { + // We can short-circuit on the limiter bools + if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; - switch (source) { - 0 => { - const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); - const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; - const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; - appends: for (content.scrobbles) |scrobble| { - // We can short-circuit on the limiter bools - if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, scrobble, rl) + else + Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date, + }; - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, scrobble, rl) - else - Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date, - }; + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - var scrobble_view = try scrobbles_view.append(.object); - var artists = try scrobble_view.put("artists", .array); + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); - try scrobble_view.put("track", formatted_scrobble.track); - try scrobble_view.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_track) |artist| { - try artists.append(artist); - } - try scrobble_view.put("date", formatted_scrobble.date); + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } - var scrobble_data = try scrobbles_data.append(.object); - var artists_album = try scrobble_data.put("artists_album", .array); - var artists_track = try scrobble_data.put("artists_track", .array); + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + }, + 1 => { + const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); + const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time(); + const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); + appends: for (content) |scrobble| { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_album) |artist| { - try artists_album.append(artist); - } + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { + limited_tracks += 1; + continue :appends; + } - for (formatted_scrobble.artists_track) |artist| { - try artists_track.append(artist); - } - try scrobble_data.put("date", formatted_scrobble.date); + const pre_formatted_scrobble: Data.ImportedScrobble = .{ + .track = scrobble.master_metadata_track_name.?, + .album = scrobble.master_metadata_album_album_name.?, + .artist = scrobble.master_metadata_album_artist_name.?, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(), + }; + + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; + + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + }, + else => unreachable, } - }, - 1 => { - const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); - const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else (try zeit.instant(.{})).time(); - const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); - appends: for (content) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } + try job.schedule(); + std.log.debug("Skipped {} tracks", .{skipped_tracks}); + std.log.debug("Filtered {} tracks", .{limited_tracks}); + } + }, + 2 => { + if (params.getT(.string, "username")) |username| { + _ = username; + const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&format=json"; + const user_agent: []const u8 = "Zuletzt/0.0.1"; + var client = Client{ .allocator = request.allocator }; + var ar = std.ArrayList(u8).init(request.allocator); + _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + const first_response = try ar.toOwnedSlice(); + const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { - limited_tracks += 1; - continue :appends; - } - - const pre_formatted_scrobble: Data.ImportedScrobble = .{ - .track = scrobble.master_metadata_track_name.?, - .album = scrobble.master_metadata_album_album_name.?, - .artist = scrobble.master_metadata_album_artist_name.?, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000, + for (json.recenttracks.track) |scrobble| { + const pre_formatted_scrobble = Data.ImportedScrobble{ + .track = scrobble.name, + .album = if (scrobble.album) |album| album.@"#text".? else "", + .artist = scrobble.artist.@"#text".?, + .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), }; const formatted_scrobble = if (rule_list) |rl| @@ -119,44 +174,76 @@ pub fn post(request: *jetzig.Request) !jetzig.View { .date = pre_formatted_scrobble.date, }; - var scrobble_view = try scrobbles_view.append(.object); - var artists = try scrobble_view.put("artists", .array); - - try scrobble_view.put("track", formatted_scrobble.track); - try scrobble_view.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_track) |artist| { - try artists.append(artist); - } - try scrobble_view.put("date", formatted_scrobble.date); + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); var scrobble_data = try scrobbles_data.append(.object); - var artists_album = try scrobble_data.put("artists_album", .array); - var artists_track = try scrobble_data.put("artists_track", .array); - - try scrobble_data.put("track", formatted_scrobble.track); try scrobble_data.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_album) |artist| { - try artists_album.append(artist); - } - - for (formatted_scrobble.artists_track) |artist| { - try artists_track.append(artist); - } + try scrobble_data.put("track", formatted_scrobble.track); try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } } - }, - else => unreachable, - } - try job.schedule(); - std.log.debug("Skipped {} tracks", .{skipped_tracks}); - std.log.debug("Filtered {} tracks", .{limited_tracks}); + const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; + for (2..max_pages) |page| { + const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&page={}&format=json", .{page}); + std.log.debug("{s}", .{rest_query}); + _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + const response = try ar.toOwnedSlice(); + const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); + + for (json2.recenttracks.track) |scrobble| { + const pre_formatted_scrobble = Data.ImportedScrobble{ + .track = scrobble.name, + .album = if (scrobble.album) |album| album.@"#text".? else "", + .artist = scrobble.artist.@"#text".?, + .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), + }; + + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; + + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + } + try job.schedule(); + } + }, + else => unreachable, } - - var upload_table = try root.put("upload_table", .array); - try upload_table.append("Track"); - try upload_table.append("Artist"); - try upload_table.append("Album"); - try upload_table.append("Date"); - return request.render(.created); } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 9043a5b..31151e1 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -9,11 +9,14 @@
- - + + + +
Last.fm Spotify + Last.fm (WebAuth) Limit to Scrobbles before: Limit to Scrobbles after:
From 5697f95355d1e8f240eef9a8c9dbfbba3b10a001 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:23:53 -0400 Subject: [PATCH 042/103] Fix LastFM uploading errors Not sure which of these actually made it work, will probably work backwards at some point to reverse engineer it --- src/types.zig | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/types.zig b/src/types.zig index e8ac141..898943a 100644 --- a/src/types.zig +++ b/src/types.zig @@ -44,34 +44,40 @@ pub const SpotifyScrobble = struct { }; pub const LastFMWeb = struct { - track: []struct { - artist: LastFMWebHyperlinkData, - album: LastFMWebHyperlinkData, - name: []const u8, - mbid: []const u8, - image: struct { - size: []const u8, - @"#text": []const u8, - }, - date: struct { - uts: []const u8, - @"#text": []const u8, + recenttracks: struct { + track: []struct { + artist: LastFMWebHyperlinkData, + album: ?LastFMWebHyperlinkData = null, + name: []const u8, + mbid: ?[]const u8 = null, + image: ?[]struct { + size: []const u8, + @"#text": []const u8, + } = null, + date: struct { + uts: []const u8, + @"#text": []const u8, + }, + @"@attr": ?struct { + nowplaying: ?[]const u8 = null, + } = null, + url: ?[]const u8 = null, }, + @"@attr": LastFMWebAttr, }, - @"@attr": LastFMWebAttr, }; pub const LastFMWebAttr = struct { - perPage: u32, - totalPages: u32, - page: u32, - user: []const u8, - total: u32, + perPage: ?[]const u8 = null, + totalPages: []const u8, + page: ?[]const u8 = null, + user: ?[]const u8 = null, + total: ?[]const u8 = null, }; pub const LastFMWebHyperlinkData = struct { - mbid: []const u8, - @"#text": []const u8, + mbid: ?[]const u8 = null, + @"#text": ?[]const u8 = null, }; pub const Rule = struct { From 614607ae7167d986aa5be71e78ccf439ef288f6e Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 16 May 2025 01:05:32 -0400 Subject: [PATCH 043/103] Fix LastFM uploadig error I figured it out; if you have a song currently being played, then it doesn't have a date --- src/app/views/upload.zig | 22 ++++++++++++---------- src/types.zig | 26 +++++++++++++------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 220e2c9..2582ff4 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -147,7 +147,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { 2 => { if (params.getT(.string, "username")) |username| { _ = username; - const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&format=json"; + const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json"; const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var ar = std.ArrayList(u8).init(request.allocator); @@ -155,12 +155,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const first_response = try ar.toOwnedSlice(); const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - for (json.recenttracks.track) |scrobble| { + appends: for (json.recenttracks.track) |scrobble| { + if (scrobble.date == null) continue :appends; const pre_formatted_scrobble = Data.ImportedScrobble{ .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text".? else "", - .artist = scrobble.artist.@"#text".?, - .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artist = scrobble.artist.@"#text", + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), }; const formatted_scrobble = if (rule_list) |rl| @@ -195,18 +196,19 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; for (2..max_pages) |page| { - const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&page={}&format=json", .{page}); + const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page}); std.log.debug("{s}", .{rest_query}); _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); const response = try ar.toOwnedSlice(); const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); - for (json2.recenttracks.track) |scrobble| { + appends: for (json2.recenttracks.track) |scrobble| { + if (scrobble.date == null) continue :appends; const pre_formatted_scrobble = Data.ImportedScrobble{ .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text".? else "", - .artist = scrobble.artist.@"#text".?, - .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artist = scrobble.artist.@"#text", + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), }; const formatted_scrobble = if (rule_list) |rl| diff --git a/src/types.zig b/src/types.zig index 898943a..202bce6 100644 --- a/src/types.zig +++ b/src/types.zig @@ -50,34 +50,34 @@ pub const LastFMWeb = struct { album: ?LastFMWebHyperlinkData = null, name: []const u8, mbid: ?[]const u8 = null, - image: ?[]struct { + image: []struct { size: []const u8, @"#text": []const u8, - } = null, - date: struct { + }, + date: ?struct { uts: []const u8, @"#text": []const u8, - }, - @"@attr": ?struct { - nowplaying: ?[]const u8 = null, } = null, - url: ?[]const u8 = null, + @"@attr": ?struct { + nowplaying: []const u8, + } = null, + url: []const u8, }, @"@attr": LastFMWebAttr, }, }; pub const LastFMWebAttr = struct { - perPage: ?[]const u8 = null, + perPage: []const u8, totalPages: []const u8, - page: ?[]const u8 = null, - user: ?[]const u8 = null, - total: ?[]const u8 = null, + page: []const u8, + user: []const u8, + total: []const u8, }; pub const LastFMWebHyperlinkData = struct { - mbid: ?[]const u8 = null, - @"#text": ?[]const u8 = null, + mbid: []const u8, + @"#text": []const u8, }; pub const Rule = struct { From 4c759433d25f2ec6e961eafd505c8d163e51a29d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 16 May 2025 05:06:59 +0000 Subject: [PATCH 044/103] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f18e7e0..8247b21 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Licensed under MIT. - [ ] Import from Discogs[^2] - [ ] Import listening history - [x] From Lastfmstats.com (.json file)[^3] - - [ ] From Last.fm (authentication) + - [x] From Last.fm (authentication) - [x] From Spotify (.json file) - [ ] From other streaming services[^4] - [ ] "Unofficial scrobbles"[^9] From 6494bbdf6085a12464f3dc1f455fce1d5dcf0e02 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 20 May 2025 09:33:15 -0400 Subject: [PATCH 045/103] Remove Rules type --- src/app/jobs/process_rule.zig | 11 +++++------ src/types.zig | 4 ---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 7b60d54..6beb880 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -16,7 +16,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig return; }, }; - const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; + const out_rules = &[_]Data.Rule{rule}; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file.writeAll(out); file.close(); @@ -35,18 +35,17 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); if (file_content.len == 0) { - const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; + const out_rules = &[_]Data.Rule{rule}; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file_write.writeAll(out); file_write.close(); return; } - const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); - try rules.appendSlice(content.rules); + const content: []Data.Rule = try std.json.parseFromSliceLeaky([]Data.Rule, allocator, file_content, .{}); + try rules.appendSlice(content); try rules.append(rule); - const out_rules = Data.Rules{ .rules = rules.items }; - const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + const out = try std.json.stringifyAlloc(allocator, rules.items, .{}); try file_write.writeAll(out); file_write.close(); diff --git a/src/types.zig b/src/types.zig index 202bce6..073b2d3 100644 --- a/src/types.zig +++ b/src/types.zig @@ -95,10 +95,6 @@ pub const Rule = struct { }, }; -pub const Rules = struct { - rules: []const Rule, -}; - // Can't import types in .zmpl files, so defining this here // doesn't really do much (except maybe in the .zig file for views?) //pub const HeaderTypes = []enum { From a2a739bc9c40b5f1a1bafe66408d43d76c2cf1fa Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 20 May 2025 15:07:51 -0400 Subject: [PATCH 046/103] Refactor upload.zig I have been unhappy with the branches, but didn't quite know what to do about it. THis feels much nicer. Also fixes datetime stuff with jetquery. The HTML element parsing isn't quite where I want it to be, but it works for the time being. --- src/app/jobs/process_scrobbles.zig | 1 - src/app/views/upload.zig | 330 +++++++++++------------------ src/app/views/upload/index.zmpl | 6 +- src/apply_rule.zig | 26 ++- src/date_fmt.zig | 2 +- src/types.zig | 77 ++++--- 6 files changed, 184 insertions(+), 258 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index c6b25b1..dcf7f1f 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -1,7 +1,6 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const lastfm = @import("../../types.zig").LastFM; const Data = @import("../../types.zig"); const rules = @import("../../apply_rule.zig"); diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 2582ff4..c86e6f0 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -23,229 +23,141 @@ pub fn post(request: *jetzig.Request) !jetzig.View { defer rule_file.close(); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; var job = try request.job("process_scrobbles"); const source = params.getT(.integer, "t").?; // This param is required in HTML - const before_limiter: bool = if (params.get("bbool")) |_| true else false; - const after_limiter: bool = if (params.get("abool")) |_| true else false; + const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { + const date = params.getT(.string, "latest-date").?; + break :blk try zeit.Time.fromISO8601(date); + } else blk: { + break :blk (try zeit.instant(.{ .source = .now })).time(); + }; + const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { + const date = params.getT(.string, "earliest-date").?; + break :blk try zeit.Time.fromISO8601(date); + } else blk: { + break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); + }; - var scrobbles_view = try root.put("scrobbles", .array); - var scrobbles_data = try job.params.put("scrobbles", .array); + const earliest_timestamp = earliest_date.instant().unixTimestamp(); + const latest_timestamp = latest_date.instant().unixTimestamp(); + + var view_params = try root.put("scrobbles", .array); + var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - switch (source) { - 0, 1 => { - if (try request.file("upload")) |file| { - std.log.debug("{s}", .{file.filename}); - switch (source) { - 0 => { - const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); - const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; - const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; - appends: for (content.scrobbles) |scrobble| { - // We can short-circuit on the limiter bools - if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; + const imported_scrobbles: Data.ImportedScrobbles = switch (source) { + 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles }, + 1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) }, + 2 => blk: { + const user_agent: []const u8 = "Zuletzt/0.0.1"; + var client = Client{ .allocator = request.allocator }; + var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); + var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator); - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, scrobble, rl) - else - Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date, - }; + const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + var page: usize = 1; + var max_pages: ?usize = null; - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - }, - 1 => { - const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); - const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time(); - const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); - appends: for (content) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { - limited_tracks += 1; - continue :appends; - } - - const pre_formatted_scrobble: Data.ImportedScrobble = .{ - .track = scrobble.master_metadata_track_name.?, - .album = scrobble.master_metadata_album_album_name.?, - .artist = scrobble.master_metadata_album_artist_name.?, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - }, - else => unreachable, - } - try job.schedule(); - std.log.debug("Skipped {} tracks", .{skipped_tracks}); - std.log.debug("Filtered {} tracks", .{limited_tracks}); + while (true) : (page += 1) { + if (max_pages != null and page > max_pages.?) break; + const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); + const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + std.log.debug("{}", .{r}); + const response_string = try lastfm_response_buffer.toOwnedSlice(); + const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); + //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); + if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); + try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track); } - }, - 2 => { - if (params.getT(.string, "username")) |username| { - _ = username; - const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json"; - const user_agent: []const u8 = "Zuletzt/0.0.1"; - var client = Client{ .allocator = request.allocator }; - var ar = std.ArrayList(u8).init(request.allocator); - _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - const first_response = try ar.toOwnedSlice(); - const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - appends: for (json.recenttracks.track) |scrobble| { - if (scrobble.date == null) continue :appends; - const pre_formatted_scrobble = Data.ImportedScrobble{ - .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artist = scrobble.artist.@"#text", - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; - for (2..max_pages) |page| { - const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page}); - std.log.debug("{s}", .{rest_query}); - _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - const response = try ar.toOwnedSlice(); - const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); - - appends: for (json2.recenttracks.track) |scrobble| { - if (scrobble.date == null) continue :appends; - const pre_formatted_scrobble = Data.ImportedScrobble{ - .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artist = scrobble.artist.@"#text", - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - } - try job.schedule(); - } + break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items }; }, else => unreachable, + }; + + // Not sure if I should be proud or feel sick + switch (imported_scrobbles) { + inline else => |scrobbles| { + appends: for (scrobbles) |scrobble| { + const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) { + Data.IgnorantScrobble => { + if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { + limited_tracks += 1; + continue :appends; + } + break :blk Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date * 1_000, + }; + }, + Data.SpotifyScrobble => { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } + + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { + limited_tracks += 1; + continue :appends; + } + + break :blk Data.Scrobble{ + .album = scrobble.master_metadata_album_album_name.?, + .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .track = scrobble.master_metadata_track_name.?, + .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, + }; + }, + Data.LastFMWebScrobble => { + break :blk Data.Scrobble{ + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, + .track = scrobble.name, + .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, + }; + }, + else => unreachable, + }; + + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + + try view_params.append(row); + var job_param = try job_params.append(.object); + try job_param.put("album", complete_scrobble.album); + try job_param.put("track", complete_scrobble.track); + try job_param.put("date", complete_scrobble.date); + + var track_artists_array = try job_param.put("artists_track", .array); + for (complete_scrobble.artists_track) |a| { + try track_artists_array.append(a); + } + + var album_artists_array = try job_param.put("artists_album", .array); + for (complete_scrobble.artists_album) |a| { + try album_artists_array.append(a); + } + } + }, } + + std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); + try job.schedule(); + return request.render(.created); } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 31151e1..420e08c 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -17,8 +17,10 @@ Last.fm Spotify Last.fm (WebAuth) - Limit to Scrobbles before: - Limit to Scrobbles after: + + Advanced Options + Limit to Scrobbles before: + Limit to Scrobbles after: diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 6163e93..fc4c0bc 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const Rules = @import("./types.zig").Rules; +const Rule = @import("./types.zig").Rule; const Data = @import("./types.zig"); // Wrapper for containsAtLeast to make the switch below to work @@ -11,18 +11,10 @@ fn eqlWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.eql(u8, haystack, needle); } -pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) !Data.Scrobble { - const artists = try allocator.alloc([]const u8, 1); - artists[0] = scrobble.artist; - var output_scrobble = Data.Scrobble{ - .track = scrobble.track, - .artists_track = artists, - .album = scrobble.album, - .artists_album = artists, - .date = scrobble.date, - }; +pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble, rules: []Rule) !Data.Scrobble { + var output_scrobble = scrobble; - for (rules.rules) |rule| { + for (rules) |rule| { var match_found: bool = switch (rule.cond_req) { .any => false, .all => true, @@ -34,10 +26,16 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc }; switch (rule.cond_req) { .any => switch (cond.match_on) { - inline else => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .album, .track => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .artists_album, .artists_track => |on| { + for (@field(scrobble, @tagName(on))) |artist| match_found = match_found or match_fn(artist, cond.match_txt); + }, }, .all => switch (cond.match_on) { - inline else => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .album, .track => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .artists_album, .artists_track => |on| { + for (@field(scrobble, @tagName(on))) |artist| match_found = match_found and match_fn(artist, cond.match_txt); + }, }, } } diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 00fe93b..7e44da7 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -4,7 +4,7 @@ const Data = @import("types.zig"); pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } diff --git a/src/types.zig b/src/types.zig index 073b2d3..3b2e4ff 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,7 +1,20 @@ -pub const ImportedScrobble = struct { +pub const ImportedScrobbles = union(ScrobbleSources) { + LastFMStats: []IgnorantScrobble, + LastFMWeb: []LastFMWebScrobble, + Spotify: []SpotifyScrobble, +}; + +const ScrobbleSources = enum { + LastFMStats, + LastFMWeb, + Spotify, +}; + +pub const IgnorantScrobble = struct { track: []const u8, artist: []const u8, - album: []const u8 = "", + album: []const u8 = "Not Provided", + //albumId: []const u8, date: i64, }; @@ -14,7 +27,7 @@ pub const Scrobble = struct { }; // From lastfmstats.com -pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble }; +pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble }; // I derived whether or not these values were optional from searching // the respective fields for null in Vim, so there may be some fields @@ -45,46 +58,48 @@ pub const SpotifyScrobble = struct { pub const LastFMWeb = struct { recenttracks: struct { - track: []struct { - artist: LastFMWebHyperlinkData, - album: ?LastFMWebHyperlinkData = null, - name: []const u8, - mbid: ?[]const u8 = null, - image: []struct { - size: []const u8, - @"#text": []const u8, - }, - date: ?struct { - uts: []const u8, - @"#text": []const u8, - } = null, - @"@attr": ?struct { - nowplaying: []const u8, - } = null, - url: []const u8, - }, - @"@attr": LastFMWebAttr, + track: []LastFMWebScrobble, + @"@attr": LastFMWebQueryInfo, }, }; -pub const LastFMWebAttr = struct { - perPage: []const u8, - totalPages: []const u8, - page: []const u8, - user: []const u8, - total: []const u8, -}; - pub const LastFMWebHyperlinkData = struct { mbid: []const u8, @"#text": []const u8, }; +pub const LastFMWebScrobble = struct { + artist: LastFMWebHyperlinkData, + album: ?LastFMWebHyperlinkData = null, + name: []const u8, + mbid: ?[]const u8 = null, + image: []struct { + size: []const u8, + @"#text": []const u8, + }, + date: ?struct { + uts: []const u8, + @"#text": []const u8, + } = null, + @"@attr": ?struct { + nowplaying: []const u8, + } = null, + url: []const u8, +}; + +pub const LastFMWebQueryInfo = struct { + perPage: []const u8, + totalPages: []const u8, + page: []const u8, + user: []const u8, + total: []const u8, +}; + pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, conditionals: []struct { - match_on: enum { artist, album, track }, + match_on: enum { artists_album, artists_track, album, track }, match_cond: enum { is, contains }, match_txt: []const u8, }, From 1734e6a4bbf83062c56dc5a4c50687cbf665c859 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 20 May 2025 16:29:53 -0400 Subject: [PATCH 047/103] Fix date formatting in scrobble view --- src/app/views/scrobbles.zig | 6 +++--- src/app/views/upload.zig | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 321af90..3fe946a 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -3,6 +3,7 @@ const jetzig = @import("jetzig"); const zeit = @import("zeit"); const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; +const Utils = @import("../../date_fmt.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -35,8 +36,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); continue :blk; } else { - var date = std.ArrayList(u8).init(request.allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(scrobble.date, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + const date = try Utils.dateFmt(request.allocator, scrobble.date); try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); @@ -44,7 +44,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { .song = .{ .name = scrobble.song_name, .id = scrobble.song_id }, .album = .{ .name = scrobble.album_name, .id = scrobble.album_id }, .artistlist = try artistlist.toOwnedSlice(), - .date = date.items, + .date = date, }; try scrobbles_view.append(row); diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index c86e6f0..af5d41d 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -66,7 +66,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { if (max_pages != null and page > max_pages.?) break; const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - std.log.debug("{}", .{r}); + std.log.debug("{}: {}", .{ page, r }); const response_string = try lastfm_response_buffer.toOwnedSlice(); const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); From 09f542e26e84b6a1e2dd9e5abed7a20d5547a7be Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 24 May 2025 13:58:31 -0400 Subject: [PATCH 048/103] Add timescale partial Bad name, idk what else to call it --- src/app/views/albums/get.zmpl | 4 ++++ src/app/views/artists/get.zmpl | 5 +++-- src/app/views/partials/_firstlast_listens.zmpl | 10 +++++++--- src/app/views/partials/_timescale.zmpl | 18 ++++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 src/app/views/partials/_timescale.zmpl diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 3a1641b..1024ec5 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -10,6 +10,10 @@ @partial partials/header

{{.album}}

+@partial partials/firstlast_listens(scrobbles: .album.scrobbles, rank: .album.rank, firstlast: .firstlast) +

Yearly Performance

+@partial partials/timescale(range: .yearly) +

Songs

@partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) \ No newline at end of file diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 48c1c25..69bd661 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -10,8 +10,9 @@ @partial partials/header

{{.artist.name}}

-@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, last_song: .last, first_song: .first) - +@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, firstlast: .firstlast) +

Yearly Performance

+@partial partials/timescale(range: .yearly)

Albums

@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl index 322a0e8..ed74361 100644 --- a/src/app/views/partials/_firstlast_listens.zmpl +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -1,9 +1,13 @@ -@args scrobbles: i64, rank: []const u8, last_song: *ZmplValue, first_song: *ZmplValue +@args scrobbles: i64, rank: []const u8, firstlast: *ZmplValue + +@zig { + const songs = firstlast.items(.array); +}
{{scrobbles}} scrobbles ({{rank}} place)
-First listen: {{first_song.name}} ({{first_song.date}}) +First listen: {{songs[0].name}} ({{songs[0].date}})
-Most recent listen: {{last_song.name}} ({{last_song.date}}) +Most recent listen: {{songs[1].name}} ({{songs[1].date}})
\ No newline at end of file diff --git a/src/app/views/partials/_timescale.zmpl b/src/app/views/partials/_timescale.zmpl new file mode 100644 index 0000000..fdec7a1 --- /dev/null +++ b/src/app/views/partials/_timescale.zmpl @@ -0,0 +1,18 @@ +@args range: *ZmplValue + + + + + + + + + +@for (range) |itm| { + + + + +} + +
YearScrobbles
{{itm.year}}:{{itm.scrobbles}}
\ No newline at end of file From 7f3778e82f74627bfe992fd03193acec06adf9ce Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 24 May 2025 13:59:28 -0400 Subject: [PATCH 049/103] Move SQL logic to separate function Idk if this makes any sense, and I don't really like the code atm, but the view .zig files lookk nicer? --- src/app/views/albums.zig | 72 ++------ src/app/views/artists.zig | 164 ++---------------- src/app/views/songs.zig | 44 +---- src/queries.zig | 343 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 370 insertions(+), 253 deletions(-) create mode 100644 src/queries.zig diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 9c6063e..af72456 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -3,81 +3,31 @@ const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var albums_view = try root.put("albums", .array); - const query = - \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN albums ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN artistalbums ON artistalbums.album_id = albums.id - \\INNER JOIN artists ON artists.id = artistalbums.artist_id - \\GROUP BY albums.id, artists.id - \\ORDER BY scrobbles DESC - ; - var albums_jq_result = try request.repo.executeSql(query, .{}); - defer albums_jq_result.deinit(); + const albums = try queries.entityQueryResult(request, queries.generateQuery(.album, .entities), .{}, .array); + try root.put("albums", albums); - const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; - - var prev_album_id: ?i32 = null; - - var row: ?TableRow = null; - var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); - - blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { - const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); - if (album.id == prev_album_id) { - try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); - continue :blk; - } else { - try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); - - row = TableRow{ - .album = .{ .name = album.name, .id = album.id }, - .artistlist = try artistlist.toOwnedSlice(), - .scrobbles = album.scrobbles, - }; - - try albums_view.append(row); - } - prev_album_id = album.id; - } return request.render(.ok); } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var songs_view = try root.put("songs", .array); - const album_name = try jetzig.database.Query(.Album).find(id).select(.{ .id, .name }).execute(request.repo); - _ = try root.put("album", album_name.?.name); - const query = - \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN songs ON albumsongs.song_id = songs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\WHERE albumsongs.album_id = $1 - \\GROUP BY songs.id - \\ORDER BY scrobbles DESC - ; + const album = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_info), .{id}, .object); + try root.put("album", album.get("entity_info")); - var songs_js_result = try request.repo.executeSql(query, .{id}); - defer songs_js_result.deinit(); + const songs = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_items), .{id}, .array); + try root.put("songs", songs); - const Song = struct { name: []const u8, id: i32, scrobbles: i64 }; + const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.album, .firstlast), .{id}, .array); + try root.put("firstlast", firstlast); - while (try songs_js_result.postgresql.result.next()) |song_row| { - const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); - const row = TableRow{ - .song = .{ .name = song.name, .id = song.id }, - .scrobbles = song.scrobbles, - }; + const timescale = try queries.entityQueryResult(request, queries.generateQuery(.album, .timescale), .{id}, .array); + try root.put("yearly", timescale); - try songs_view.append(row); - } return request.render(.ok); } diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 9fd9dcd..37ca155 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -4,35 +4,13 @@ const jetquery = @import("jetzig").jetquery; const TableRow = @import("../../types.zig").TableRow; const dateFmt = @import("../../date_fmt.zig").dateFmt; const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var artists_view = try root.put("artists", .array); + const artists = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entities), .{}, .array); - const query = - \\SELECT artists.name, artists.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongsartists - \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id - \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\GROUP BY artists.id - \\ORDER BY scrobbles DESC; - ; - - var artists_jq_result = try request.repo.executeSql(query, .{}); - defer artists_jq_result.deinit(); - - const Artist = struct { name: []const u8, id: i32, scrobbles: i64 }; - - while (try artists_jq_result.postgresql.result.next()) |artist_row| { - const artist = try artist_row.to(Artist, .{ .dupe = true, .allocator = request.allocator }); - const row = TableRow{ - .artist = .{ .name = artist.name, .id = artist.id }, - .scrobbles = artist.scrobbles, - }; - - try artists_view.append(row); - } + try root.put("artists", artists); return request.render(.ok); } @@ -40,136 +18,20 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const ArtistResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: i64 }; - const AlbumsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; - const ScrobbleResult = struct { name: []const u8, id: i32, date: i64 }; + const artist = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_info), .{id}, .object); + try root.put("artist", artist.get("entity_info")); - const artist_info_query = - \\SELECT * FROM - \\(SELECT *, ROW_NUMBER() OVER (ORDER BY scrobbles DESC) AS rank FROM - \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongsartists - \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id - \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\GROUP BY artists.id)) - \\WHERE id = $1 - ; + const albums = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_items), .{id}, .array); + try root.put("albums", albums); - var artist_jq_result = try request.repo.executeSql(artist_info_query, .{id}); - defer artist_jq_result.deinit(); + const appears = try queries.entityQueryResult(request, queries.generateQuery(.artist, .appears), .{id}, .array); + try root.put("appears", appears); - if (try artist_jq_result.postgresql.result.next()) |artist_row| { - const artist = try artist_row.to(ArtistResult, .{ .dupe = true, .allocator = request.allocator }); - var artist_view = try root.put("artist", .object); - try artist_view.put("name", artist.name); - try artist_view.put("id", artist.id); - try artist_view.put("scrobbles", artist.scrobbles); - try artist_view.put("rank", ordinalFmt(request.allocator, artist.rank)); - } - try artist_jq_result.drain(); + const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.artist, .firstlast), .{id}, .array); + try root.put("firstlast", firstlast); - const albums_query = - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles - \\FROM artistalbums - \\INNER JOIN albums ON albums.id = artistalbums.album_id - \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\WHERE artistalbums.artist_id = $1 - \\GROUP BY albums.id - \\ORDER BY scrobbles DESC - ; - - var albums_view = try root.put("albums", .array); - - var albums_jq_result = try request.repo.executeSql(albums_query, .{id}); - defer albums_jq_result.deinit(); - - while (try albums_jq_result.postgresql.result.next()) |album_row| { - const album = try album_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - const album_table_row = TableRow{ - .album = .{ .name = album.name, .id = album.id }, - .scrobbles = album.scrobbles, - }; - - try albums_view.append(album_table_row); - } - //albums_jq_result.drain(); - - const appears_query = - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles - \\FROM artistalbums - \\INNER JOIN albums ON albums.id = artistalbums.album_id - \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 AND artistalbums.artist_id != $1 - \\GROUP BY albums.id - \\ORDER BY scrobbles DESC; - ; - - var appears_view = try root.put("appears", .array); - - var appears_jq_result = try request.repo.executeSql(appears_query, .{id}); - defer appears_jq_result.deinit(); - - while (try appears_jq_result.postgresql.result.next()) |appears_row| { - const appears = try appears_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - const appears_table_row = TableRow{ - .album = .{ .name = appears.name, .id = appears.id }, - .scrobbles = appears.scrobbles, - }; - - try appears_view.append(appears_table_row); - } - - //appears_jq_result.drain(); - - const first_last_songs_query = - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date - \\FROM albumsongs - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 - \\ORDER BY scrobbles.datetime DESC - \\LIMIT 1) - \\ - \\UNION ALL - \\ - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date - \\FROM albumsongs - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 - \\ORDER BY scrobbles.datetime ASC - \\LIMIT 1) - ; - - var first_last_songs_jq_result = try request.repo.executeSql(first_last_songs_query, .{id}); - defer first_last_songs_jq_result.deinit(); - - // These look backwards, but it's correct this way - var last_song_view = try root.put("last", .object); - - if (try first_last_songs_jq_result.postgresql.result.next()) |last_song_row| { - const last_song = try last_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try last_song_view.put("name", last_song.name); - try last_song_view.put("id", last_song.id); - try last_song_view.put("date", (try dateFmt(request.allocator, last_song.date))); - } else unreachable; - - var first_song_view = try root.put("first", .object); - - if (try first_last_songs_jq_result.postgresql.result.next()) |first_song_row| { - const first_song = try first_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try first_song_view.put("name", first_song.name); - try first_song_view.put("id", first_song.id); - try first_song_view.put("date", (try dateFmt(request.allocator, first_song.date))); - } else unreachable; - - try first_last_songs_jq_result.drain(); + const timescale = try queries.entityQueryResult(request, queries.generateQuery(.artist, .timescale), .{id}, .array); + try root.put("yearly", timescale); return request.render(.ok); } diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index f91b5ed..d5a7944 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -1,51 +1,13 @@ const std = @import("std"); const jetzig = @import("jetzig"); -const TableRow = @import("../../types.zig").TableRow; -const HyperlinkData = @import("../../types.zig").HyperlinkData; +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var songs_view = try root.put("songs", .array); - const query = - \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN songs ON albumsongs.song_id = songs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\GROUP BY songs.id, artists.id - \\ORDER BY scrobbles DESC, songs.name ASC - ; + const songs = try queries.entityQueryResult(request, queries.generateQuery(.song, .entities), .{}, .array); + try root.put("songs", songs); - var songs_js_result = try request.repo.executeSql(query, .{}); - defer songs_js_result.deinit(); - - const Song = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; - - var prev_song_id: ?i32 = null; - - var row: ?TableRow = null; - var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); - - blk: while (try songs_js_result.postgresql.result.next()) |song_row| { - const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); - if (song.id == prev_song_id) { - try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); - continue :blk; - } else { - try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); - - row = TableRow{ - .song = .{ .name = song.name, .id = song.id }, - .artistlist = try artistlist.toOwnedSlice(), - .scrobbles = song.scrobbles, - }; - - try songs_view.append(row); - } - prev_song_id = song.id; - } return request.render(.ok); } diff --git a/src/queries.zig b/src/queries.zig new file mode 100644 index 0000000..cc5137b --- /dev/null +++ b/src/queries.zig @@ -0,0 +1,343 @@ +// Probably the worst code in the project +const jetzig = @import("jetzig"); +const TableRow = @import("types.zig").TableRow; +const HyperlinkData = @import("types.zig").HyperlinkData; +const std = @import("std"); + +pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: anytype, root_type: enum { object, array }) !*jetzig.Data.Value { + var result = try request.repo.executeSql(query.query, args); + + defer result.deinit(); + + var Data = jetzig.Data.init(request.allocator); + var out: *jetzig.Data.Value = switch (root_type) { + .array => try Data.array(), + .object => try Data.object(), + }; + + var artist_list = if (query.ResultType == EntitiesAlbumResult or query.ResultType == EntitiesArtistResult or query.ResultType == EntitiesScrobbleResult or query.ResultType == EntitiesSongResult) + std.ArrayList(HyperlinkData).init(request.allocator); + + blk: while (try result.postgresql.result.next()) |entity_row| { + switch (query.ResultType) { + FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult => |T| { + const entity = try entity_row.to(T, .{ .dupe = true, .allocator = request.allocator }); + const item: ?TableRow = switch (query.ResultType) { + EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { + .artist => TableRow{ .artist = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + .album => album_entities: { + const last_artist = artist_list.getLastOrNull(); + try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); + if (last_artist) |la| { + if (la.id == entity.artist_id) continue :blk; + } + break :album_entities TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; + }, + .song => song_entities: { + const last_artist = artist_list.getLastOrNull(); + try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); + if (last_artist) |la| { + if (la.id == entity.artist_id) continue :blk; + } + break :song_entities TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; + }, + else => unreachable, + }, + EntityItemsResult => switch (query.entity) { + .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + .album => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + else => unreachable, + }, + AppearsResult => switch (query.entity) { + .song, .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + .album, .scrobble => unreachable, + }, + else => null, + }; + + if (item) |itm| { + switch (root_type) { + .array => try out.append(itm), + .object => switch (query.query_type) { + inline else => |qt| try out.put(@tagName(qt), itm), + }, + } + } else { + switch (root_type) { + .array => try out.append(entity), + .object => switch (query.query_type) { + inline else => |qt| try out.put(@tagName(qt), entity), + }, + } + } + }, + else => unreachable, + } + } + if (root_type == .object) {} + return out; +} + +const GeneratedQuery = struct { + entity: EntityType, + query_type: QueryTypeEnum, + ResultType: type, + query: []const u8, +}; + +const EntityType = enum { scrobble, song, album, artist }; +const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info }; +const FirstlastResult = struct { name: []const u8, id: i32, date: []const u8 }; +const TimescaleResult = struct { year: []const u8, scrobbles: i64 }; +const EntitiesArtistResult = struct { name: []const u8, id: i32, scrobbles: i64 }; +const EntitiesAlbumResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; +const EntitiesSongResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; +const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; +const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; +const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; +const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; + +pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { + return GeneratedQuery{ + .entity = entity, + .query_type = query_type, + .ResultType = switch (query_type) { + .firstlast => FirstlastResult, + .timescale => TimescaleResult, + .entities => switch (entity) { + .scrobble => EntitiesScrobbleResult, + .song => EntitiesSongResult, + .album => EntitiesAlbumResult, + .artist => EntitiesArtistResult, + }, + .entity_items => EntityItemsResult, + .appears => AppearsResult, + .entity_info => EntityInfoResult, + }, + .query = switch (query_type) { + .firstlast => + //.ResultType = FirstlastResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime ASC + \\LIMIT 1) + \\ + \\UNION ALL + \\ + \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime DESC + \\LIMIT 1) + , + + .album => + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1 + \\ORDER BY scrobbles.datetime ASC + \\LIMIT 1) + \\ + \\UNION ALL + \\ + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1 + \\ORDER BY scrobbles.datetime DESC + \\LIMIT 1) + , + + .artist => + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime ASC + \\LIMIT 1) + \\ + \\UNION ALL + \\ + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime DESC + \\LIMIT 1) + , + }, + + .timescale => + //.ResultType = TimescaleResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE artists.id = $1 + \\GROUP BY year + \\ORDER BY year ASC; + , + + .album => + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1 + \\GROUP BY year + \\ORDER BY year ASC; + , + + .artist => + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE artists.id = $1 + \\GROUP BY year + \\ORDER BY year ASC; + , + }, + + .entities => + //.ResultType = EntitiesResult, + switch (entity) { + .scrobble => + \\none + , + .song => + \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\GROUP BY songs.id, artists.id + \\ORDER BY scrobbles DESC, songs.name ASC + , + .album => + \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN albums ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN artistalbums ON artistalbums.album_id = albums.id + \\INNER JOIN artists ON artists.id = artistalbums.artist_id + \\GROUP BY albums.id, artists.id + \\ORDER BY scrobbles DESC + , + .artist => + \\SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongsartists + \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id + \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY artists.id + \\ORDER BY scrobbles DESC; + , + }, + + .appears => + //.ResultType = AppearsResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\ NOTHING YET + , + .album => + \\nope + , + .artist => + \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 AND artistalbums.artist_id != $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC; + , + }, + + .entity_items => + //.ResultType = EntityItemsResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\get all scrobbles + , + .album => + \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albumsongs.album_id = $1 + \\GROUP BY songs.id + \\ORDER BY scrobbles DESC + , + .artist => + \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE artistalbums.artist_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + , + }, + + .entity_info => switch (entity) { + .scrobble => unreachable, + .song => + \\lol idk + , + .album => + \\SELECT * FROM + \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM + \\(SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN albums ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY albums.id)) + \\WHERE id = $1 + , + .artist => + \\SELECT * FROM + \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM + \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongsartists + \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id + \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY artists.id)) + \\WHERE id = $1 + , + }, + }, + }; +} From aab61631a397012f581fc2b9fd078eb9e09650cf Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 25 May 2025 16:16:18 -0400 Subject: [PATCH 050/103] Directly append complete_scrobble in upload.zig Thanks bob :) --- build.zig.zon | 4 ++-- src/app/views/upload.zig | 15 +-------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index d7fb785..5db05e3 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -17,8 +17,8 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/7be1d137fcab5c422e05f12092f6e04a02900d6f.tar.gz", - .hash = "jetzig-0.0.0-IpAgLURbDwB1NlywUH7lnQ3zptNvSQWVosaA1k7l1cNz", + .url = "https://github.com/jetzig-framework/jetzig/archive/695664bc10e6e73389ee2fe7fbcaedc27fa8adc9.tar.gz", + .hash = "jetzig-0.0.0-IpAgLSFeDwBEhfEaDBo0JqhRbvQA3ScbWjssBqhJHFmb", }, .zeit = .{ .url = "https://github.com/rockorager/zeit/archive/refs/tags/v0.6.0.tar.gz", diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index af5d41d..f605982 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -138,20 +138,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - var job_param = try job_params.append(.object); - try job_param.put("album", complete_scrobble.album); - try job_param.put("track", complete_scrobble.track); - try job_param.put("date", complete_scrobble.date); - - var track_artists_array = try job_param.put("artists_track", .array); - for (complete_scrobble.artists_track) |a| { - try track_artists_array.append(a); - } - - var album_artists_array = try job_param.put("artists_album", .array); - for (complete_scrobble.artists_album) |a| { - try album_artists_array.append(a); - } + try job_params.append(complete_scrobble); } }, } From f59eec79a82584bd7bcc88277b3a0b6f86ba0eb8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 26 May 2025 11:15:19 -0400 Subject: [PATCH 051/103] Removed inline else from upload.zig If I can figure out a way to get an array of a union instead of a union of arrays, we're in business to make this even better, but this is fine right now. The inline else was just a dumb way to keep the for on the outside --- src/app/views/upload.zig | 108 +++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index f605982..bdb5cef 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -81,58 +81,68 @@ pub fn post(request: *jetzig.Request) !jetzig.View { // Not sure if I should be proud or feel sick switch (imported_scrobbles) { - inline else => |scrobbles| { + .LastFMStats => |scrobbles| { appends: for (scrobbles) |scrobble| { - const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) { - Data.IgnorantScrobble => { - if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { - limited_tracks += 1; - continue :appends; - } - break :blk Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date * 1_000, - }; - }, - Data.SpotifyScrobble => { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { - limited_tracks += 1; - continue :appends; - } - - break :blk Data.Scrobble{ - .album = scrobble.master_metadata_album_album_name.?, - .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .track = scrobble.master_metadata_track_name.?, - .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, - }; - }, - Data.LastFMWebScrobble => { - break :blk Data.Scrobble{ - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, - .track = scrobble.name, - .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, - }; - }, - else => unreachable, + if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { + limited_tracks += 1; + continue :appends; + } + const filtered_scrobble = Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date * 1_000, }; + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + + try view_params.append(row); + try job_params.append(complete_scrobble); + } + }, + .LastFMWeb => |scrobbles| { + for (scrobbles) |scrobble| { + const filtered_scrobble = Data.Scrobble{ + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, + .track = scrobble.name, + .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, + }; + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + + try view_params.append(row); + try job_params.append(complete_scrobble); + } + }, + .Spotify => |scrobbles| { + appends: for (scrobbles) |scrobble| { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } + + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { + limited_tracks += 1; + continue :appends; + } + + const filtered_scrobble = Data.Scrobble{ + .album = scrobble.master_metadata_album_album_name.?, + .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .track = scrobble.master_metadata_track_name.?, + .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, + }; const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); From 3ff973e193f331676c0470a6dfc4ff38330aff64 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 26 May 2025 11:15:51 -0400 Subject: [PATCH 052/103] Use queries.zig in scrobbles view --- src/app/views/scrobbles.zig | 50 +++---------------------------------- src/queries.zig | 37 ++++++++++++++------------- 2 files changed, 23 insertions(+), 64 deletions(-) diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 3fe946a..e61df4d 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -1,56 +1,12 @@ const std = @import("std"); const jetzig = @import("jetzig"); -const zeit = @import("zeit"); -const TableRow = @import("../../types.zig").TableRow; -const HyperlinkData = @import("../../types.zig").HyperlinkData; -const Utils = @import("../../date_fmt.zig"); +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var scrobbles_view = try root.put("scrobbles", .array); - const query = - \\SELECT songs.name, songs.id, albums.name, albums.id, artists.name, artists.id, scrobbles.id, scrobbles.datetime - \\FROM albumsongs - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN albums ON albums.id = albumsongs.album_id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\ORDER BY scrobbles.datetime ASC - ; - - var scrobbles_jq_result = try request.repo.executeSql(query, .{}); - defer scrobbles_jq_result.deinit(); - - const Scrobble = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; - - var prev_s_id: ?i32 = null; - - var row: ?TableRow = null; - var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); - - blk: while (try scrobbles_jq_result.postgresql.result.next()) |scrobble_row| { - const scrobble = try scrobble_row.to(Scrobble, .{ .dupe = true, .allocator = request.allocator }); - if (scrobble.s_id == prev_s_id) { - try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); - continue :blk; - } else { - const date = try Utils.dateFmt(request.allocator, scrobble.date); - - try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); - - row = TableRow{ - .song = .{ .name = scrobble.song_name, .id = scrobble.song_id }, - .album = .{ .name = scrobble.album_name, .id = scrobble.album_id }, - .artistlist = try artistlist.toOwnedSlice(), - .date = date, - }; - - try scrobbles_view.append(row); - } - prev_s_id = scrobble.s_id; - } + const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.scrobble, .entities), .{}, .array); + try root.put("scrobbles", scrobbles); return request.render(.ok); } diff --git a/src/queries.zig b/src/queries.zig index cc5137b..faba14d 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -25,23 +25,19 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: const item: ?TableRow = switch (query.ResultType) { EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { .artist => TableRow{ .artist = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .album => album_entities: { + .scrobble, .song, .album => album_entities: { const last_artist = artist_list.getLastOrNull(); try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); if (last_artist) |la| { if (la.id == entity.artist_id) continue :blk; } - break :album_entities TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; + break :album_entities switch (query.entity) { + .scrobble => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .artistlist = try artist_list.toOwnedSlice(), .date = entity.date }, + .song => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, + .album => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, + else => unreachable, + }; }, - .song => song_entities: { - const last_artist = artist_list.getLastOrNull(); - try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); - if (last_artist) |la| { - if (la.id == entity.artist_id) continue :blk; - } - break :song_entities TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; - }, - else => unreachable, }, EntityItemsResult => switch (query.entity) { .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, @@ -92,7 +88,7 @@ const TimescaleResult = struct { year: []const u8, scrobbles: i64 }; const EntitiesArtistResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const EntitiesAlbumResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; const EntitiesSongResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; -const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; +const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, date: []const u8 }; const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; @@ -142,7 +138,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .album => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -153,7 +149,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -164,7 +160,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .artist => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -175,7 +171,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -227,7 +223,14 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue //.ResultType = EntitiesResult, switch (entity) { .scrobble => - \\none + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\ORDER BY scrobbles.datetime ASC , .song => \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles From d638fa66c550e79ccd04101e093b06b5d56970a2 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 29 May 2025 15:33:10 -0400 Subject: [PATCH 053/103] Create GET function for a song view --- src/app/views/songs.zig | 17 ++++++- src/app/views/songs/get.zmpl | 22 +++++++-- src/queries.zig | 88 +++++++++++++++++++++++++++++------- 3 files changed, 106 insertions(+), 21 deletions(-) diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index d5a7944..3143f01 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -12,6 +12,21 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - _ = id; + var root = try request.data(.object); + + const song = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_info), .{id}, .object); + try root.put("song", song.get("entity_info")); + + const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_items), .{id}, .array); + try root.put("scrobbles", scrobbles); + + const appears = try queries.entityQueryResult(request, queries.generateQuery(.song, .appears), .{id}, .array); + try root.put("appears", appears); + + const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.song, .firstlast), .{id}, .array); + try root.put("firstlast", firstlast); + + const timescale = try queries.entityQueryResult(request, queries.generateQuery(.song, .timescale), .{id}, .array); + try root.put("yearly", timescale); return request.render(.ok); } diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 76457d0..0a051a4 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -1,3 +1,19 @@ -
- Content goes here -
+@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; +} + + + + + + +@partial partials/header +

{{.song}}

+@partial partials/firstlast_listens(scrobbles: .song.scrobbles, rank: .song.rank, firstlast: .firstlast) +

Yearly Performance

+@partial partials/timescale(range: .yearly) +

Scrobbles

+@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) + + \ No newline at end of file diff --git a/src/queries.zig b/src/queries.zig index faba14d..061b5b3 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -20,7 +20,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: blk: while (try result.postgresql.result.next()) |entity_row| { switch (query.ResultType) { - FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult => |T| { + FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult, EntityItemsSongResult => |T| { const entity = try entity_row.to(T, .{ .dupe = true, .allocator = request.allocator }); const item: ?TableRow = switch (query.ResultType) { EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { @@ -39,7 +39,8 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: }; }, }, - EntityItemsResult => switch (query.entity) { + EntityItemsResult, EntityItemsSongResult => switch (query.entity) { + .song => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .date = entity.date }, .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, .album => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, else => unreachable, @@ -92,6 +93,17 @@ const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, alb const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; +const EntityItemsSongResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, date: []const u8 }; +const UnifiedResult = struct { + album_name: ?[]const u8 = null, + album_id: ?i32 = null, + song_name: ?[]const u8 = null, + song_id: ?i32 = null, + artist_names: ?[]const []const u8 = null, + artist_ids: ?[]i32 = null, + scrobbles: ?i64 = null, + date: ?[]const u8 = null, +}; pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { return GeneratedQuery{ @@ -106,7 +118,11 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue .album => EntitiesAlbumResult, .artist => EntitiesArtistResult, }, - .entity_items => EntityItemsResult, + .entity_items => switch (entity) { + .scrobble => unreachable, + .song => EntityItemsSongResult, + else => EntityItemsResult, + }, .appears => AppearsResult, .entity_info => EntityInfoResult, }, @@ -116,23 +132,21 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime,'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 + \\WHERE albumsongs.song_id = $1 \\ORDER BY scrobbles.datetime ASC \\LIMIT 1) \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 + \\WHERE albumsongs.song_id = $1 \\ORDER BY scrobbles.datetime DESC \\LIMIT 1) , @@ -190,9 +204,8 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\WHERE artists.id = $1 + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\WHERE songs.id = $1 \\GROUP BY year \\ORDER BY year ASC; , @@ -268,13 +281,21 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\ NOTHING YET + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongs.song_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC; , .album => \\nope , .artist => - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums \\INNER JOIN albums ON albums.id = artistalbums.album_id \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id @@ -291,7 +312,13 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\get all scrobbles + \\SELECT songs.name, songs.id, albums.name, albums.id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE songs.id = $1 + \\ORDER BY date ASC , .album => \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles @@ -317,12 +344,19 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue .entity_info => switch (entity) { .scrobble => unreachable, .song => - \\lol idk + \\SELECT * FROM + \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM99999th') AS rank FROM + \\(SELECT songs.name AS name, songs.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY songs.id)) + \\WHERE id = $1 , .album => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\(SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id @@ -341,6 +375,26 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\WHERE id = $1 , }, + + .datestreak => switch (entity) { + .song => + \\SELECT songs.name, albums.name, streak.maxseq, streak.ds, streak.de FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT albumsong, grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\COUNT(DISTINCT datetime::date) AS numdays + \\FROM (SELECT scrobbles.*, + \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY albumsong ORDER BY datetime::date)) AS grp + \\FROM scrobbles + \\) scrobbles + \\GROUP BY albumsong, grp + \\) scrobbles + \\GROUP BY albumsong, ds, de) AS streak + \\INNER JOIN albumsongs ON albumsongs.id = streak.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\ORDER BY streak.maxseq DESC; + , + else => unreachable, + }, }, }; } From 62590fee374fab21c36976724d014a5a1b15f787 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 29 May 2025 19:39:51 -0400 Subject: [PATCH 054/103] Made queries.zig look significantly nicer There's a little bit of weird stuff happening, but holy cannoli, that's so much easier to maintain and parse --- src/app/views/albums.zig | 12 +- src/app/views/artists.zig | 14 +- .../views/partials/_firstlast_listens.zmpl | 4 +- src/app/views/partials/_timescale.zmpl | 2 +- src/app/views/scrobbles.zig | 2 +- src/app/views/songs.zig | 14 +- src/queries.zig | 179 ++++++------------ 7 files changed, 87 insertions(+), 140 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index af72456..713e6d6 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -8,7 +8,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const albums = try queries.entityQueryResult(request, queries.generateQuery(.album, .entities), .{}, .array); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.album, .entities), .{}); try root.put("albums", albums); return request.render(.ok); @@ -17,16 +17,16 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const album = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_info), .{id}, .object); - try root.put("album", album.get("entity_info")); + const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id}); + try root.put("album", album); - const songs = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_items), .{id}, .array); + const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_items), .{id}); try root.put("songs", songs); - const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.album, .firstlast), .{id}, .array); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.generateQuery(.album, .timescale), .{id}, .array); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.album, .timescale), .{id}); try root.put("yearly", timescale); return request.render(.ok); diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 37ca155..1f7ff85 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -8,7 +8,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const artists = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entities), .{}, .array); + const artists = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entities), .{}); try root.put("artists", artists); @@ -18,19 +18,19 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const artist = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_info), .{id}, .object); - try root.put("artist", artist.get("entity_info")); + const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id}); + try root.put("artist", artist); - const albums = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_items), .{id}, .array); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_items), .{id}); try root.put("albums", albums); - const appears = try queries.entityQueryResult(request, queries.generateQuery(.artist, .appears), .{id}, .array); + const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id}); try root.put("appears", appears); - const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.artist, .firstlast), .{id}, .array); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.artist, .firstlast), .{id}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.generateQuery(.artist, .timescale), .{id}, .array); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.artist, .timescale), .{id}); try root.put("yearly", timescale); return request.render(.ok); diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl index ed74361..a21114b 100644 --- a/src/app/views/partials/_firstlast_listens.zmpl +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -7,7 +7,7 @@
{{scrobbles}} scrobbles ({{rank}} place)
-First listen: {{songs[0].name}} ({{songs[0].date}}) +First listen: {{songs[0].song.name}} ({{songs[0].date}})
-Most recent listen: {{songs[1].name}} ({{songs[1].date}}) +Most recent listen: {{songs[1].song.name}} ({{songs[1].date}})
\ No newline at end of file diff --git a/src/app/views/partials/_timescale.zmpl b/src/app/views/partials/_timescale.zmpl index fdec7a1..9937ec6 100644 --- a/src/app/views/partials/_timescale.zmpl +++ b/src/app/views/partials/_timescale.zmpl @@ -10,7 +10,7 @@ @for (range) |itm| { - {{itm.year}}: + {{itm.date}}: {{itm.scrobbles}} } diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index e61df4d..754a7c8 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -5,7 +5,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.scrobble, .entities), .{}, .array); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.scrobble, .entities), .{}); try root.put("scrobbles", scrobbles); return request.render(.ok); diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 3143f01..b24dbf5 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -5,7 +5,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const songs = try queries.entityQueryResult(request, queries.generateQuery(.song, .entities), .{}, .array); + const songs = try queries.entityQueryResult(request, queries.loadQuery(.song, .entities), .{}); try root.put("songs", songs); return request.render(.ok); @@ -14,19 +14,19 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const song = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_info), .{id}, .object); - try root.put("song", song.get("entity_info")); + const song = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_info), .{id}); + try root.put("song", song); - const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_items), .{id}, .array); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_items), .{id}); try root.put("scrobbles", scrobbles); - const appears = try queries.entityQueryResult(request, queries.generateQuery(.song, .appears), .{id}, .array); + const appears = try queries.entityQueryResult(request, queries.loadQuery(.song, .appears), .{id}); try root.put("appears", appears); - const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.song, .firstlast), .{id}, .array); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.song, .firstlast), .{id}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.generateQuery(.song, .timescale), .{id}, .array); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.song, .timescale), .{id}); try root.put("yearly", timescale); return request.render(.ok); } diff --git a/src/queries.zig b/src/queries.zig index 061b5b3..0411022 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -4,135 +4,82 @@ const TableRow = @import("types.zig").TableRow; const HyperlinkData = @import("types.zig").HyperlinkData; const std = @import("std"); -pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: anytype, root_type: enum { object, array }) !*jetzig.Data.Value { - var result = try request.repo.executeSql(query.query, args); +pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: anytype) !*jetzig.Data.Value { + //var result = try request.repo.executeSql(query.query, args); + // + var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, args, .{ .allocator = request.allocator, .column_names = true }); defer result.deinit(); var Data = jetzig.Data.init(request.allocator); - var out: *jetzig.Data.Value = switch (root_type) { - .array => try Data.array(), - .object => try Data.object(), - }; - var artist_list = if (query.ResultType == EntitiesAlbumResult or query.ResultType == EntitiesArtistResult or query.ResultType == EntitiesScrobbleResult or query.ResultType == EntitiesSongResult) - std.ArrayList(HyperlinkData).init(request.allocator); + var artist_list = std.ArrayList(HyperlinkData).init(request.allocator); - blk: while (try result.postgresql.result.next()) |entity_row| { - switch (query.ResultType) { - FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult, EntityItemsSongResult => |T| { - const entity = try entity_row.to(T, .{ .dupe = true, .allocator = request.allocator }); - const item: ?TableRow = switch (query.ResultType) { - EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { - .artist => TableRow{ .artist = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .scrobble, .song, .album => album_entities: { - const last_artist = artist_list.getLastOrNull(); - try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); - if (last_artist) |la| { - if (la.id == entity.artist_id) continue :blk; - } - break :album_entities switch (query.entity) { - .scrobble => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .artistlist = try artist_list.toOwnedSlice(), .date = entity.date }, - .song => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, - .album => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, - else => unreachable, - }; - }, - }, - EntityItemsResult, EntityItemsSongResult => switch (query.entity) { - .song => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .date = entity.date }, - .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .album => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - else => unreachable, - }, - AppearsResult => switch (query.entity) { - .song, .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .album, .scrobble => unreachable, - }, - else => null, - }; - - if (item) |itm| { - switch (root_type) { - .array => try out.append(itm), - .object => switch (query.query_type) { - inline else => |qt| try out.put(@tagName(qt), itm), - }, - } - } else { - switch (root_type) { - .array => try out.append(entity), - .object => switch (query.query_type) { - inline else => |qt| try out.put(@tagName(qt), entity), - }, - } - } - }, - else => unreachable, - } + if (query.query_type == .entity_info) { + var out: *jetzig.Data.Value = try Data.object(); + const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); + const entity_struct = .{ .name = entity.name, .id = entity.id, .scrobbles = entity.scrobbles, .rank = entity.rank }; + try out.put("entity_info", entity_struct); + try result.drain(); + return out.get("entity_info").?; } - if (root_type == .object) {} + + var out: *jetzig.Data.Value = try Data.array(); + var mapper = result.mapper(UnifiedResult, .{ .dupe = true, .allocator = request.allocator }); + + blk: while (try mapper.next()) |entity| { + if (entity.artist_id) |_| { + const last_artist = artist_list.getLastOrNull(); + try artist_list.append(.{ .name = entity.artist_name.?, .id = entity.artist_id.? }); + if (last_artist) |la| { + if (la.id == entity.artist_id) continue :blk; + } + } + + try out.append(TableRow{ + .artist = if (entity.artist_id) |_| .{ .id = entity.artist_id.?, .name = entity.artist_name.? } else null, + .album = if (entity.album_id) |_| .{ .id = entity.album_id.?, .name = entity.album_name.? } else null, + .song = if (entity.song_id) |_| .{ .id = entity.song_id.?, .name = entity.song_name.? } else null, + .artistlist = if (artist_list.getLastOrNull()) |_| try artist_list.toOwnedSlice() else null, + .scrobbles = if (entity.scrobbles) |scrobbles| scrobbles else null, + .date = if (entity.date) |date| date else null, + }); + } + return out; } +const EntityType = enum { scrobble, song, album, artist }; +const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak }; + const GeneratedQuery = struct { entity: EntityType, query_type: QueryTypeEnum, - ResultType: type, query: []const u8, }; -const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info }; -const FirstlastResult = struct { name: []const u8, id: i32, date: []const u8 }; -const TimescaleResult = struct { year: []const u8, scrobbles: i64 }; -const EntitiesArtistResult = struct { name: []const u8, id: i32, scrobbles: i64 }; -const EntitiesAlbumResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; -const EntitiesSongResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; -const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, date: []const u8 }; -const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; -const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; -const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; -const EntityItemsSongResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, date: []const u8 }; const UnifiedResult = struct { album_name: ?[]const u8 = null, album_id: ?i32 = null, song_name: ?[]const u8 = null, song_id: ?i32 = null, - artist_names: ?[]const []const u8 = null, - artist_ids: ?[]i32 = null, + artist_name: ?[]const u8 = null, + artist_id: ?i32 = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, }; -pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { +pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { return GeneratedQuery{ .entity = entity, .query_type = query_type, - .ResultType = switch (query_type) { - .firstlast => FirstlastResult, - .timescale => TimescaleResult, - .entities => switch (entity) { - .scrobble => EntitiesScrobbleResult, - .song => EntitiesSongResult, - .album => EntitiesAlbumResult, - .artist => EntitiesArtistResult, - }, - .entity_items => switch (entity) { - .scrobble => unreachable, - .song => EntityItemsSongResult, - else => EntityItemsResult, - }, - .appears => AppearsResult, - .entity_info => EntityInfoResult, - }, .query = switch (query_type) { .firstlast => //.ResultType = FirstlastResult, switch (entity) { .scrobble => unreachable, .song => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime,'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime,'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -142,7 +89,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -152,7 +99,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .album => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -163,7 +110,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -174,7 +121,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .artist => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -185,7 +132,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -201,34 +148,34 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN songs ON songs.id = albumsongs.song_id \\WHERE songs.id = $1 - \\GROUP BY year - \\ORDER BY year ASC; + \\GROUP BY date + \\ORDER BY date ASC; , .album => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN albums ON albums.id = albumsongs.album_id \\WHERE albums.id = $1 - \\GROUP BY year - \\ORDER BY year ASC; + \\GROUP BY date + \\ORDER BY date ASC; , .artist => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id \\WHERE artists.id = $1 - \\GROUP BY year - \\ORDER BY year ASC; + \\GROUP BY date + \\ORDER BY date ASC; , }, @@ -246,7 +193,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles.datetime ASC , .song => - \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\SELECT songs.name AS song_name, songs.id AS song_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id @@ -256,7 +203,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles DESC, songs.name ASC , .album => - \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -266,7 +213,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles DESC , .artist => - \\SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\SELECT artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongsartists \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id @@ -312,7 +259,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\SELECT songs.name, songs.id, albums.name, albums.id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN albums ON albums.id = albumsongs.album_id \\INNER JOIN songs ON songs.id = albumsongs.song_id @@ -321,7 +268,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY date ASC , .album => - \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\SELECT songs.name AS song_name, songs.id AS song_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id @@ -330,7 +277,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles DESC , .artist => - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums \\INNER JOIN albums ON albums.id = artistalbums.album_id \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id @@ -378,7 +325,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue .datestreak => switch (entity) { .song => - \\SELECT songs.name, albums.name, streak.maxseq, streak.ds, streak.de FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de + \\SELECT songs.name AS song_name, albums.name AS album_name, streak.maxseq AS intval, FORMAT('%s - %s, streak.ds, streak.de) AS date FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de \\FROM (SELECT albumsong, grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, \\COUNT(DISTINCT datetime::date) AS numdays \\FROM (SELECT scrobbles.*, From d81681e6983df014a405d4a3c2fb57362f19011d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 13:37:34 -0400 Subject: [PATCH 055/103] Move scrobble rank from firstlast partial to view. Eventually moving this to its own partial (probably) --- src/app/views/albums/get.zmpl | 6 ++++-- src/app/views/artists/get.zmpl | 9 +++++++-- src/app/views/partials/_firstlast_listens.zmpl | 4 +--- src/app/views/partials/_timescale.zmpl | 18 ++++++++++-------- src/app/views/songs/get.zmpl | 5 +++-- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 1024ec5..22f99d8 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -9,8 +9,10 @@ @partial partials/header -

{{.album}}

-@partial partials/firstlast_listens(scrobbles: .album.scrobbles, rank: .album.rank, firstlast: .firstlast) +

{{.album.name}}

+
{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
+
{{.album.song_num}} songs
+@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly)

Songs

diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 69bd661..14976b5 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -10,9 +10,14 @@ @partial partials/header

{{.artist.name}}

-@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, firstlast: .firstlast) -

Yearly Performance

+
+
{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
+
{{.artist.song_num}} songs
+
{{.artist.album_num}} albums
+
@partial partials/timescale(range: .yearly) +
+@partial partials/firstlast_listens(firstlast: .firstlast)

Albums

@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl index a21114b..3cf3ac9 100644 --- a/src/app/views/partials/_firstlast_listens.zmpl +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -1,12 +1,10 @@ -@args scrobbles: i64, rank: []const u8, firstlast: *ZmplValue +@args firstlast: *ZmplValue @zig { const songs = firstlast.items(.array); }
-{{scrobbles}} scrobbles ({{rank}} place) -
First listen: {{songs[0].song.name}} ({{songs[0].date}})
Most recent listen: {{songs[1].song.name}} ({{songs[1].date}}) diff --git a/src/app/views/partials/_timescale.zmpl b/src/app/views/partials/_timescale.zmpl index 9937ec6..24ef925 100644 --- a/src/app/views/partials/_timescale.zmpl +++ b/src/app/views/partials/_timescale.zmpl @@ -1,5 +1,6 @@ @args range: *ZmplValue +
@@ -8,11 +9,12 @@ -@for (range) |itm| { - - - - -} - -
{{itm.date}}:{{itm.scrobbles}}
\ No newline at end of file + @for (range) |itm| { + + {{itm.date}}: + {{itm.scrobbles}} + + } + + +
\ No newline at end of file diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 0a051a4..5b68145 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -9,8 +9,9 @@ @partial partials/header -

{{.song}}

-@partial partials/firstlast_listens(scrobbles: .song.scrobbles, rank: .song.rank, firstlast: .firstlast) +

{{.song.name}}

+
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
+@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly)

Scrobbles

From c57bf186273c34be8b19093db67a1a06afde9622 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 13:39:03 -0400 Subject: [PATCH 056/103] Update queries Adds datestreak query, provides the number of songs/albums when relevant, and provides timescale with all years, regardless of the number of plays (defaults to 0) --- src/queries.zig | 75 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/src/queries.zig b/src/queries.zig index 0411022..a22b68e 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -17,13 +17,18 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: if (query.query_type == .entity_info) { var out: *jetzig.Data.Value = try Data.object(); - const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); - const entity_struct = .{ .name = entity.name, .id = entity.id, .scrobbles = entity.scrobbles, .rank = entity.rank }; - try out.put("entity_info", entity_struct); + const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, album_num: ?i64 = null, song_num: ?i64 = null, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator, .map = .name }); + try out.put("entity_info", entity); try result.drain(); return out.get("entity_info").?; } + //if (query.query_type == .datestreak) { + // var out: *jetzig.Data.Value = try Data.object(); + // const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); + // + //} + var out: *jetzig.Data.Value = try Data.array(); var mapper = result.mapper(UnifiedResult, .{ .dupe = true, .allocator = request.allocator }); @@ -168,14 +173,17 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , .artist => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles + \\SELECT y.year AS date, COALESCE(DT.scrobbles, 0) AS scrobbles + \\FROM (SELECT GENERATE_SERIES(2016,date_part('year',now())::int)::text) AS y(year) + \\LEFT JOIN (SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id \\WHERE artists.id = $1 \\GROUP BY date - \\ORDER BY date ASC; + \\ORDER BY date ASC) AS DT + \\ON DT.date = y.year; , }, @@ -303,21 +311,24 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .album => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\(SELECT albums.name AS name, albums.id AS id, COUNT(DISTINCT songs.id) AS song_num, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN songs ON songs.id = albumsongs.song_id \\GROUP BY albums.id)) - \\WHERE id = $1 + \\WHERE id = $1; , .artist => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles, COUNT(DISTINCT albums.id) AS album_num, COUNT(DISTINCT songs.id) AS song_num \\FROM albumsongsartists \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id \\GROUP BY artists.id)) \\WHERE id = $1 , @@ -325,22 +336,44 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .datestreak => switch (entity) { .song => - \\SELECT songs.name AS song_name, albums.name AS album_name, streak.maxseq AS intval, FORMAT('%s - %s, streak.ds, streak.de) AS date FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de - \\FROM (SELECT albumsong, grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\SELECT maxseq AS streak, FORMAT('%s - %s', ds, de) AS date FROM (SELECT MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, \\COUNT(DISTINCT datetime::date) AS numdays - \\FROM (SELECT scrobbles.*, - \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY albumsong ORDER BY datetime::date)) AS grp - \\FROM scrobbles + \\FROM (SELECT scrobbles.datetime, + \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY songs.id ORDER BY datetime::date)) AS grp + \\FROM scrobbles INNER JOIN albumsongs ON albumsongs.id = albumsong INNER JOIN songs ON songs.id = albumsongs.song_id + \\WHERE songs.id = $1) scrobbles + \\GROUP BY grp \\) scrobbles - \\GROUP BY albumsong, grp - \\) scrobbles - \\GROUP BY albumsong, ds, de) AS streak - \\INNER JOIN albumsongs ON albumsongs.id = streak.albumsong - \\INNER JOIN albums ON albums.id = albumsongs.album_id - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\ORDER BY streak.maxseq DESC; + \\GROUP BY ds, de) + \\ORDER BY maxseq DESC; , - else => unreachable, + .artist => + \\SELECT maxseq AS streak, FORMAT('%s - %s' ,ds,de) AS date FROM (SELECT MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\COUNT(DISTINCT datetime::date) AS numdays + \\FROM (SELECT scrobbles.datetime, + \\((scrobbles.datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY artists.id ORDER BY scrobbles.datetime::date)) AS grp + \\FROM scrobbles INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = scrobbles.albumsong INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE artists.id = $1) scrobbles + \\GROUP BY grp + \\) scrobbles + \\GROUP BY ds, de) + \\ORDER BY maxseq DESC; + , + .album => + \\SELECT maxseq AS streak, FORMAT('%s - %s', ds, de) AS date FROM (SELECT MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\COUNT(DISTINCT datetime::date) AS numdays + \\FROM (SELECT scrobbles.datetime, + \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY albums.id ORDER BY datetime::date)) AS grp FROM scrobbles INNER JOIN albumsongs ON albumsongs.id = albumsong INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1) scrobbles + \\GROUP BY grp + \\) scrobbles + \\GROUP BY ds, de) + \\ORDER BY maxseq DESC; + , + .scrobble => unreachable, }, }, }; From a314fd447d4b7e21df030e57c349eb7bfdf0ca2d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 14:31:15 -0400 Subject: [PATCH 057/103] Fix LastFM API scrobble parsing when song is currently playing --- src/app/views/upload.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index bdb5cef..ebe9395 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -103,7 +103,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } }, .LastFMWeb => |scrobbles| { - for (scrobbles) |scrobble| { + appends: for (scrobbles) |scrobble| { + if (scrobble.date == null) continue :appends; const filtered_scrobble = Data.Scrobble{ .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, From 3777b818e3aae4fbe9489084f9770bf81ff2fe41 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 14:45:45 -0400 Subject: [PATCH 058/103] Create view for groups One of the largest components that makes zuletzt unique - implementing groups the way MusicBrainz has (release groups in particular). I thought for a while that I would just connect songs via a shared ID, but for remixes and such, I don't think they should be so tightly coupled. This also gives the user freedom for how they want to do the grouping (a remix can be included in the group if they choose to, or it may not). This will allow someone to see a combined scrobble number for an album with, for example, a regular release, a deluxe release, and an anniversary release, in addition to the individual releases. This will complicate SQL queries rather significantly I imagine, and I'm not sure what the interface for creating/deleting groups will be (although it will likely be easier when I have full use of TS), but it's a necessity for the project. --- src/app/views/groups.zig | 104 +++++++++++++++++++++++++++++++ src/app/views/groups/delete.zmpl | 3 + src/app/views/groups/edit.zmpl | 3 + src/app/views/groups/get.zmpl | 3 + src/app/views/groups/index.zmpl | 3 + src/app/views/groups/new.zmpl | 3 + src/app/views/groups/patch.zmpl | 3 + src/app/views/groups/post.zmpl | 3 + src/app/views/groups/put.zmpl | 3 + 9 files changed, 128 insertions(+) create mode 100644 src/app/views/groups.zig create mode 100644 src/app/views/groups/delete.zmpl create mode 100644 src/app/views/groups/edit.zmpl create mode 100644 src/app/views/groups/get.zmpl create mode 100644 src/app/views/groups/index.zmpl create mode 100644 src/app/views/groups/new.zmpl create mode 100644 src/app/views/groups/patch.zmpl create mode 100644 src/app/views/groups/post.zmpl create mode 100644 src/app/views/groups/put.zmpl diff --git a/src/app/views/groups.zig b/src/app/views/groups.zig new file mode 100644 index 0000000..85505c8 --- /dev/null +++ b/src/app/views/groups.zig @@ -0,0 +1,104 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +pub fn index(request: *jetzig.Request) !jetzig.View { + return request.render(.ok); +} + +pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn new(request: *jetzig.Request) !jetzig.View { + return request.render(.ok); +} + +pub fn edit(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn post(request: *jetzig.Request) !jetzig.View { + return request.render(.created); +} + +pub fn put(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn patch(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn delete(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + + +test "index" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups", .{}); + try response.expectStatus(.ok); +} + +test "get" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} + +test "new" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups/new", .{}); + try response.expectStatus(.ok); +} + +test "edit" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups/example-id/edit", .{}); + try response.expectStatus(.ok); +} + +test "post" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.POST, "/groups", .{}); + try response.expectStatus(.created); +} + +test "put" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.PUT, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} + +test "patch" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.PATCH, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} + +test "delete" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.DELETE, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} diff --git a/src/app/views/groups/delete.zmpl b/src/app/views/groups/delete.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/delete.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/edit.zmpl b/src/app/views/groups/edit.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/edit.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/get.zmpl b/src/app/views/groups/get.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/get.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/index.zmpl b/src/app/views/groups/index.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/index.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/new.zmpl b/src/app/views/groups/new.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/new.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/patch.zmpl b/src/app/views/groups/patch.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/patch.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/post.zmpl b/src/app/views/groups/post.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/post.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/put.zmpl b/src/app/views/groups/put.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/put.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
From 906ba6d2e5f41dc9ad16ea65a6868fb598a1ce19 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 14:47:52 -0400 Subject: [PATCH 059/103] Update header partial and remoev table partial Long overdue --- src/app/views/partials/_header.zmpl | 4 ++++ src/app/views/partials/_table.zmpl | 22 ---------------------- 2 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 src/app/views/partials/_table.zmpl diff --git a/src/app/views/partials/_header.zmpl b/src/app/views/partials/_header.zmpl index bf80e53..3f9089d 100644 --- a/src/app/views/partials/_header.zmpl +++ b/src/app/views/partials/_header.zmpl @@ -1,7 +1,11 @@ Zuletzt +Artists +Albums +Songs Scrobbles Concerts Collection Ratings Lists +Groups
\ No newline at end of file diff --git a/src/app/views/partials/_table.zmpl b/src/app/views/partials/_table.zmpl deleted file mode 100644 index 83c1cd4..0000000 --- a/src/app/views/partials/_table.zmpl +++ /dev/null @@ -1,22 +0,0 @@ -@args table_data: *ZmplValue, table_headers: *ZmplValue - - - -@for (table_headers) |text| { - -} - - - @for (table_data) |value| { - - - - - - - } -
{{text}}
{{value.track}} - @for (value.get("artists").?) |artist| { - {{artist}} - } - {{value.album}}{{value.date}}
\ No newline at end of file From 566edf18189391559576af803f73c956951a3e91 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 15:48:30 -0400 Subject: [PATCH 060/103] Include artist(s) name in album GET view This also makes the entity_info struct very similar to the UnifiedResult struct, so we'll probably see a merge at some point. Would be nice if we used the fields from the entity_info result more commonly. --- src/app/views/albums/get.zmpl | 3 ++- src/app/views/artists/get.zmpl | 2 +- src/app/views/songs/get.zmpl | 2 +- src/queries.zig | 34 +++++++++++++++++++++++++--------- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 22f99d8..7e089e7 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -9,7 +9,8 @@ @partial partials/header -

{{.album.name}}

+

{{.album.album_name}}

+

{{.album.artist_name}}

{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
{{.album.song_num}} songs
@partial partials/firstlast_listens(firstlast: .firstlast) diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 14976b5..9f1ff07 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -9,7 +9,7 @@ @partial partials/header -

{{.artist.name}}

+

{{.artist.artist_name}}

{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
{{.artist.song_num}} songs
diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 5b68145..e912930 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -9,7 +9,7 @@ @partial partials/header -

{{.song.name}}

+

{{.song.song_name}}

{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

diff --git a/src/queries.zig b/src/queries.zig index a22b68e..194ef7f 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -17,7 +17,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: if (query.query_type == .entity_info) { var out: *jetzig.Data.Value = try Data.object(); - const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, album_num: ?i64 = null, song_num: ?i64 = null, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator, .map = .name }); + const entity = try (try result.next()).?.to(EntityInfoResult, .{ .dupe = true, .allocator = request.allocator, .map = .name }); try out.put("entity_info", entity); try result.drain(); return out.get("entity_info").?; @@ -74,6 +74,20 @@ const UnifiedResult = struct { date: ?[]const u8 = null, }; +const EntityInfoResult = struct { + album_name: ?[]const u8 = null, + album_id: ?i32 = null, + song_name: ?[]const u8 = null, + song_id: ?i32 = null, + artist_name: ?[]const u8 = null, + artist_id: ?i32 = null, + scrobbles: ?i64 = null, + date: ?[]const u8 = null, + rank: []const u8, + song_num: ?i64 = null, + album_num: ?i64 = null, +}; + pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { return GeneratedQuery{ .entity = entity, @@ -301,28 +315,30 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .song => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM99999th') AS rank FROM - \\(SELECT songs.name AS name, songs.id AS id, COUNT(scrobbles) AS scrobbles + \\(SELECT songs.name AS song_name, songs.id AS song_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id \\GROUP BY songs.id)) - \\WHERE id = $1 + \\WHERE song_id = $1 , .album => - \\SELECT * FROM + \\SELECT album_name, t.album_id, artists.name AS artist_name, artists.id AS artist_id, song_num, scrobbles, rank FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT albums.name AS name, albums.id AS id, COUNT(DISTINCT songs.id) AS song_num, COUNT(scrobbles) AS scrobbles + \\(SELECT albums.name AS album_name, albums.id AS album_id, COUNT(DISTINCT songs.id) AS song_num, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\GROUP BY albums.id)) - \\WHERE id = $1; + \\GROUP BY albums.id)) AS t + \\INNER JOIN artistalbums ON artistalbums.album_id = t.album_id + \\INNER JOIN artists ON artists.id = artistalbums.artist_id + \\WHERE t.album_id = $1 , .artist => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles, COUNT(DISTINCT albums.id) AS album_num, COUNT(DISTINCT songs.id) AS song_num + \\(SELECT artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles, COUNT(DISTINCT albums.id) AS album_num, COUNT(DISTINCT songs.id) AS song_num \\FROM albumsongsartists \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id @@ -330,7 +346,7 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\INNER JOIN albums ON albums.id = albumsongs.album_id \\INNER JOIN songs ON songs.id = albumsongs.song_id \\GROUP BY artists.id)) - \\WHERE id = $1 + \\WHERE artist_id = $1 , }, From adcaff34ea5726fb9e1e3e87394ecaea136349f5 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 2 Jun 2025 00:13:27 -0400 Subject: [PATCH 061/103] Fix dumb appears query for albums --- src/queries.zig | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/queries.zig b/src/queries.zig index 194ef7f..8db7809 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -248,7 +248,7 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .appears => //.ResultType = AppearsResult, switch (entity) { - .scrobble => unreachable, + .scrobble, .album => unreachable, .song => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums @@ -260,9 +260,6 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\GROUP BY albums.id \\ORDER BY scrobbles DESC; , - .album => - \\nope - , .artist => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums From 3ef17fcd468d710186d0448424d15ea42786b395 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 6 Jun 2025 14:28:15 -0400 Subject: [PATCH 062/103] Split entity_items and appears query into more granular queries We can be a bit more specific about the information we get this way --- src/app/views/albums.zig | 2 +- src/app/views/artists.zig | 2 +- src/app/views/songs.zig | 6 +-- src/queries.zig | 91 +++++++++++++++++++++++++++------------ 4 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 713e6d6..03046b3 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -20,7 +20,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id}); try root.put("album", album); - const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_items), .{id}); + const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .get_songs), .{id}); try root.put("songs", songs); const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id}); diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 1f7ff85..3ec8eba 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -21,7 +21,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id}); try root.put("artist", artist); - const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_items), .{id}); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .get_albums), .{id}); try root.put("albums", albums); const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id}); diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index b24dbf5..164f928 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -17,11 +17,11 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const song = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_info), .{id}); try root.put("song", song); - const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_items), .{id}); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_scrobbles), .{id}); try root.put("scrobbles", scrobbles); - const appears = try queries.entityQueryResult(request, queries.loadQuery(.song, .appears), .{id}); - try root.put("appears", appears); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_albums), .{id}); + try root.put("albums", albums); const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.song, .firstlast), .{id}); try root.put("firstlast", firstlast); diff --git a/src/queries.zig b/src/queries.zig index 8db7809..5d92869 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -55,7 +55,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: } const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak }; +const QueryTypeEnum = enum { firstlast, timescale, entities, get_songs, get_albums, get_scrobbles, appears, entity_info, datestreak }; const GeneratedQuery = struct { entity: EntityType, @@ -246,20 +246,9 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { }, .appears => - //.ResultType = AppearsResult, + // Not sure how I feel about this one switch (entity) { - .scrobble, .album => unreachable, - .song => - \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles - \\FROM artistalbums - \\INNER JOIN albums ON albums.id = artistalbums.album_id - \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongs.song_id = $1 - \\GROUP BY albums.id - \\ORDER BY scrobbles DESC; - , + .scrobble, .song, .album => unreachable, .artist => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums @@ -273,19 +262,8 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , }, - .entity_items => - //.ResultType = EntityItemsResult, - switch (entity) { - .scrobble => unreachable, - .song => - \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date - \\FROM albumsongs - \\INNER JOIN albums ON albums.id = albumsongs.album_id - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\WHERE songs.id = $1 - \\ORDER BY date ASC - , + .get_songs => switch (entity) { + .scrobble, .song => unreachable, // Might be able to use this with SongGroups? .album => \\SELECT songs.name AS song_name, songs.id AS song_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs @@ -296,6 +274,34 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\ORDER BY scrobbles DESC , .artist => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\GROUP BY songs.id, albums.id + \\ORDER BY scrobbles DESC + , + }, + + .get_albums => + //.ResultType = EntityItemsResult, + switch (entity) { + .scrobble, .album => unreachable, // Might be able to use this with ReleaseGroups? + .song => + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongs.song_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + , + .artist => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums \\INNER JOIN albums ON albums.id = artistalbums.album_id @@ -307,6 +313,37 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , }, + .get_scrobbles => switch (entity) { + .scrobble => unreachable, + .song => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE songs.id = $1 + \\ORDER BY date ASC + , + .album => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albums.id = $1 + \\ORDER BY date ASC + , + .artist => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE artists.id = $1 + \\ORDER BY date ASC + , + }, + .entity_info => switch (entity) { .scrobble => unreachable, .song => From c8f2ef57c8f1f8932f5faa1612be30ed4792d592 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 6 Jun 2025 15:55:20 -0400 Subject: [PATCH 063/103] Add some tyling to songs view This can (will) be easily replicated for the other views, I just first tested it on songs. I think this looks much nicer, and I'll probably roll with a layout similar to this for the other views, with some minor adjustments for each particular view. --- src/app/views/partials/_newtable.zmpl | 4 +++- src/app/views/songs/get.zmpl | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/app/views/partials/_newtable.zmpl b/src/app/views/partials/_newtable.zmpl index 6746a2a..f9dc618 100644 --- a/src/app/views/partials/_newtable.zmpl +++ b/src/app/views/partials/_newtable.zmpl @@ -1,5 +1,6 @@ @args T: type, table_data: *ZmplValue, columns: T +
@@ -70,4 +71,5 @@ } } -
\ No newline at end of file + +
\ No newline at end of file diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index e912930..1ceddef 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -9,12 +9,24 @@ @partial partials/header +

{{.song.song_name}}

-
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
-@partial partials/firstlast_listens(firstlast: .firstlast) -

Yearly Performance

-@partial partials/timescale(range: .yearly) -

Scrobbles

-@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) +
+ +
+
+
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
+ @partial partials/firstlast_listens(firstlast: .firstlast) +

Yearly Performance

+ @partial partials/timescale(range: .yearly) +

Scrobbles

+ @partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) +
+
+

Rating

+ + +
+
\ No newline at end of file From 162341fb5f696196375c33f5ae3c795f8058c952 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 9 Jun 2025 21:41:52 -0400 Subject: [PATCH 064/103] Convert ids to i64 The birthday paradox is a real problem with the size of our datasets. i64 is the largest numerical value we can use, and there's a 0.1% chance of collision with ~2,000,000 values, so I feel pretty comfortable with this --- .../2025-04-07_14-31-14_create_songs.zig | 2 +- .../2025-04-07_14-31-45_create_albums.zig | 2 +- .../2025-04-07_14-34-39_create_albumsongs.zig | 6 +++--- .../2025-04-07_14-35-53_create_scrobbles.zig | 4 ++-- .../2025-04-07_14-38-02_create_artists.zig | 2 +- ...04-07_14-39-09_create_albumsongsartists.zig | 6 +++--- ...2025-04-07_14-40-17_create_artistalbums.zig | 6 +++--- src/queries.zig | 18 ++++++------------ src/types.zig | 2 +- 9 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig index 9a52b6b..e8ae1d6 100644 --- a/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig +++ b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "songs", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("length", .float, .{ .optional = true }), t.column("hidden", .boolean, .{}), diff --git a/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig index d706cfe..86b6184 100644 --- a/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig +++ b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albums", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("length", .float, .{ .optional = true }), t.timestamps(.{}), diff --git a/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig index 4381a5b..96c3063 100644 --- a/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig +++ b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albumsongs", &.{ - t.primaryKey("id", .{}), - t.column("song_id", .integer, .{ .reference = .{ "songs", "id" } }), - t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("song_id", .bigint, .{ .reference = .{ "songs", "id" } }), + t.column("album_id", .bigint, .{ .reference = .{ "albums", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index 088a43c..c429e19 100644 --- a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -6,8 +6,8 @@ pub fn up(repo: anytype) !void { try repo.createTable( "scrobbles", &.{ - t.primaryKey("id", .{}), - t.column("albumsong", .integer, .{ .reference = .{ "albumsongs", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("albumsong", .bigint, .{ .reference = .{ "albumsongs", "id" } }), t.column("datetime", .datetime, .{}), t.timestamps(.{}), }, diff --git a/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig index ed0d3d4..97c5bfe 100644 --- a/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig +++ b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "artists", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("disambiguation", .string, .{ .optional = true }), t.timestamps(.{}), diff --git a/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig index 55c7d84..3355196 100644 --- a/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig +++ b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albumsongsartists", &.{ - t.primaryKey("id", .{}), - t.column("albumsong_id", .integer, .{ .reference = .{ "albumsongs", "id" } }), - t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("albumsong_id", .bigint, .{ .reference = .{ "albumsongs", "id" } }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig index 76bac1e..3c3ea7f 100644 --- a/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig +++ b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "artistalbums", &.{ - t.primaryKey("id", .{}), - t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), - t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("album_id", .bigint, .{ .reference = .{ "albums", "id" } }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/queries.zig b/src/queries.zig index 5d92869..ba2a734 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -23,12 +23,6 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: return out.get("entity_info").?; } - //if (query.query_type == .datestreak) { - // var out: *jetzig.Data.Value = try Data.object(); - // const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); - // - //} - var out: *jetzig.Data.Value = try Data.array(); var mapper = result.mapper(UnifiedResult, .{ .dupe = true, .allocator = request.allocator }); @@ -65,22 +59,22 @@ const GeneratedQuery = struct { const UnifiedResult = struct { album_name: ?[]const u8 = null, - album_id: ?i32 = null, + album_id: ?i64 = null, song_name: ?[]const u8 = null, - song_id: ?i32 = null, + song_id: ?i64 = null, artist_name: ?[]const u8 = null, - artist_id: ?i32 = null, + artist_id: ?i64 = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, }; const EntityInfoResult = struct { album_name: ?[]const u8 = null, - album_id: ?i32 = null, + album_id: ?i64 = null, song_name: ?[]const u8 = null, - song_id: ?i32 = null, + song_id: ?i64 = null, artist_name: ?[]const u8 = null, - artist_id: ?i32 = null, + artist_id: ?i64 = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, rank: []const u8, diff --git a/src/types.zig b/src/types.zig index 3b2e4ff..5f3e0e3 100644 --- a/src/types.zig +++ b/src/types.zig @@ -133,5 +133,5 @@ pub const TableRow = struct { pub const HyperlinkData = struct { name: []const u8, - id: i32, + id: i64, }; From 85552f39c1145acf18d9b74b1252a1b32ac12168 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 9 Jun 2025 21:45:41 -0400 Subject: [PATCH 065/103] Add Artistsongs table Whether or not a song is covered, there was an original artist who originally performed the song. The only issue is that an Artistsongs table will almost be the exact same as the Albumsongsartists table, since most songs aren't covered. So, it may be better not to populate that table by default, and then if two albumsongs with different artists share the same song, then fill the Artistsongs table. --- src/app/database/Schema.zig | 56 ++++++++++++------- ...2025-06-06_19-46-32_create_artistsongs.zig | 20 +++++++ 2 files changed, 56 insertions(+), 20 deletions(-) create mode 100644 src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig diff --git a/src/app/database/Schema.zig b/src/app/database/Schema.zig index eeec984..0a30a19 100644 --- a/src/app/database/Schema.zig +++ b/src/app/database/Schema.zig @@ -4,7 +4,7 @@ pub const Album = jetquery.Model( @This(), "albums", struct { - id: i32, + id: i64, name: []const u8, length: ?f32, created_at: jetquery.DateTime, @@ -22,9 +22,9 @@ pub const Albumsong = jetquery.Model( @This(), "albumsongs", struct { - id: i32, - song_id: i32, - album_id: i32, + id: i64, + song_id: i64, + album_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -32,9 +32,7 @@ pub const Albumsong = jetquery.Model( .relations = .{ .song = jetquery.belongsTo(.Song, .{}), .album = jetquery.belongsTo(.Album, .{}), - .scrobbles = jetquery.hasMany(.Scrobble, .{ - .foreign_key = "albumsong", - }), + .scrobbles = jetquery.hasMany(.Scrobble, .{ .foreign_key = "albumsong" }), .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), }, }, @@ -44,9 +42,9 @@ pub const Albumsongsartist = jetquery.Model( @This(), "albumsongsartists", struct { - id: i32, - albumsong_id: i32, - artist_id: i32, + id: i64, + albumsong_id: i64, + artist_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -62,9 +60,9 @@ pub const Artistalbum = jetquery.Model( @This(), "artistalbums", struct { - id: i32, - album_id: i32, - artist_id: i32, + id: i64, + album_id: i64, + artist_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -80,7 +78,7 @@ pub const Artist = jetquery.Model( @This(), "artists", struct { - id: i32, + id: i64, name: []const u8, disambiguation: ?[]const u8, created_at: jetquery.DateTime, @@ -90,6 +88,7 @@ pub const Artist = jetquery.Model( .relations = .{ .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), .artistalbums = jetquery.hasMany(.Artistalbum, .{}), + .artistsongs = jetquery.hasMany(.Artistsong, .{}), }, }, ); @@ -98,17 +97,15 @@ pub const Scrobble = jetquery.Model( @This(), "scrobbles", struct { - id: i32, - albumsong: i32, + id: i64, + albumsong: i64, datetime: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .albumsong = jetquery.belongsTo(.Albumsong, .{ - .foreign_key = "albumsong", - }), + .albumsong = jetquery.belongsTo(.Albumsong, .{ .foreign_key = "albumsong" }), }, }, ); @@ -117,7 +114,7 @@ pub const Song = jetquery.Model( @This(), "songs", struct { - id: i32, + id: i64, name: []const u8, length: ?f32, hidden: bool, @@ -127,6 +124,25 @@ pub const Song = jetquery.Model( .{ .relations = .{ .albumsongs = jetquery.hasMany(.Albumsong, .{}), + .artistsongs = jetquery.hasMany(.Artistsong, .{}), + }, + }, +); + +pub const Artistsong = jetquery.Model( + @This(), + "artistsongs", + struct { + id: i64, + artist_id: i64, + song_id: i64, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .artist = jetquery.belongsTo(.Artist, .{}), + .song = jetquery.belongsTo(.Song, .{}), }, }, ); diff --git a/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig b/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig new file mode 100644 index 0000000..7d0a6c1 --- /dev/null +++ b/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig @@ -0,0 +1,20 @@ +const std = @import("std"); +const jetquery = @import("jetquery"); +const t = jetquery.schema.table; + +pub fn up(repo: anytype) !void { + try repo.createTable( + "artistsongs", + &.{ + t.primaryKey("id", .{ .type = .bigint }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), + t.column("song_id", .bigint, .{ .reference = .{ "songs", "id" } }), + t.timestamps(.{}), + }, + .{}, + ); +} + +pub fn down(repo: anytype) !void { + try repo.dropTable("artistsongs", .{}); +} From 1e4a271b8d7e78c37140e6e129d5292dda2cd117 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:22:06 -0400 Subject: [PATCH 066/103] Update README --- README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8247b21..06f45c2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ Inspired by [Last.fm](https://last.fm), [Maloja](https://github.com/krateng/maloja), and [Lastfmstats.com](https://www.lastfmstats.com). - **Z**uletzt is written with [**Z**ig](https://github.com/ziglang/zig) and [Jetzig](https://github.com/jetzig-framework/jetzig) as a means of learning the language, reintroducing myself to programming, and combining the functionality of the aforementioned inspirations. @@ -12,6 +11,35 @@ Zuletzt means "last" in German. Licensed under MIT. +## Usage +Zuletzt allows uploads of Scrobbles at the `/upload` page, where you +can import Scrobbles from a Spotify data export, Last.FM data export (a `.json` +file from lastfmstats.com), or by providing a Last.FM username and connecting +to Last.FM directly. + +Zuletzt will not make any assumptions about the data, and only change metadata when asked to by a rule. Two albums will be considered the same if: +- They share the same title (case/diacritic sensitive) +- The album artist(s) are the same + +Zuletzt allows you to list multiple artists under an album using rules, but +does not try to automatically split artists along common delimiters. For +example, there's no way to know that "Mermaid Avenue" by "Billy Bragg, Wilco" +is performed by two artists, while "Ants From Up There" by "Black Country, New +Road" is performed by one artist. Thus, a rule needs to be made to tell Zuletzt +"Mermaid Avenue" is performed by "Billy Bragg" and "Wilco". + +Two songs will only be considered the same if: +- They share the same title (case/diacritic sensitive) +- They appear on the same album + +If two or more songs with the same spelling appear on an album, they are +necessarily grouped under the same name, as there is no way to differentiate +them (see "Once In Royal David's City" on Sufjan Stevens's "Songs For +Christmas", for example). Every artist that performs on those songs with +receive attribution for the combined song. + +If two artists have the same name, they are necessarily listed as the same artist, but can be separated with a rule, or after the fact, with a disambiguation string. + ## To-Do List: - [ ] Entity statistics - [x] See all artists under "/artists" @@ -60,6 +88,7 @@ Licensed under MIT. - [ ] Rank songs - [ ] Custom statistics[^7] - [ ] "Playlists"[^8] +- [ ] First launch setup [^1]: I do not intend to exactly replicate all the statistics Lastfmstats.com provides, but I would at least like to give the user the option to see those kinds of statistics, or generate them themselves (see 7). From a8a4ed27c46baeb213b687f89fbb8fb71cdc04f6 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:22:38 -0400 Subject: [PATCH 067/103] Make process_scrobbles vars more readable, and change hashing --- src/app/jobs/process_scrobbles.zig | 107 +++++++++++++++-------------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index dcf7f1f..ea743b9 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -20,12 +20,13 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig // Probably want to include artist name here, but not sure how to yet - const track_artist_count = item.getT(.array, "artists_track").?.count(); - const album_artist_count = item.getT(.array, "artists_album").?.count(); - var track_artist_name_buffer = try allocator.alloc([]const u8, track_artist_count); - var album_artist_name_buffer = try allocator.alloc([]const u8, album_artist_count); - var track_artist_id_buffer = try allocator.alloc(i32, track_artist_count); - var album_artist_id_buffer = try allocator.alloc(i32, album_artist_count); + const track_artists = item.getT(.array, "artists_track").?.items(); + const album_artists = item.getT(.array, "artists_album").?.items(); + + var track_artist_name_buffer = try allocator.alloc([]const u8, track_artists.len); + var album_artist_name_buffer = try allocator.alloc([]const u8, album_artists.len); + var track_artist_id_buffer = try allocator.alloc(i64, track_artists.len); + var album_artist_id_buffer = try allocator.alloc(i64, album_artists.len); const scrobble: Data.Scrobble = .{ .track = item.getT(.string, "track").?, @@ -35,95 +36,99 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig .date = @as(i64, @truncate(item.getT(.integer, "date").?)), }; - var id_prehash = std.ArrayList(u8).init(allocator); + var album_hash_string = std.ArrayList(u8).init(allocator); + var track_hash_string = std.ArrayList(u8).init(allocator); - for (item.getT(.array, "artists_track").?.items(), 0..track_artist_count) |artist, i| { + // I theoretically don't need this for loop + for (track_artists, 0..track_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); track_artist_name_buffer[i] = artist_name; - track_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + track_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); } - for (item.getT(.array, "artists_album").?.items(), 0..album_artist_count) |artist, i| { + for (album_artists, 0..album_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; - album_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); - try id_prehash.appendSlice(artist_name); + album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + try album_hash_string.appendSlice(artist_name); } - try id_prehash.appendSlice(scrobble.album); - const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); - try id_prehash.appendSlice(scrobble.track); - const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); + try album_hash_string.appendSlice(scrobble.album); + try track_hash_string.appendSlice(scrobble.album); + const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(album_hash_string.items))); + try track_hash_string.appendSlice(scrobble.track); + const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(track_hash_string.items))); - var albumsong = try jetzig.database.Query(.Albumsong) - .findBy(.{ .album_id = album_id, .song_id = song_id }) + var albumsong_id = try jetzig.database.Query(.Albumsong) + .find(album_hash ^ track_hash) .select(.{.id}).execute(env.repo); - var ins_album = try jetzig.database.Query(.Album) - .find(album_id) + var album_id = try jetzig.database.Query(.Album) + .find(album_hash) .select(.{.id}).execute(env.repo); - for (track_artist_name_buffer, track_artist_id_buffer) |artist_name, artist_id| { - var ins_artist = try jetzig.database.Query(.Artist) - .find(artist_id) + for (track_artist_name_buffer, track_artist_id_buffer) |scrobble_track_artist, track_artist_hash| { + var artist_id = try jetzig.database.Query(.Artist) + .find(track_artist_hash) .select(.{.id}).execute(env.repo); - if (ins_artist == null) - ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + if (artist_id == null) + artist_id = try jetzig.database.Query(.Artist) + .insert(.{ .id = track_artist_hash, .name = scrobble_track_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); - if (albumsong == null) { - var ins_song = try jetzig.database.Query(.Song) - .find(song_id) + if (albumsong_id == null) { + var track_id = try jetzig.database.Query(.Song) + .find(track_hash) .select(.{.id}).execute(env.repo); - if (ins_song == null) - ins_song = try jetzig.database.Query(.Song) - .insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }) + if (track_id == null) + track_id = try jetzig.database.Query(.Song) + .insert(.{ .id = track_hash, .name = scrobble.track, .length = null, .hidden = false }) .returning(.{.id}).execute(env.repo); - if (ins_album == null) - ins_album = try jetzig.database.Query(.Album) - .insert(.{ .id = album_id, .name = scrobble.album, .length = null }) + if (album_id == null) + album_id = try jetzig.database.Query(.Album) + .insert(.{ .id = album_hash, .name = scrobble.album, .length = null }) .returning(.{.id}).execute(env.repo); - albumsong = try jetzig.database.Query(.Albumsong) - .insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }) + albumsong_id = try jetzig.database.Query(.Albumsong) + .insert(.{ .song_id = track_id.?.id, .album_id = album_id.?.id }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } else { const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) - .findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }) + .findBy(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }) .select(.{.id}).execute(env.repo); if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } } - for (album_artist_name_buffer, album_artist_id_buffer) |artist_name, artist_id| { - const ins_artistalbum = try jetzig.database.Query(.Artistalbum) - .findBy(.{ .album_id = ins_album.?.id, .artist_id = artist_id }) + + for (album_artist_name_buffer, album_artist_id_buffer) |scrobble_album_artist, album_artist_hash| { + const artistalbum_id = try jetzig.database.Query(.Artistalbum) + .findBy(.{ .album_id = album_id.?.id, .artist_id = album_artist_hash }) .select(.{.id}).execute(env.repo); - if (ins_artistalbum == null) { - var ins_artist = try jetzig.database.Query(.Artist) - .find(artist_id) + if (artistalbum_id == null) { + var artist_id = try jetzig.database.Query(.Artist) + .find(album_artist_hash) .select(.{.id}).execute(env.repo); - if (ins_artist == null) - ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + if (artist_id == null) + artist_id = try jetzig.database.Query(.Artist) + .insert(.{ .id = artist_id, .name = scrobble_album_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Artistalbum) - .insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .album_id = album_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } } try jetzig.database.Query(.Scrobble) - .insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); + .insert(.{ .albumsong = albumsong_id.?.id, .datetime = scrobble.date }).execute(env.repo); } } } From 6a1c82242074e296d7576913c467be236e9be336 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:25:40 -0400 Subject: [PATCH 068/103] Add entities_by_name query Will probably be used for disambiguation pages (among other things, but disambiguation pages are coming up soon) --- src/queries.zig | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/queries.zig b/src/queries.zig index 8db7809..adb1447 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -55,7 +55,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: } const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak }; +const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak, entities_by_name }; const GeneratedQuery = struct { entity: EntityType, @@ -215,13 +215,14 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\ORDER BY scrobbles.datetime ASC , .song => - \\SELECT songs.name AS song_name, songs.id AS song_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\GROUP BY songs.id, artists.id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\GROUP BY songs.id, albums.id, artists.id \\ORDER BY scrobbles DESC, songs.name ASC , .album => @@ -388,6 +389,21 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , .scrobble => unreachable, }, + .entities_by_name => switch (entity) { + .song => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE LOWER(songs.name) LIKE LOWER($1) + \\GROUP BY songs.id, albums.id, artists.id + \\ORDER BY songs.name ASC, scrobbles DESC; + , + else => unreachable, + }, }, }; } From 2f420bc5ce6d86822e62a819a616a1a10d3fcf19 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:26:12 -0400 Subject: [PATCH 069/103] Testing with groups and htmx --- src/app/views/groups/index.zmpl | 14 +++++++++++--- src/app/views/songs.zig | 10 +++++++++- src/app/views/songs/index.zmpl | 4 +++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/app/views/groups/index.zmpl b/src/app/views/groups/index.zmpl index 76457d0..1522924 100644 --- a/src/app/views/groups/index.zmpl +++ b/src/app/views/groups/index.zmpl @@ -1,3 +1,11 @@ -
- Content goes here -
+ + + + +@partial partials/header + +

Merge Songs

+
+ +
+
\ No newline at end of file diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index b24dbf5..2ed2e06 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -5,7 +5,15 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const songs = try queries.entityQueryResult(request, queries.loadQuery(.song, .entities), .{}); + const htmx_query = (try request.queryParams()).getT(.string, "s"); + + try root.put("htmx", htmx_query != null); + + const songs = if (htmx_query) |name| + try queries.entityQueryResult(request, queries.loadQuery(.song, .entities_by_name), .{name}) + else + try queries.entityQueryResult(request, queries.loadQuery(.song, .entities), .{}); + try root.put("songs", songs); return request.render(.ok); diff --git a/src/app/views/songs/index.zmpl b/src/app/views/songs/index.zmpl index 6964630..862b753 100644 --- a/src/app/views/songs/index.zmpl +++ b/src/app/views/songs/index.zmpl @@ -1,6 +1,6 @@ @zig { const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; - const columns: ColumnChoices = &.{.song, .artistlist, .scrobbles}; + const columns: ColumnChoices = &.{.song, .album, .artistlist, .scrobbles}; } @@ -8,8 +8,10 @@ + @if (! $.htmx) @partial partials/header

Songs

+ @end @partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) \ No newline at end of file From 36873053bca53dc5d39f3d0578701805c966f4a9 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 20:08:13 -0400 Subject: [PATCH 070/103] Make new scrobble processing function This uses the zmpl data as a hash map to check if we've already checked the db for some song/album/artist/etc. and now only checks once per entity/assoc. table entry to speed things up. Previously, for each scrobble, we checked if its metadata appeared in the respective table, regardless of whether or not we've scrobbled that albumsong before. So, a song like Starless had to be checked (at the time of writing) ~180 times, but is now only checked once. Similarly, Wilco was checked ~3000+ times, as Hurry Up, We're Dreaming was cheked ~700 times. The only problem now is the way it was implemented. Obviously, copying and pasting those huge chunks of code isn't very nice looking. ATM, I don't really care, and I'm more happy about the overall speed increase, as well as the readability increase of the job. However, I don't want to leave it like that. The way I see it, I have two options: either create a funcion which does this, or I can do something even better, which is create a jsonParse function, which, if my thought process works, would remove the need for an intermediary source type, meaning we no longer need to switch on that type, which means we can just have one for loop that does everything, which would mean we just need to have that code in one place. Also not entirely happy with the code concerning all the conversions to i64s and []const u8s, but I think I have to. --- .../2025-04-07_14-35-53_create_scrobbles.zig | 2 +- src/app/jobs/process_scrobbles.zig | 8 +- src/app/jobs/process_scrobbles2.zig | 131 +++++++++++ src/app/views/upload.zig | 207 +++++++++++++++++- 4 files changed, 332 insertions(+), 16 deletions(-) create mode 100644 src/app/jobs/process_scrobbles2.zig diff --git a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index c429e19..c3bcd12 100644 --- a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "scrobbles", &.{ - t.primaryKey("id", .{ .type = .bigint }), + t.primaryKey("id", .{}), t.column("albumsong", .bigint, .{ .reference = .{ "albumsongs", "id" } }), t.column("datetime", .datetime, .{}), t.timestamps(.{}), diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index ea743b9..770bde7 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -49,15 +49,15 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig for (album_artists, 0..album_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; - album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); try album_hash_string.appendSlice(artist_name); } try album_hash_string.appendSlice(scrobble.album); try track_hash_string.appendSlice(scrobble.album); - const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(album_hash_string.items))); + const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(album_hash_string.items))); try track_hash_string.appendSlice(scrobble.track); - const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(track_hash_string.items))); + const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(track_hash_string.items))); var albumsong_id = try jetzig.database.Query(.Albumsong) .find(album_hash ^ track_hash) @@ -120,7 +120,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig .select(.{.id}).execute(env.repo); if (artist_id == null) artist_id = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = scrobble_album_artist, .disambiguation = null }) + .insert(.{ .id = album_artist_hash, .name = scrobble_album_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Artistalbum) .insert(.{ .album_id = album_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); diff --git a/src/app/jobs/process_scrobbles2.zig b/src/app/jobs/process_scrobbles2.zig new file mode 100644 index 0000000..8b3ae66 --- /dev/null +++ b/src/app/jobs/process_scrobbles2.zig @@ -0,0 +1,131 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +// The `run` function for a job is invoked every time the job is processed by a queue worker +// (or by the Jetzig server if the job is processed in-line). +// +// Arguments: +// * allocator: Arena allocator for use during the job execution process. +// * params: Params assigned to a job (from a request, values added to response data). +// * env: Provides the following fields: +// - logger: Logger attached to the same stream as the Jetzig server. +// - environment: Enum of `{ production, development }`. +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + + for (params.getT(.object, "tracks").?.items()) |track| { + const id = try std.fmt.parseInt(i64, track.key, 10); + + const track_query = try jetzig.database.Query(.Song) + .find(id).execute(env.repo); + + if (track_query == null) { + const name = try track.value.coerce([]const u8); + try jetzig.database.Query(.Song) + .insert(.{ .id = id, .name = name, .length = null, .hidden = false }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albums").?.items()) |album| { + const id = try std.fmt.parseInt(i64, album.key, 10); + + const album_query = try jetzig.database.Query(.Album) + .find(id).execute(env.repo); + + if (album_query == null) { + const name = try album.value.coerce([]const u8); + try jetzig.database.Query(.Album) + .insert(.{ .id = id, .name = name, .length = null }) + .execute(env.repo); + } + } + + for (params.getT(.object, "artists").?.items()) |artist| { + const id = try std.fmt.parseInt(i64, artist.key, 10); + + const artist_query = try jetzig.database.Query(.Artist) + .find(id).execute(env.repo); + + if (artist_query == null) { + const name = try artist.value.coerce([]const u8); + try jetzig.database.Query(.Artist) + .insert(.{ .id = id, .name = name }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albumsongs").?.items()) |as| { + const id = try std.fmt.parseInt(i64, as.key, 10); + + const as_query = try jetzig.database.Query(.Albumsong) + .find(id).execute(env.repo); + + if (as_query == null) { + const track_id = @as(i64, @intCast(as.value.getT(.integer, "song").?)); + const album_id = @as(i64, @intCast(as.value.getT(.integer, "album").?)); + try jetzig.database.Query(.Albumsong) + .insert(.{ .id = id, .song_id = track_id, .album_id = album_id }) + .execute(env.repo); + } + + const scrobbles = as.value.getT(.array, "scrobbles").?; + for (scrobbles.items()) |date| { + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = id, .datetime = date }) + .execute(env.repo); + } + } + + for (params.getT(.object, "artistalbums").?.items()) |aa| { + const id = try std.fmt.parseInt(i64, aa.key, 10); + + const aa_query = try jetzig.database.Query(.Artistalbum) + .find(id).execute(env.repo); + + if (aa_query == null) { + const artist_id = @as(i64, @intCast(aa.value.getT(.integer, "artist").?)); + const album_id = @as(i64, @intCast(aa.value.getT(.integer, "album").?)); + try jetzig.database.Query(.Artistalbum) + .insert(.{ .id = id, .artist_id = artist_id, .album_id = album_id }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albumsongsartists").?.items()) |asa| { + const id = try std.fmt.parseInt(i64, asa.key, 10); + + const asa_query = try jetzig.database.Query(.Albumsongsartist) + .find(id).execute(env.repo); + + if (asa_query == null) { + const albumsong_id = @as(i64, @intCast(asa.value.getT(.integer, "albumsong").?)); + const artist_id = @as(i64, @intCast(asa.value.getT(.integer, "artist").?)); + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ .id = id, .albumsong_id = albumsong_id, .artist_id = artist_id }) + .execute(env.repo); + } + } + + //for ((params.getT(.object, "albumsongsartists")).?.items(.object)) |asa| { + // const id = try std.fmt.parseInt(i64, asa.key, 10); + // const albumsong_id = asa.value.getT(.integer, "albumsong"); + // const track_artist_id = asa.value.getT(.integer, "artist"); + + // const albumsongartist = try jetzig.database.Query(.Albumsongsartist) + // .find(id) + // .select(.{.id}).execute(env.repo); + + // if (albumsongartist == null) { + // var artist_id = try jetzig.database.Query(.Artist) + // .find(track_artist_id) + // .select(.{.id}).execute(env.repo); + // + // if (artist_id == null) { + // const artist = params.chain(.{"artists",}) + // artist_id = try jetzig.database.Query(.Artist) + // .insert(.{ .id = track_artist_hash, .name = scrobble_track_artist, .disambiguation = null }) + // .execute(env.repo); + // } + // } + //} +} diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index ebe9395..ac2c665 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -24,26 +24,25 @@ pub fn post(request: *jetzig.Request) !jetzig.View { defer rule_file.close(); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; - var job = try request.job("process_scrobbles"); + //var job = try request.job("process_scrobbles"); + var job = try request.job("process_scrobbles2"); const source = params.getT(.integer, "t").?; // This param is required in HTML + const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { const date = params.getT(.string, "latest-date").?; break :blk try zeit.Time.fromISO8601(date); - } else blk: { - break :blk (try zeit.instant(.{ .source = .now })).time(); - }; + } else (try zeit.instant(.{ .source = .now })).time(); + const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { const date = params.getT(.string, "earliest-date").?; break :blk try zeit.Time.fromISO8601(date); - } else blk: { - break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); - }; + } else (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); const earliest_timestamp = earliest_date.instant().unixTimestamp(); const latest_timestamp = latest_date.instant().unixTimestamp(); var view_params = try root.put("scrobbles", .array); - var job_params = try job.params.put("scrobbles", .array); + //var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; @@ -79,6 +78,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View { else => unreachable, }; + var artists = try job.params.put("artists", .object); + var albums = try job.params.put("albums", .object); + var tracks = try job.params.put("tracks", .object); + var artistalbums = try job.params.put("artistalbums", .object); + var albumsongs = try job.params.put("albumsongs", .object); + var albumsongsartists = try job.params.put("albumsongsartists", .object); + // Not sure if I should be proud or feel sick switch (imported_scrobbles) { .LastFMStats => |scrobbles| { @@ -99,7 +105,65 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, .LastFMWeb => |scrobbles| { @@ -117,7 +181,65 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, .Spotify => |scrobbles| { @@ -149,7 +271,66 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, } @@ -159,3 +340,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { return request.render(.created); } + +fn pair(a: u64, b: u64) u64 { + return @divFloor((a +% b) *% (a +% b +% 1) +% b, 2); +} From 0b07947b8a2ce8140a08fb9d699fca4d43969ab8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 18 Jun 2025 02:10:51 -0400 Subject: [PATCH 071/103] Create urlDecode function for redirects --- src/date_fmt.zig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 7e44da7..9746fe6 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -33,3 +33,20 @@ pub fn scrobbleToRow(allocator: std.mem.Allocator, scrobble: Data.Scrobble) !Dat .date = try dateFmt(allocator, scrobble.date), }; } + +pub fn urlDecode(allocator: std.mem.Allocator, str: []const u8) ![]const u8 { + var decoded = std.ArrayList(u8).init(allocator); + var i: usize = 0; + while (i < str.len) : (i += 1) { + const v = str[i]; + if (v == '%') { + if (i + 2 < str.len) { + const hex = str[i + 1 .. i + 3]; + const char = try std.fmt.parseInt(u8, hex, 16); + try decoded.append(char); + i += 2; + } else return error.InvalidInput; + } else try decoded.append(v); + } + return decoded.toOwnedSlice(); +} From 2d7d2835fd7f4d026fea9863b39d6b60150c89c8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 18 Jun 2025 02:11:30 -0400 Subject: [PATCH 072/103] Proof of concept artist name in url string Some of my favorite code in the project. Just need a disambiguation page, and we're in business here --- src/app/views/artists.zig | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 3ec8eba..12be28c 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -2,9 +2,10 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; const TableRow = @import("../../types.zig").TableRow; -const dateFmt = @import("../../date_fmt.zig").dateFmt; -const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; +//const dateFmt = @import("../../date_fmt.zig").dateFmt; +//const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; const queries = @import("../../queries.zig"); +const decode = @import("../../date_fmt.zig").urlDecode; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -16,21 +17,45 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { + const parse_err = blk: { + const rdr_id = std.fmt.parseInt(i64, id, 10) catch |err| break :blk err; + const artist = try jetzig.database.Query(.Artist).find(rdr_id).execute(request.repo); + if (artist == null) break :blk error.InvalidCharacter; + var name = std.ArrayList(u8).init(request.allocator); + try name.appendSlice("http://127.0.0.1:8080/artists/"); + try name.appendSlice(artist.?.name); + return request.redirect(try name.toOwnedSlice(), .found); + }; + + const id_int = switch (parse_err) { + error.Overflow => return request.fail(.not_found), + error.InvalidCharacter => blk: { + const rn = try decode(request.allocator, id); + std.log.debug("{s}", .{rn}); + const artists = try jetzig.database.Query(.Artist).where(.{ .name = rn }).all(request.repo); + + if (artists.len == 0) return request.fail(.not_found); + if (artists.len > 1) return request.redirect("http://127.0.0.1:8080", .found); + break :blk artists[0].id; + }, + else => unreachable, + }; + var root = try request.data(.object); - const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id}); + const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id_int}); try root.put("artist", artist); - const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .get_albums), .{id}); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .get_albums), .{id_int}); try root.put("albums", albums); - const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id}); + const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id_int}); try root.put("appears", appears); - const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.artist, .firstlast), .{id}); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.artist, .firstlast), .{id_int}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.loadQuery(.artist, .timescale), .{id}); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.artist, .timescale), .{id_int}); try root.put("yearly", timescale); return request.render(.ok); From 9c90c683c654c5043df9da0c120fc86dc4677a79 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 22 Jun 2025 14:36:59 -0400 Subject: [PATCH 073/103] Temporary fix to keep using LLVM --- build.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.zig b/build.zig index 3bf89aa..3b17d0b 100644 --- a/build.zig +++ b/build.zig @@ -12,6 +12,8 @@ pub fn build(b: *std.Build) !void { .optimize = optimize, }); + exe.use_llvm = true; + // Example dependency: // const zig_time_dep = b.dependency("zeit", .{}); From 77a9c24dab83a576263e706e0fa5e6d2566cfdbd Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 22 Jun 2025 14:37:10 -0400 Subject: [PATCH 074/103] Create ratings tables --- src/app/database/Schema.zig | 65 ++++++++++++++++++- ...2025-06-22_18-32-57_create_songratings.zig | 22 +++++++ ...025-06-22_18-33-34_create_albumratings.zig | 22 +++++++ ...25-06-22_18-34-00_create_artistratings.zig | 22 +++++++ 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig create mode 100644 src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig create mode 100644 src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig diff --git a/src/app/database/Schema.zig b/src/app/database/Schema.zig index 0a30a19..23a6897 100644 --- a/src/app/database/Schema.zig +++ b/src/app/database/Schema.zig @@ -97,7 +97,7 @@ pub const Scrobble = jetquery.Model( @This(), "scrobbles", struct { - id: i64, + id: i32, albumsong: i64, datetime: jetquery.DateTime, created_at: jetquery.DateTime, @@ -146,3 +146,66 @@ pub const Artistsong = jetquery.Model( }, }, ); + +pub const Albumrating = jetquery.Model( + @This(), + "albumratings", + struct { + id: i32, + album: i64, + rating: i16, + rating_text: []const u8, + date: jetquery.DateTime, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .album = jetquery.belongsTo(.Album, .{ + .foreign_key = "album", + }), + }, + }, +); + +pub const Artistrating = jetquery.Model( + @This(), + "artistratings", + struct { + id: i32, + artist: i64, + rating: i16, + rating_text: []const u8, + date: jetquery.DateTime, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .artist = jetquery.belongsTo(.Artist, .{ + .foreign_key = "artist", + }), + }, + }, +); + +pub const Songrating = jetquery.Model( + @This(), + "songratings", + struct { + id: i32, + song: i64, + rating: i16, + rating_text: []const u8, + date: jetquery.DateTime, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .albumsong = jetquery.belongsTo(.Albumsong, .{ + .foreign_key = "song", + }), + }, + }, +); diff --git a/src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig b/src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig new file mode 100644 index 0000000..1ddf750 --- /dev/null +++ b/src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig @@ -0,0 +1,22 @@ +const std = @import("std"); +const jetquery = @import("jetquery"); +const t = jetquery.schema.table; + +pub fn up(repo: anytype) !void { + try repo.createTable( + "songratings", + &.{ + t.primaryKey("id", .{}), + t.column("song", .bigint, .{ .reference = .{ "albumsongs", "id" } }), + t.column("rating", .smallint, .{}), + t.column("rating_text", .text, .{}), + t.column("date", .datetime, .{}), + t.timestamps(.{}), + }, + .{}, + ); +} + +pub fn down(repo: anytype) !void { + try repo.dropTable("songratings", .{}); +} diff --git a/src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig b/src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig new file mode 100644 index 0000000..187fdfb --- /dev/null +++ b/src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig @@ -0,0 +1,22 @@ +const std = @import("std"); +const jetquery = @import("jetquery"); +const t = jetquery.schema.table; + +pub fn up(repo: anytype) !void { + try repo.createTable( + "albumratings", + &.{ + t.primaryKey("id", .{}), + t.column("album", .bigint, .{ .reference = .{ "albums", "id" } }), + t.column("rating", .smallint, .{}), + t.column("rating_text", .text, .{}), + t.column("date", .datetime, .{}), + t.timestamps(.{}), + }, + .{}, + ); +} + +pub fn down(repo: anytype) !void { + try repo.dropTable("albumratings", .{}); +} diff --git a/src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig b/src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig new file mode 100644 index 0000000..956d121 --- /dev/null +++ b/src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig @@ -0,0 +1,22 @@ +const std = @import("std"); +const jetquery = @import("jetquery"); +const t = jetquery.schema.table; + +pub fn up(repo: anytype) !void { + try repo.createTable( + "artistratings", + &.{ + t.primaryKey("id", .{}), + t.column("artist", .bigint, .{ .reference = .{ "artists", "id" } }), + t.column("rating", .smallint, .{}), + t.column("rating_text", .text, .{}), + t.column("date", .datetime, .{}), + t.timestamps(.{}), + }, + .{}, + ); +} + +pub fn down(repo: anytype) !void { + try repo.dropTable("artistratings", .{}); +} From 93da50652a5487f83e85c412e4724f2ee8aecfac Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 23 Jun 2025 10:14:25 -0400 Subject: [PATCH 075/103] Remove unnecessary else --- src/app/views/artists.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 12be28c..505c3d1 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -38,7 +38,6 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { if (artists.len > 1) return request.redirect("http://127.0.0.1:8080", .found); break :blk artists[0].id; }, - else => unreachable, }; var root = try request.data(.object); From f292368947279e6d208505848602931378668cd8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 23 Jun 2025 10:15:17 -0400 Subject: [PATCH 076/103] Song name in url string --- src/app/views/songs.zig | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index b634534..00ec085 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -1,6 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const queries = @import("../../queries.zig"); +const decode = @import("../../date_fmt.zig").urlDecode; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -20,21 +21,43 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { + const parse_err = blk: { + const rdr_id = std.fmt.parseInt(i64, id, 10) catch |err| break :blk err; + const song = try jetzig.database.Query(.Song).find(rdr_id).execute(request.repo); + if (song == null) break :blk error.InvalidCharacter; + var name = std.ArrayList(u8).init(request.allocator); + try name.appendSlice("http://127.0.0.1:8080/songs/"); + try name.appendSlice(song.?.name); + return request.redirect(try name.toOwnedSlice(), .found); + }; + + const id_int = switch (parse_err) { + error.Overflow => return request.fail(.not_found), + error.InvalidCharacter => blk: { + const rn = try decode(request.allocator, id); + std.log.debug("{s}", .{rn}); + const songs = try jetzig.database.Query(.Song).where(.{ .name = rn }).all(request.repo); + + if (songs.len == 0) return request.fail(.not_found); + if (songs.len > 1) return request.redirect("http://127.0.0.1:8080", .found); + break :blk songs[0].id; + }, + }; var root = try request.data(.object); - const song = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_info), .{id}); + const song = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_info), .{id_int}); try root.put("song", song); - const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_scrobbles), .{id}); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_scrobbles), .{id_int}); try root.put("scrobbles", scrobbles); - const albums = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_albums), .{id}); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_albums), .{id_int}); try root.put("albums", albums); - const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.song, .firstlast), .{id}); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.song, .firstlast), .{id_int}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.loadQuery(.song, .timescale), .{id}); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.song, .timescale), .{id_int}); try root.put("yearly", timescale); return request.render(.ok); } From b7e625dd9844e455496b729ddb9433a4560dcdc3 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 23 Jun 2025 10:18:17 -0400 Subject: [PATCH 077/103] Start ratings This is actually fantastic, I'm really happy with how this has worked so far. My only concern for the future is how posting reviews from the `/ratings` path might work, since it's currently designed around posting reviews from the song page itself, but I think some HTMX and/or JS wil alleviate any problems I run into --- src/app/views/ratings/songs.zig | 12 ++++++++++++ src/app/views/ratings/songs/post.zmpl | 2 ++ src/app/views/songs/get.zmpl | 9 +++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 src/app/views/ratings/songs.zig create mode 100644 src/app/views/ratings/songs/post.zmpl diff --git a/src/app/views/ratings/songs.zig b/src/app/views/ratings/songs.zig new file mode 100644 index 0000000..3145fb4 --- /dev/null +++ b/src/app/views/ratings/songs.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); +pub fn post(request: *jetzig.Request) !jetzig.View { + var root = try request.data(.object); + const params = try request.params(); + const id = params.getT(.integer, "song_id"); + const review = params.getT(.string, "review"); + try root.put("song_id", id); + try root.put("review", review); + + return request.render(.created); +} diff --git a/src/app/views/ratings/songs/post.zmpl b/src/app/views/ratings/songs/post.zmpl new file mode 100644 index 0000000..3366a0c --- /dev/null +++ b/src/app/views/ratings/songs/post.zmpl @@ -0,0 +1,2 @@ +{{.song_id}} +{{.review}} \ No newline at end of file diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 1ceddef..33e6886 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -5,6 +5,7 @@ + @@ -24,9 +25,13 @@

Rating

- - +
+ + + +
+
No reviews
\ No newline at end of file From 996022fe5f3558b25149512dcd522d88d0ee7f01 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 23 Jun 2025 16:47:46 -0400 Subject: [PATCH 078/103] Make rating data optional Use HTML to enfore at least one of the two fields has a value, but I don't want to require both --- src/app/database/Schema.zig | 24 +++++++------------ ...2025-06-22_18-32-57_create_songratings.zig | 6 ++--- ...025-06-22_18-33-34_create_albumratings.zig | 4 ++-- ...25-06-22_18-34-00_create_artistratings.zig | 4 ++-- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/app/database/Schema.zig b/src/app/database/Schema.zig index 23a6897..a7118d4 100644 --- a/src/app/database/Schema.zig +++ b/src/app/database/Schema.zig @@ -153,17 +153,15 @@ pub const Albumrating = jetquery.Model( struct { id: i32, album: i64, - rating: i16, - rating_text: []const u8, + rating: ?i16, + rating_text: ?[]const u8, date: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .album = jetquery.belongsTo(.Album, .{ - .foreign_key = "album", - }), + .album = jetquery.belongsTo(.Album, .{ .foreign_key = "album" }), }, }, ); @@ -174,17 +172,15 @@ pub const Artistrating = jetquery.Model( struct { id: i32, artist: i64, - rating: i16, - rating_text: []const u8, + rating: ?i16, + rating_text: ?[]const u8, date: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .artist = jetquery.belongsTo(.Artist, .{ - .foreign_key = "artist", - }), + .artist = jetquery.belongsTo(.Artist, .{ .foreign_key = "artist" }), }, }, ); @@ -195,17 +191,15 @@ pub const Songrating = jetquery.Model( struct { id: i32, song: i64, - rating: i16, - rating_text: []const u8, + rating: ?i16, + rating_text: ?[]const u8, date: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .albumsong = jetquery.belongsTo(.Albumsong, .{ - .foreign_key = "song", - }), + .albumsong = jetquery.belongsTo(.Albumsong, .{ .foreign_key = "song" }), }, }, ); diff --git a/src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig b/src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig index 1ddf750..d11d6a8 100644 --- a/src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig +++ b/src/app/database/migrations/2025-06-22_18-32-57_create_songratings.zig @@ -7,9 +7,9 @@ pub fn up(repo: anytype) !void { "songratings", &.{ t.primaryKey("id", .{}), - t.column("song", .bigint, .{ .reference = .{ "albumsongs", "id" } }), - t.column("rating", .smallint, .{}), - t.column("rating_text", .text, .{}), + t.column("song", .bigint, .{ .reference = .{ "songs", "id" } }), + t.column("rating", .smallint, .{ .optional = true }), + t.column("rating_text", .text, .{ .optional = true }), t.column("date", .datetime, .{}), t.timestamps(.{}), }, diff --git a/src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig b/src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig index 187fdfb..ecb31a4 100644 --- a/src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig +++ b/src/app/database/migrations/2025-06-22_18-33-34_create_albumratings.zig @@ -8,8 +8,8 @@ pub fn up(repo: anytype) !void { &.{ t.primaryKey("id", .{}), t.column("album", .bigint, .{ .reference = .{ "albums", "id" } }), - t.column("rating", .smallint, .{}), - t.column("rating_text", .text, .{}), + t.column("rating", .smallint, .{ .optional = true }), + t.column("rating_text", .text, .{ .optional = true }), t.column("date", .datetime, .{}), t.timestamps(.{}), }, diff --git a/src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig b/src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig index 956d121..d47f130 100644 --- a/src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig +++ b/src/app/database/migrations/2025-06-22_18-34-00_create_artistratings.zig @@ -8,8 +8,8 @@ pub fn up(repo: anytype) !void { &.{ t.primaryKey("id", .{}), t.column("artist", .bigint, .{ .reference = .{ "artists", "id" } }), - t.column("rating", .smallint, .{}), - t.column("rating_text", .text, .{}), + t.column("rating", .smallint, .{ .optional = true }), + t.column("rating_text", .text, .{ .optional = true }), t.column("date", .datetime, .{}), t.timestamps(.{}), }, From 6f6aaecb8faeff5bc39e161b931404957fd43957 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 23 Jun 2025 16:50:13 -0400 Subject: [PATCH 079/103] Create rating interface on songs view If no ratings are present, provide a textbox to make a rating. If a rating is present, show the rating. Eventually, there will be a button that allows an additional rating to be made, and the ability to delete ratings --- src/app/views/ratings/songs.zig | 6 ++++-- src/app/views/ratings/songs/post.zmpl | 3 +-- src/app/views/songs.zig | 4 ++++ src/app/views/songs/get.zmpl | 13 +++++++++++-- src/queries.zig | 15 ++++++++++++++- src/types.zig | 2 ++ 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/app/views/ratings/songs.zig b/src/app/views/ratings/songs.zig index 3145fb4..e77c2ac 100644 --- a/src/app/views/ratings/songs.zig +++ b/src/app/views/ratings/songs.zig @@ -3,9 +3,11 @@ const jetzig = @import("jetzig"); pub fn post(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); const params = try request.params(); - const id = params.getT(.integer, "song_id"); + const id = params.getT(.integer, "song_id").?; + const score = if (params.getT(.integer, "score")) |score| @as(i16, @truncate(score)) else null; const review = params.getT(.string, "review"); - try root.put("song_id", id); + try jetzig.database.Query(.Songrating).insert(.{ .song = id, .rating = score, .rating_text = review, .date = @divFloor(request.start_time, 1_000) }).execute(request.repo); + try root.put("score", score); try root.put("review", review); return request.render(.created); diff --git a/src/app/views/ratings/songs/post.zmpl b/src/app/views/ratings/songs/post.zmpl index 3366a0c..ca54fd7 100644 --- a/src/app/views/ratings/songs/post.zmpl +++ b/src/app/views/ratings/songs/post.zmpl @@ -1,2 +1 @@ -{{.song_id}} -{{.review}} \ No newline at end of file + {{.score}}: {{.review}} (Today) \ No newline at end of file diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 00ec085..2f1772b 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -59,5 +59,9 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const timescale = try queries.entityQueryResult(request, queries.loadQuery(.song, .timescale), .{id_int}); try root.put("yearly", timescale); + + const ratings = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_ratings), .{id_int}); + try root.put("reviews", ratings); + return request.render(.ok); } diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 33e6886..f07b03c 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -1,6 +1,7 @@ @zig { const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; + const reviews = try zmpl.coerceArray(".reviews"); } @@ -25,13 +26,21 @@

Rating

+
+ @zig { + if (reviews.len == 0) {
-
-
No reviews
+ } else { + for (reviews) |review| { + {{review.score}}: {{review.review}} ({{review.date}}) + } +} + } +
\ No newline at end of file diff --git a/src/queries.zig b/src/queries.zig index fcc5f07..0c9359e 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -42,6 +42,8 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: .artistlist = if (artist_list.getLastOrNull()) |_| try artist_list.toOwnedSlice() else null, .scrobbles = if (entity.scrobbles) |scrobbles| scrobbles else null, .date = if (entity.date) |date| date else null, + .score = if (entity.score) |score| score else null, + .review = if (entity.review) |review| review else null, }); } @@ -49,7 +51,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: } const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, get_songs, get_albums, get_scrobbles, appears, entity_info, datestreak, entities_by_name }; +const QueryTypeEnum = enum { firstlast, timescale, entities, get_songs, get_albums, get_scrobbles, appears, entity_info, datestreak, entities_by_name, get_ratings }; const GeneratedQuery = struct { entity: EntityType, @@ -66,6 +68,8 @@ const UnifiedResult = struct { artist_id: ?i64 = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, + score: ?i16 = null, + review: ?[]const u8 = null, }; const EntityInfoResult = struct { @@ -435,6 +439,15 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , else => unreachable, }, + .get_ratings => switch (entity) { + .song => + \\SELECT rating AS score, rating_text AS review, TO_CHAR(date, 'YYYY-MM-DD') AS date + \\FROM songratings + \\WHERE song = $1 + \\ORDER BY date DESC; + , + else => unreachable, + }, }, }; } diff --git a/src/types.zig b/src/types.zig index 5f3e0e3..0a66ae7 100644 --- a/src/types.zig +++ b/src/types.zig @@ -129,6 +129,8 @@ pub const TableRow = struct { artistlist: ?[]HyperlinkData = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, + score: ?i16 = null, + review: ?[]const u8 = null, }; pub const HyperlinkData = struct { From 9f27fad2357cfdeaa549d425a07a715a8b0b421f Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 23 Jun 2025 17:02:08 -0400 Subject: [PATCH 080/103] Add section on SongGroups They don't exist yet, but I was trying to decide in my head if there was a meaningful difference between emrging two songs, and a SongGroup, and I decided they are indeed different, but really only in a small way --- README.md | 101 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 06f45c2..903f88f 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,27 @@ -# Zuletzt +# Zuletzt **Zuletzt** gives you the statistics of your music listening habits. -Inspired by [Last.fm](https://last.fm), [Maloja](https://github.com/krateng/maloja), and [Lastfmstats.com](https://www.lastfmstats.com). +Inspired by [Last.fm](https://last.fm), +[Maloja](https://github.com/krateng/maloja), and +[Lastfmstats.com](https://www.lastfmstats.com). -**Z**uletzt is written with [**Z**ig](https://github.com/ziglang/zig) and [Jetzig](https://github.com/jetzig-framework/jetzig) as a means of learning the -language, reintroducing myself to programming, and combining -the functionality of the aforementioned inspirations. +**Z**uletzt is written with [**Z**ig](https://github.com/ziglang/zig) and +[Jetzig](https://github.com/jetzig-framework/jetzig) as a means of learning the +language, reintroducing myself to programming, and combining the functionality +of the aforementioned inspirations. Zuletzt means "last" in German. Licensed under MIT. -## Usage +## Usage Zuletzt allows uploads of Scrobbles at the `/upload` page, where you can import Scrobbles from a Spotify data export, Last.FM data export (a `.json` file from lastfmstats.com), or by providing a Last.FM username and connecting to Last.FM directly. -Zuletzt will not make any assumptions about the data, and only change metadata when asked to by a rule. Two albums will be considered the same if: +Zuletzt will not make any assumptions about the data, and only change metadata +when asked to by a rule. Two albums will be considered the same if: - They share the same title (case/diacritic sensitive) - The album artist(s) are the same @@ -38,7 +42,29 @@ them (see "Once In Royal David's City" on Sufjan Stevens's "Songs For Christmas", for example). Every artist that performs on those songs with receive attribution for the combined song. -If two artists have the same name, they are necessarily listed as the same artist, but can be separated with a rule, or after the fact, with a disambiguation string. +If two artists have the same name, they are necessarily listed as the same +artist, but can be separated with a rule, or after the fact, with a +disambiguation string. + +## Quirks +Zuletzt does not assume any two songs are the same song unless they +share the exact same metadata. However, there are plenty of situations where a +song might appear on more than one album (consider a greatest hits album). +Thus, a song which was played on one album 30 times, and also played on a +different album 20 times, would not receive the credit of being played a total +of 50 times. To resolve this, Zuletzt lets the user specify that these two +songs are the same. This is, however, different from SongGroups. SongGroups, +while superficially providing very similar functionality, does not permanently +combine the statistics of the two songs, but will show their combined +statistics anyways. This is useful if, for example, one song is a remix of +another - they are, in reality, different songs, but there is a clear +connection between them, and it may be interesting to see what their combined +statistics are. The decision to merge songs or make a SongGroup, or neither, is +left to the user, but the general thought is: +- If they're the *exact* same song, merge them, and the data becomes more + accurate for that song +- If one is somehow remixed/covered/altered in some way, make a SongGroup, and + see the combined info *as if* you had merged them. ## To-Do List: - [ ] Entity statistics @@ -80,7 +106,8 @@ If two artists have the same name, they are necessarily listed as the same artis - [ ] Genres - [ ] Owned - [ ] Holiday -- [ ] [MusicBrainz integration](https://musicbrainz.org/doc/libmusicbrainz)[^11] +- [ ] [MusicBrainz + integration](https://musicbrainz.org/doc/libmusicbrainz)[^11] - [ ] Concerts - [ ] Import from Setlist.fm[^5] - [ ] Ratings @@ -90,27 +117,61 @@ If two artists have the same name, they are necessarily listed as the same artis - [ ] "Playlists"[^8] - [ ] First launch setup -[^1]: I do not intend to exactly replicate all the statistics Lastfmstats.com provides, but I would at least like to give the user the option to see those kinds of statistics, or generate them themselves (see 7). +[^1]: I do not intend to exactly replicate all the statistics Lastfmstats.com +provides, but I would at least like to give the user the option to see those +kinds of statistics, or generate them themselves (see 7). -[^2]: I do not intend to provide the level of granularity that Discogs provides, but a simple toggle that means "I own some version of this release" is all that is necessary. +[^2]: I do not intend to provide the level of granularity that Discogs +provides, but a simple toggle that means "I own some version of this release" +is all that is necessary. -[^3]: I have not investigated any other service for downloading your listening history from Last.fm, but providing the listening history as a JSON rather than a CSV is highly preferred. I may eventually provide my own way of downloading Last.fm data as a JSON, but I would prefer to allow users to enter their username, or authenticate, and avoid needing to upload a file altogether. +[^3]: I have not investigated any other service for downloading your listening +history from Last.fm, but providing the listening history as a JSON rather than +a CSV is highly preferred. I may eventually provide my own way of downloading +Last.fm data as a JSON, but I would prefer to allow users to enter their +username, or authenticate, and avoid needing to upload a file altogether. -[^4]: I only intend to allow imports from Last.fm and Spotify at the moment because those are the only data sources I currently rely on. To that extent, I imagine I could import from other sources as well fairly easily, although I do not know what their data dumps look like. +[^4]: I only intend to allow imports from Last.fm and Spotify at the moment +because those are the only data sources I currently rely on. To that extent, I +imagine I could import from other sources as well fairly easily, although I do +not know what their data dumps look like. -[^5]: I only intend to allow imports from Setlist.fm at the moment because that is the only data source I currently rely on. +[^5]: I only intend to allow imports from Setlist.fm at the moment because that +is the only data source I currently rely on. -[^6]: RYM has the most data, and once it has an API, will be the only user-driven review site that *has* an API. In this context, "integration" simply means displaying the critic score and user score next to the album. You will be able to write reviews and ranks songs/albums(/artists?), but not for them to be published to RYM. +[^6]: RYM has the most data, and once it has an API, will be the only +user-driven review site that *has* an API. In this context, "integration" +simply means displaying the critic score and user score next to the album. You +will be able to write reviews and ranks songs/albums(/artists?), but not for +them to be published to RYM. -[^7]: I envision something akin to the Custom Reports from [Actual Budget](https://github.com/actualbudget/actual) that will allow users to create their own ways of rating/ranking songs/albums, and view their listening habits. +[^7]: I envision something akin to the Custom Reports from [Actual +Budget](https://github.com/actualbudget/actual) that will allow users to create +their own ways of rating/ranking songs/albums, and view their listening habits. -[^8]: Misleading title, but same functionality as "Lists" on AlbumOfTheYear, although I would like to allow albums and songs to appear on the same list. +[^8]: Misleading title, but same functionality as "Lists" on AlbumOfTheYear, +although I would like to allow albums and songs to appear on the same list. -[^9]: This is a working title, but I have sources (iPods) that provide a play count, but no play dates, so I can't list them among my usual Scrobbles. However, I would still like to display that information along with everything else, so I would like to provide a way of entering this data into a separate category that can be toggled to display alongside "official" Scrobbles. +[^9]: This is a working title, but I have sources (iPods) that provide a play +count, but no play dates, so I can't list them among my usual Scrobbles. +However, I would still like to display that information along with everything +else, so I would like to provide a way of entering this data into a separate +category that can be toggled to display alongside "official" Scrobbles. [^10]: Would probably select the album with the most scrobbles -[^11]: I probably don't understand it well enough, but it appears that I should be able to do this using `@cImport` and/or `translate-c` on the original MusicBrainz source, but it's not all clear to me on how that would work yet. This is a necessary step for what I have planned however, so we'll see where it goes. **Update 3/25/25:** A Zig implementation for what Zuletzt requires (and *only* what Zuletzt requires) has been (mostly) written. +[^11]: I probably don't understand it well enough, but it appears that I should +be able to do this using `@cImport` and/or `translate-c` on the original +MusicBrainz source, but it's not all clear to me on how that would work yet. +This is a necessary step for what I have planned however, so we'll see where it +goes. **Update 3/25/25:** A Zig implementation for what Zuletzt requires (and +*only* what Zuletzt requires) has been (mostly) written. ## Contributing -I am a math student who is interested in programming. I will not be writing quality code. That said, Zuletzt is something that, at the moment, I am very excited about making, and using to relearn some things about programming. Unless contributions are given in the form of code review, or some kind of constructive criticism, it's not likely that I accept pull requests. The project is, however, licensed under the MIT License, so feel free to do what you like with it in your own way. +I am a math student who is interested in programming. I will +not be writing quality code. That said, Zuletzt is something that, at the +moment, I am very excited about making, and using to relearn some things about +programming. Unless contributions are given in the form of code review, or some +kind of constructive criticism, it's not likely that I accept pull requests. +The project is, however, licensed under the MIT License, so feel free to do +what you like with it in your own way. From 0b7efc3420e7e3d5926a04341a93ae492232aaae Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 24 Jun 2025 00:05:25 -0400 Subject: [PATCH 081/103] Begin album reviews Album reviews would ideally allow you to rate tracks at the same time, so we'll have to work on that next. Also, disambiguation pages are becoming more and more necessary (Little Talks in inaccessible atm) Preferably, we start working on the `INDEX` for `/ratings` as well, and maybe use a unified language for these things (is it review, rating, rating_text, score,...?) --- src/app/views/albums.zig | 37 +++++++++++++++++++++--- src/app/views/albums/get.zmpl | 40 +++++++++++++++++++++----- src/app/views/ratings/albums.zig | 14 +++++++++ src/app/views/ratings/albums/post.zmpl | 1 + src/app/views/songs/get.zmpl | 22 +++++++------- src/queries.zig | 6 ++++ 6 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 src/app/views/ratings/albums.zig create mode 100644 src/app/views/ratings/albums/post.zmpl diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 03046b3..17421e7 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -4,6 +4,7 @@ const jetquery = @import("jetzig").jetquery; const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; const queries = @import("../../queries.zig"); +const decode = @import("../../date_fmt.zig").urlDecode; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -15,19 +16,47 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { + const parse_err = blk: { + const rdr_id = std.fmt.parseInt(i64, id, 10) catch |err| break :blk err; + const album = try jetzig.database.Query(.Album).find(rdr_id).execute(request.repo); + if (album == null) break :blk error.InvalidCharacter; + var name = std.ArrayList(u8).init(request.allocator); + try name.appendSlice("http://127.0.0.1:8080/albums/"); + try name.appendSlice(album.?.name); + return request.redirect(try name.toOwnedSlice(), .found); + }; + + const id_int = switch (parse_err) { + error.Overflow => return request.fail(.not_found), + error.InvalidCharacter => blk: { + const rn = try decode(request.allocator, id); + std.log.debug("{s}", .{rn}); + const songs = try jetzig.database.Query(.Album).where(.{ .name = rn }).all(request.repo); + + if (songs.len == 0) return request.fail(.not_found); + if (songs.len > 1) return request.redirect("http://127.0.0.1:8080", .found); + break :blk songs[0].id; + }, + }; var root = try request.data(.object); - const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id}); + const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id_int}); try root.put("album", album); - const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .get_songs), .{id}); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_scrobbles), .{id_int}); + try root.put("scrobbles", scrobbles); + + const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .get_songs), .{id_int}); try root.put("songs", songs); - const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id}); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id_int}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.loadQuery(.album, .timescale), .{id}); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.album, .timescale), .{id_int}); try root.put("yearly", timescale); + const ratings = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_ratings), .{id_int}); + try root.put("reviews", ratings); + return request.render(.ok); } diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 7e089e7..6dfd7f7 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -1,22 +1,48 @@ @zig { const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; const columns: ColumnChoices = &.{.song, .scrobbles}; + const reviews = try zmpl.coerceArray(".reviews"); } + @partial partials/header +

{{.album.album_name}}

{{.album.artist_name}}

-
{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
-
{{.album.song_num}} songs
-@partial partials/firstlast_listens(firstlast: .firstlast) -

Yearly Performance

-@partial partials/timescale(range: .yearly) -

Songs

-@partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) +
+ +
+
+
{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
+
{{.album.song_num}} songs
+ @partial partials/firstlast_listens(firstlast: .firstlast) +

Yearly Performance

+ @partial partials/timescale(range: .yearly) +

Songs

+ @partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) +
+
+

Rating

+
+ @zig { + if (reviews.len == 0) { +
+ + + +
+ } else { + for (reviews) |review| { + {{review.score}}: {{review.review}} ({{review.date}}) + } + } + } +
+
\ No newline at end of file diff --git a/src/app/views/ratings/albums.zig b/src/app/views/ratings/albums.zig new file mode 100644 index 0000000..a57adfa --- /dev/null +++ b/src/app/views/ratings/albums.zig @@ -0,0 +1,14 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); +pub fn post(request: *jetzig.Request) !jetzig.View { + var root = try request.data(.object); + const params = try request.params(); + const id = params.getT(.integer, "album_id").?; + const score = if (params.getT(.integer, "score")) |score| @as(i16, @truncate(score)) else null; + const review = params.getT(.string, "review"); + try jetzig.database.Query(.Albumrating).insert(.{ .album = id, .rating = score, .rating_text = review, .date = @divFloor(request.start_time, 1_000) }).execute(request.repo); + try root.put("score", score); + try root.put("review", review); + + return request.render(.created); +} diff --git a/src/app/views/ratings/albums/post.zmpl b/src/app/views/ratings/albums/post.zmpl new file mode 100644 index 0000000..ca54fd7 --- /dev/null +++ b/src/app/views/ratings/albums/post.zmpl @@ -0,0 +1 @@ + {{.score}}: {{.review}} (Today) \ No newline at end of file diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index f07b03c..c092ea5 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -28,17 +28,17 @@

Rating

@zig { - if (reviews.len == 0) { -
- - - -
- } else { - for (reviews) |review| { - {{review.score}}: {{review.review}} ({{review.date}}) - } -} + if (reviews.len == 0) { +
+ + + +
+ } else { + for (reviews) |review| { + {{review.score}}: {{review.review}} ({{review.date}}) + } + } }
diff --git a/src/queries.zig b/src/queries.zig index 0c9359e..9010942 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -446,6 +446,12 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\WHERE song = $1 \\ORDER BY date DESC; , + .album => + \\SELECT rating AS score, rating_text AS review, TO_CHAR(date, 'YYY-MM-DD') AS date + \\FROM albumratings + \\WHERE album = $1 + \\ORDER BY date DESC; + , else => unreachable, }, }, From 29041044e7d424b878142c66f707b57bde1bedaa Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 27 Jun 2025 00:31:47 -0400 Subject: [PATCH 082/103] Remove unnecessary null checks --- src/queries.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/queries.zig b/src/queries.zig index 9010942..e298ea3 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -40,10 +40,10 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: .album = if (entity.album_id) |_| .{ .id = entity.album_id.?, .name = entity.album_name.? } else null, .song = if (entity.song_id) |_| .{ .id = entity.song_id.?, .name = entity.song_name.? } else null, .artistlist = if (artist_list.getLastOrNull()) |_| try artist_list.toOwnedSlice() else null, - .scrobbles = if (entity.scrobbles) |scrobbles| scrobbles else null, - .date = if (entity.date) |date| date else null, - .score = if (entity.score) |score| score else null, - .review = if (entity.review) |review| review else null, + .scrobbles = entity.scrobbles, + .date = entity.date, + .score = entity.score, + .review = entity.review, }); } From 5739f89e0d5241d70ea98c103e1ecf950ac5cf32 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 28 Jun 2025 00:19:48 -0400 Subject: [PATCH 083/103] Write "friends" query Will tell you which songs you listen to the most on the days you listen to some specified song --- src/queries.zig | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/queries.zig b/src/queries.zig index e298ea3..a665a58 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -457,3 +457,18 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { }, }; } + +// I'm pretty sure this query will tell you the number of days a song was played on the same day as a specified song +// The output looked right at least. Can easily be changed into the number of times a song was played on the same day +// as a specified song by removing DISTINCT from the first subquery (which would increase the count if a song was +// played more than once in a day) + +//SELECT COUNT(albumsong), name +//FROM ( +// SELECT DISTINCT scrobbles.albumsong, songs.name, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD') +// FROM scrobbles +// INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong +// INNER JOIN songs ON songs.id = albumsongs.song_id +// WHERE TO_CHAR(datetime,'YYYY-MM-DD') IN ( +// SELECT DISTINCT TO_CHAR(datetime,'YYYY-MM-DD') FROM scrobbles WHERE albumsong = $1 +//)) GROUP BY albumsong, name ORDER BY COUNT(albumsong) DESC, name ASC; From 9fa90ff129c6eed92d83f9ac3c1953dffad07dc4 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 2 Jul 2025 16:10:55 -0400 Subject: [PATCH 084/103] Add check if there is a tie in scrobble count --- src/app/views/songs/get.zmpl | 4 ++++ src/queries.zig | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index c092ea5..4d4accc 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -17,7 +17,11 @@
+ @if ($.song.is_tie) +
{{.song.scrobbles}} scrobbles ({{.song.rank}} place, tied)
+ @else
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
+ @end @partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly) diff --git a/src/queries.zig b/src/queries.zig index a665a58..27d1f65 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -346,14 +346,17 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .entity_info => switch (entity) { .scrobble => unreachable, .song => - \\SELECT * FROM - \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM99999th') AS rank FROM - \\(SELECT songs.name AS song_name, songs.id AS song_id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN songs ON albumsongs.song_id = songs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\GROUP BY songs.id)) - \\WHERE song_id = $1 + \\WITH ranked AS ( + \\SELECT songs.name AS song_name, COUNT(songs.id) AS scrobbles, RANK() OVER ( ORDER BY COUNT(songs.id) DESC) AS rank, songs.id AS song_id + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\GROUP BY songs.id) + \\SELECT * FROM (SELECT song_name, scrobbles, TO_CHAR(rank,'FM9999th') AS rank, song_id, + \\( rank = lag(rank, 1, -1::bigint) OVER (ORDER BY rank) + \\OR rank = lead(rank, 1, -1::bigint) OVER (ORDER BY rank) + \\)AS is_tie + \\FROM ranked) WHERE song_id = $1; , .album => \\SELECT album_name, t.album_id, artists.name AS artist_name, artists.id AS artist_id, song_num, scrobbles, rank FROM From f9718f3a370372dae14f034e22fc4435f88cfd49 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 2 Jul 2025 16:11:37 -0400 Subject: [PATCH 085/103] Make loadQuery comptime Will eventually do this for all views --- src/app/views/songs.zig | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 2f1772b..0e85ce6 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -45,22 +45,22 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { }; var root = try request.data(.object); - const song = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_info), .{id_int}); + const song = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .entity_info), .{id_int}); try root.put("song", song); - const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_scrobbles), .{id_int}); + const scrobbles = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_scrobbles), .{id_int}); try root.put("scrobbles", scrobbles); - const albums = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_albums), .{id_int}); + const albums = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_albums), .{id_int}); try root.put("albums", albums); - const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.song, .firstlast), .{id_int}); + const firstlast = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .firstlast), .{id_int}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.loadQuery(.song, .timescale), .{id_int}); + const timescale = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .timescale), .{id_int}); try root.put("yearly", timescale); - const ratings = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_ratings), .{id_int}); + const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); return request.render(.ok); From b0727e77e108c783befb1964af95e1487d06f103 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 2 Jul 2025 19:37:28 -0400 Subject: [PATCH 086/103] Addd peak query and tie detection for rank Also begins friends query for songs --- src/app/views/albums.zig | 3 + src/app/views/albums/get.zmpl | 5 ++ src/app/views/artists.zig | 3 + src/app/views/artists/get.zmpl | 7 +- src/app/views/songs.zig | 3 + src/app/views/songs/get.zmpl | 1 + src/queries.zig | 157 ++++++++++++++++++++++++--------- 7 files changed, 135 insertions(+), 44 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 17421e7..55db929 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -58,5 +58,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const ratings = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); + const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int}); + try root.put("peak", peak); + return request.render(.ok); } diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 6dfd7f7..e4e2196 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -18,7 +18,12 @@
+ @if ($.album.is_tie) +
{{.album.scrobbles}} scrobbles ({{.album.rank}} place, tied)
+ @else
{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
+ @end +
All-time peak: {{.peak.rank}} ({{.peak.date}})
{{.album.song_num}} songs
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 505c3d1..e5f13d5 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -57,5 +57,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const timescale = try queries.entityQueryResult(request, queries.loadQuery(.artist, .timescale), .{id_int}); try root.put("yearly", timescale); + const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int}); + try root.put("peak", peak); + return request.render(.ok); } diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 9f1ff07..9ae4ad3 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -11,7 +11,12 @@ @partial partials/header

{{.artist.artist_name}}

-
{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
+ @if ($.artist.is_tie) +
{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place, tied)
+ @else +
{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
+ @end +
All-time peak: {{.peak.rank}} ({{.peak.date}})
{{.artist.song_num}} songs
{{.artist.album_num}} albums
diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 0e85ce6..31876ba 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -63,5 +63,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); + const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .peak), .{id_int}); + try root.put("peak", peak); + return request.render(.ok); } diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 4d4accc..2acb7d4 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -22,6 +22,7 @@ @else
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
@end +
All-time peak: {{.peak.rank}} ({{.peak.date}})
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly) diff --git a/src/queries.zig b/src/queries.zig index 27d1f65..778e64b 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -3,16 +3,43 @@ const jetzig = @import("jetzig"); const TableRow = @import("types.zig").TableRow; const HyperlinkData = @import("types.zig").HyperlinkData; const std = @import("std"); +const ordinalFmt = @import("./ordinal_fmt.zig").ordinalFmt; -pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: anytype) !*jetzig.Data.Value { +pub fn entityQueryResult(request: *jetzig.Request, comptime query: GeneratedQuery, args: anytype) !*jetzig.Data.Value { //var result = try request.repo.executeSql(query.query, args); // + var Data = jetzig.Data.init(request.allocator); + + if (query.query_type == .peak) { + const id = switch (@TypeOf(args)) { + struct { i64 } => args[0], + else => unreachable, + }; + var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, .{}, .{ .allocator = request.allocator, .column_names = true }); + var out: *jetzig.Data.Value = try Data.object(); + var mapper = result.mapper(PeakResult, .{ .dupe = true, .allocator = request.allocator }); + var rank = comptime (std.math.pow(u64, 2, 63) - 1); + var date: []const u8 = undefined; + var scrobbles = std.AutoArrayHashMap(i64, u32).init(request.allocator); + + while (try mapper.next()) |scrobble| { + const res = try scrobbles.getOrPut(scrobble.eid); + if (res.found_existing) res.value_ptr.* += 1 else res.value_ptr.* = 1; + scrobbles.sort(PeakContext{ .keys = scrobbles.keys(), .vals = scrobbles.values(), .preferred = id }); + const idx = scrobbles.getIndex(id); + if (idx != null and idx.? <= rank) { + if (idx.? < rank) rank = idx.?; + date = scrobble.datetime; + } + } + try out.put("rank", ordinalFmt(request.allocator, @as(i64, @bitCast(rank + 1)))); + try out.put("date", date); + return out; + } var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, args, .{ .allocator = request.allocator, .column_names = true }); defer result.deinit(); - var Data = jetzig.Data.init(request.allocator); - var artist_list = std.ArrayList(HyperlinkData).init(request.allocator); if (query.query_type == .entity_info) { @@ -51,7 +78,22 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: } const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, get_songs, get_albums, get_scrobbles, appears, entity_info, datestreak, entities_by_name, get_ratings }; +const QueryTypeEnum = enum { firstlast, timescale, entities, get_songs, get_albums, get_scrobbles, appears, entity_info, datestreak, entities_by_name, get_ratings, friends, peak }; + +const PeakResult = struct { + eid: i64, + datetime: []const u8, +}; + +const PeakContext = struct { + keys: []i64, + vals: []u32, + preferred: i64, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + if (ctx.vals[a_index] == ctx.vals[b_index] and ctx.keys[a_index] == ctx.preferred) return true; + return ctx.vals[a_index] > ctx.vals[b_index]; + } +}; const GeneratedQuery = struct { entity: EntityType, @@ -84,9 +126,10 @@ const EntityInfoResult = struct { rank: []const u8, song_num: ?i64 = null, album_num: ?i64 = null, + is_tie: bool, }; -pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { +pub fn loadQuery(comptime entity: EntityType, comptime query_type: QueryTypeEnum) GeneratedQuery { return GeneratedQuery{ .entity = entity, .query_type = query_type, @@ -344,7 +387,7 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { }, .entity_info => switch (entity) { - .scrobble => unreachable, + .scrobble => @compileError("Cannot specify scrobble for entity_info"), .song => \\WITH ranked AS ( \\SELECT songs.name AS song_name, COUNT(songs.id) AS scrobbles, RANK() OVER ( ORDER BY COUNT(songs.id) DESC) AS rank, songs.id AS song_id @@ -359,30 +402,34 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\FROM ranked) WHERE song_id = $1; , .album => - \\SELECT album_name, t.album_id, artists.name AS artist_name, artists.id AS artist_id, song_num, scrobbles, rank FROM - \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT albums.name AS album_name, albums.id AS album_id, COUNT(DISTINCT songs.id) AS song_num, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN albums ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\GROUP BY albums.id)) AS t - \\INNER JOIN artistalbums ON artistalbums.album_id = t.album_id - \\INNER JOIN artists ON artists.id = artistalbums.artist_id - \\WHERE t.album_id = $1 - , - .artist => - \\SELECT * FROM - \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles, COUNT(DISTINCT albums.id) AS album_num, COUNT(DISTINCT songs.id) AS song_num - \\FROM albumsongsartists - \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id - \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WITH ranked AS ( + \\SELECT albums.name AS album_name, COUNT(albums.id) AS scrobbles, RANK() OVER ( ORDER BY COUNT(albums.id) DESC) AS rank, albums.id AS album_id, COUNT(DISTINCT songs.id) AS song_num + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN albums ON albums.id = albumsongs.album_id \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\GROUP BY artists.id)) - \\WHERE artist_id = $1 + \\GROUP BY albums.id) + \\SELECT * FROM (SELECT album_name, scrobbles, TO_CHAR(rank,'FM9999th') AS rank, album_id, song_num, + \\( rank = lag(rank, 1, -1::bigint) OVER (ORDER BY rank) + \\OR rank = lead(rank, 1, -1::bigint) OVER (ORDER BY rank) + \\)AS is_tie + \\FROM ranked) WHERE album_id = $1; + , + .artist => + \\WITH ranked AS ( + \\SELECT artists.name AS artist_name, COUNT(artists.id) AS scrobbles, RANK() OVER ( ORDER BY COUNT(artists.id) DESC) AS rank, artists.id AS artist_id, COUNT(DISTINCT songs.id) AS song_num, COUNT(DISTINCT albums.id) AS album_num + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\GROUP BY artists.id) + \\SELECT * FROM (SELECT artist_name, scrobbles, TO_CHAR(rank,'FM9999th') AS rank, artist_id, song_num, album_num, + \\( rank = lag(rank, 1, -1::bigint) OVER (ORDER BY rank) + \\OR rank = lead(rank, 1, -1::bigint) OVER (ORDER BY rank) + \\)AS is_tie + \\FROM ranked) WHERE artist_id = $1; , }, @@ -457,21 +504,45 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , else => unreachable, }, + .friends => switch (entity) { + .song => + \\SELECT name, COUNT(DISTINCT dt) AS days, COUNT(dt) AS plays + \\FROM ( + \\ SELECT scrobbles.albumsong, songs.name, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD') AS dt + \\ FROM scrobbles + \\ INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\ INNER JOIN songs ON songs.id = albumsongs.song_id + \\ WHERE TO_CHAR(datetime,'YYYY-MM-DD') IN ( + \\ SELECT DISTINCT TO_CHAR(datetime,'YYYY-MM-DD') + \\ FROM scrobbles + \\ WHERE albumsong = $1 + \\ ) + \\) GROUP BY albumsong, name ORDER BY days DESC, plays DESC, name ASC; + , + else => unreachable, + }, + .peak => switch (entity) { + .scrobble => @compileError("Cannot specify scrobble for peak"), + .song => + \\SELECT songs.id AS eid, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD') AS datetime FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\ORDER BY datetime ASC; + , + .album => + \\SELECT albums.id AS eid, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD') AS datetime FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\ORDER BY datetime ASC; + , + .artist => + \\SELECT artists.id AS eid, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD') AS datetime FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\ORDER BY datetime ASC; + , + }, }, }; } - -// I'm pretty sure this query will tell you the number of days a song was played on the same day as a specified song -// The output looked right at least. Can easily be changed into the number of times a song was played on the same day -// as a specified song by removing DISTINCT from the first subquery (which would increase the count if a song was -// played more than once in a day) - -//SELECT COUNT(albumsong), name -//FROM ( -// SELECT DISTINCT scrobbles.albumsong, songs.name, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD') -// FROM scrobbles -// INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong -// INNER JOIN songs ON songs.id = albumsongs.song_id -// WHERE TO_CHAR(datetime,'YYYY-MM-DD') IN ( -// SELECT DISTINCT TO_CHAR(datetime,'YYYY-MM-DD') FROM scrobbles WHERE albumsong = $1 -//)) GROUP BY albumsong, name ORDER BY COUNT(albumsong) DESC, name ASC; From 12722f282d09798fc02c087321295754d7d29ec6 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 2 Jul 2025 23:05:57 -0400 Subject: [PATCH 087/103] Revert forced redirect after id in url and begin disambiguation page At first, this was a nightmare. Now, I think I have a good idea about how to do disambiguation pages --- src/app/views/songs.zig | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 31876ba..4be4f43 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -21,29 +21,18 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - const parse_err = blk: { - const rdr_id = std.fmt.parseInt(i64, id, 10) catch |err| break :blk err; - const song = try jetzig.database.Query(.Song).find(rdr_id).execute(request.repo); - if (song == null) break :blk error.InvalidCharacter; - var name = std.ArrayList(u8).init(request.allocator); - try name.appendSlice("http://127.0.0.1:8080/songs/"); - try name.appendSlice(song.?.name); - return request.redirect(try name.toOwnedSlice(), .found); - }; - - const id_int = switch (parse_err) { - error.Overflow => return request.fail(.not_found), - error.InvalidCharacter => blk: { - const rn = try decode(request.allocator, id); - std.log.debug("{s}", .{rn}); - const songs = try jetzig.database.Query(.Song).where(.{ .name = rn }).all(request.repo); - - if (songs.len == 0) return request.fail(.not_found); - if (songs.len > 1) return request.redirect("http://127.0.0.1:8080", .found); - break :blk songs[0].id; - }, - }; var root = try request.data(.object); + const id_int = blk: { + const rn = try decode(request.allocator, id); + const songs = try jetzig.database.Query(.Song).select(.{.id}).where(.{ .name = rn }).all(request.repo); + if (songs.len == 0) { + break :blk std.fmt.parseInt(i64, id, 10) catch return request.fail(.not_found); + } else if (songs.len == 1) { + break :blk songs[0].id; + } else { + return request.redirect("http://127.0.0.1:8080", .found); + } + }; const song = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .entity_info), .{id_int}); try root.put("song", song); From da9934ae1e65d7e62e0c46b2cae14c386e1f81e0 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 2 Jul 2025 23:27:39 -0400 Subject: [PATCH 088/103] Create disambiguation for songs This was way easier than I expected, but I am rather unhappy with some things now. In particular, the GET page is pretty gross. I think there are some things I can do, but I'm not 100% confident. Maybe I'll bring some things up to bob once I have a better picture, but I really want to try to clean up my code --- src/app/views/songs.zig | 17 ++++++++++++----- src/app/views/songs/get.zmpl | 11 ++++++++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 4be4f43..24e4f86 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -24,13 +24,20 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); const id_int = blk: { const rn = try decode(request.allocator, id); - const songs = try jetzig.database.Query(.Song).select(.{.id}).where(.{ .name = rn }).all(request.repo); - if (songs.len == 0) { + // Try to find the song by name + const queried_songs = try jetzig.database.Query(.Song).select(.{.id}).where(.{ .name = rn }).all(request.repo); + if (queried_songs.len == 0) { + // Either we've been given an id in the db, or the song doesn't exist break :blk std.fmt.parseInt(i64, id, 10) catch return request.fail(.not_found); - } else if (songs.len == 1) { - break :blk songs[0].id; + } else if (queried_songs.len == 1) { + // It can only be one song + break :blk queried_songs[0].id; } else { - return request.redirect("http://127.0.0.1:8080", .found); + // It could be a variety of songs + const songs = try queries.entityQueryResult(request, queries.loadQuery(.song, .entities_by_name), .{rn}); + try root.put("songs", songs); + try root.put("disambiguation", true); + return request.render(.ok); } }; diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 2acb7d4..e2f6b54 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -1,7 +1,7 @@ @zig { const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; - const reviews = try zmpl.coerceArray(".reviews"); + const dis_columns: ColumnChoices = &.{.song, .album, .artistlist, .scrobbles}; } @@ -11,6 +11,14 @@ @partial partials/header +@if ($.disambiguation) +

Songs

+@partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: dis_columns) +@else + +@zig { + const reviews = try zmpl.coerceArray(".reviews"); +}

{{.song.song_name}}

@@ -47,5 +55,6 @@ }
+@end \ No newline at end of file From 7b1fc6dd71b72ed355af4b47125c0f6e59ed9566 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 3 Jul 2025 00:09:31 -0400 Subject: [PATCH 089/103] Include track name in disambiguation --- src/app/views/songs.zig | 1 + src/app/views/songs/get.zmpl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 24e4f86..da69feb 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -35,6 +35,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { } else { // It could be a variety of songs const songs = try queries.entityQueryResult(request, queries.loadQuery(.song, .entities_by_name), .{rn}); + try root.put("name", rn); try root.put("songs", songs); try root.put("disambiguation", true); return request.render(.ok); diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index e2f6b54..05e2313 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -12,7 +12,7 @@ @partial partials/header @if ($.disambiguation) -

Songs

+

{{.name}} (disambiguation)

@partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: dis_columns) @else From 15e72ea3262de56b2e15823a354a979b3d649637 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 3 Jul 2025 12:23:57 -0400 Subject: [PATCH 090/103] Revert entity name in url redirect in all cases and create disambiguation pages for all entities Useless for artists right now --- src/app/views/albums.zig | 53 ++++++++++++++++------------------ src/app/views/albums/get.zmpl | 11 ++++++- src/app/views/artists.zig | 53 ++++++++++++++++------------------ src/app/views/artists/get.zmpl | 7 +++++ src/app/views/songs.zig | 6 ++-- src/queries.zig | 20 +++++++++++++ 6 files changed, 90 insertions(+), 60 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 55db929..5f3e0e4 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -16,46 +16,43 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - const parse_err = blk: { - const rdr_id = std.fmt.parseInt(i64, id, 10) catch |err| break :blk err; - const album = try jetzig.database.Query(.Album).find(rdr_id).execute(request.repo); - if (album == null) break :blk error.InvalidCharacter; - var name = std.ArrayList(u8).init(request.allocator); - try name.appendSlice("http://127.0.0.1:8080/albums/"); - try name.appendSlice(album.?.name); - return request.redirect(try name.toOwnedSlice(), .found); - }; - - const id_int = switch (parse_err) { - error.Overflow => return request.fail(.not_found), - error.InvalidCharacter => blk: { - const rn = try decode(request.allocator, id); - std.log.debug("{s}", .{rn}); - const songs = try jetzig.database.Query(.Album).where(.{ .name = rn }).all(request.repo); - - if (songs.len == 0) return request.fail(.not_found); - if (songs.len > 1) return request.redirect("http://127.0.0.1:8080", .found); - break :blk songs[0].id; - }, - }; var root = try request.data(.object); - const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id_int}); + const id_int = blk: { + const rn = try decode(request.allocator, id); + // Try to find the song by name + const queried_albums = try jetzig.database.Query(.Album).select(.{.id}).where(.{ .name = rn }).all(request.repo); + if (queried_albums.len == 0) { + // Either we've been given an id in the db, or the song doesn't exist + break :blk std.fmt.parseInt(i64, id, 10) catch return request.fail(.not_found); + } else if (queried_albums.len == 1) { + // It can only be one song + break :blk queried_albums[0].id; + } else { + // It could be a variety of songs + const albums = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .entities_by_name), .{rn}); + try root.put("name", rn); + try root.put("albums", albums); + try root.put("disambiguation", true); + return request.render(.ok); + } + }; + const album = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .entity_info), .{id_int}); try root.put("album", album); - const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_scrobbles), .{id_int}); + const scrobbles = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_scrobbles), .{id_int}); try root.put("scrobbles", scrobbles); - const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .get_songs), .{id_int}); + const songs = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .get_songs), .{id_int}); try root.put("songs", songs); - const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id_int}); + const firstlast = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .firstlast), .{id_int}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.loadQuery(.album, .timescale), .{id_int}); + const timescale = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .timescale), .{id_int}); try root.put("yearly", timescale); - const ratings = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_ratings), .{id_int}); + const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int}); diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index e4e2196..d62af1c 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -1,7 +1,7 @@ @zig { const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; const columns: ColumnChoices = &.{.song, .scrobbles}; - const reviews = try zmpl.coerceArray(".reviews"); + const dis_columns: ColumnChoices = &.{.album, .artistlist, .scrobbles}; } @@ -11,6 +11,14 @@ @partial partials/header +@if ($.disambiguation) +

{{.name}} (disambiguation)

+@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: dis_columns) +@else + +@zig { + const reviews = try zmpl.coerceArray(".reviews"); +}

{{.album.album_name}}

{{.album.artist_name}}

@@ -49,5 +57,6 @@ }
+@end \ No newline at end of file diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index e5f13d5..540ccd3 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -17,44 +17,41 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - const parse_err = blk: { - const rdr_id = std.fmt.parseInt(i64, id, 10) catch |err| break :blk err; - const artist = try jetzig.database.Query(.Artist).find(rdr_id).execute(request.repo); - if (artist == null) break :blk error.InvalidCharacter; - var name = std.ArrayList(u8).init(request.allocator); - try name.appendSlice("http://127.0.0.1:8080/artists/"); - try name.appendSlice(artist.?.name); - return request.redirect(try name.toOwnedSlice(), .found); - }; - - const id_int = switch (parse_err) { - error.Overflow => return request.fail(.not_found), - error.InvalidCharacter => blk: { - const rn = try decode(request.allocator, id); - std.log.debug("{s}", .{rn}); - const artists = try jetzig.database.Query(.Artist).where(.{ .name = rn }).all(request.repo); - - if (artists.len == 0) return request.fail(.not_found); - if (artists.len > 1) return request.redirect("http://127.0.0.1:8080", .found); - break :blk artists[0].id; - }, - }; - var root = try request.data(.object); - const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id_int}); + const id_int = blk: { + const rn = try decode(request.allocator, id); + // Try to find the song by name + const queried_artists = try jetzig.database.Query(.Artist).select(.{.id}).where(.{ .name = rn }).all(request.repo); + if (queried_artists.len == 0) { + // Either we've been given an id in the db, or the song doesn't exist + break :blk std.fmt.parseInt(i64, id, 10) catch return request.fail(.not_found); + } else if (queried_artists.len == 1) { + // It can only be one song + break :blk queried_artists[0].id; + } else { + // It could be a variety of songs + const artists = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .entities_by_name), .{rn}); + try root.put("name", rn); + try root.put("artists", artists); + try root.put("disambiguation", true); + return request.render(.ok); + } + }; + + const artist = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .entity_info), .{id_int}); try root.put("artist", artist); - const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .get_albums), .{id_int}); + const albums = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .get_albums), .{id_int}); try root.put("albums", albums); - const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id_int}); + const appears = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .appears), .{id_int}); try root.put("appears", appears); - const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.artist, .firstlast), .{id_int}); + const firstlast = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .firstlast), .{id_int}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.loadQuery(.artist, .timescale), .{id_int}); + const timescale = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .timescale), .{id_int}); try root.put("yearly", timescale); const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int}); diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 9ae4ad3..37a5379 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -1,6 +1,7 @@ @zig { const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; const columns: ColumnChoices = &.{.album, .scrobbles}; + const dis_columns: ColumnChoices = &.{.artist, .scrobbles}; } @@ -9,6 +10,11 @@ @partial partials/header +@if ($.disambiguation) +

{{.name}} (disambiguation)

+@partial partials/newtable(T: ColumnChoices, table_data: .artists, columns: dis_columns) +@else +

{{.artist.artist_name}}

@if ($.artist.is_tie) @@ -29,5 +35,6 @@

Albums Featured On

@partial partials/newtable(T: ColumnChoices, table_data: .appears, columns: columns) +@end \ No newline at end of file diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index da69feb..f3d4bf5 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -11,9 +11,9 @@ pub fn index(request: *jetzig.Request) !jetzig.View { try root.put("htmx", htmx_query != null); const songs = if (htmx_query) |name| - try queries.entityQueryResult(request, queries.loadQuery(.song, .entities_by_name), .{name}) + try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .entities_by_name), .{name}) else - try queries.entityQueryResult(request, queries.loadQuery(.song, .entities), .{}); + try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .entities), .{}); try root.put("songs", songs); @@ -34,7 +34,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { break :blk queried_songs[0].id; } else { // It could be a variety of songs - const songs = try queries.entityQueryResult(request, queries.loadQuery(.song, .entities_by_name), .{rn}); + const songs = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .entities_by_name), .{rn}); try root.put("name", rn); try root.put("songs", songs); try root.put("disambiguation", true); diff --git a/src/queries.zig b/src/queries.zig index 778e64b..371c15e 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -487,6 +487,26 @@ pub fn loadQuery(comptime entity: EntityType, comptime query_type: QueryTypeEnum \\GROUP BY songs.id, albums.id, artists.id \\ORDER BY songs.name ASC, scrobbles DESC; , + .album => + \\SELECT albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN albums ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE LOWER(albums.name) LIKE LOWER($1) + \\GROUP BY albums.id, artists.id + \\ORDER BY albums.name ASC, scrobbles DESC; + , + .artist => + \\SELECT artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles + \\FROM scrobbles + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = scrobbles.albumsong + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE LOWER(artists.name) LIKE LOWER($1) + \\GROUP BY artists.id + \\ORDER BY artists.name ASC, scrobbles DESC; + , else => unreachable, }, .get_ratings => switch (entity) { From 851aec3a975be5fd7924f93a47d645dd876fa277 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 12:14:27 -0400 Subject: [PATCH 091/103] I learned about std.math.maxInt --- src/queries.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queries.zig b/src/queries.zig index 371c15e..51d4500 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -18,7 +18,7 @@ pub fn entityQueryResult(request: *jetzig.Request, comptime query: GeneratedQuer var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, .{}, .{ .allocator = request.allocator, .column_names = true }); var out: *jetzig.Data.Value = try Data.object(); var mapper = result.mapper(PeakResult, .{ .dupe = true, .allocator = request.allocator }); - var rank = comptime (std.math.pow(u64, 2, 63) - 1); + var rank = comptime (std.math.maxInt(u64)); var date: []const u8 = undefined; var scrobbles = std.AutoArrayHashMap(i64, u32).init(request.allocator); From 0dec52af0100b42b0451836c453cd24488be5d94 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 16:09:30 -0400 Subject: [PATCH 092/103] Type rank --- src/queries.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queries.zig b/src/queries.zig index 51d4500..c8c313b 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -18,7 +18,7 @@ pub fn entityQueryResult(request: *jetzig.Request, comptime query: GeneratedQuer var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, .{}, .{ .allocator = request.allocator, .column_names = true }); var out: *jetzig.Data.Value = try Data.object(); var mapper = result.mapper(PeakResult, .{ .dupe = true, .allocator = request.allocator }); - var rank = comptime (std.math.maxInt(u64)); + var rank: u64 = comptime std.math.maxInt(u64); var date: []const u8 = undefined; var scrobbles = std.AutoArrayHashMap(i64, u32).init(request.allocator); From 6fe885132a9c227c26364fc5428b2d726da7208f Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 16:40:56 -0400 Subject: [PATCH 093/103] Do not use peak I only tested it on small datasets, and it wasn't so bad, but with my whole LastFM dataset, it is very bad --- src/app/views/albums.zig | 4 ++-- src/app/views/albums/get.zmpl | 1 - src/app/views/artists.zig | 4 ++-- src/app/views/artists/get.zmpl | 1 - src/app/views/songs.zig | 4 ++-- src/app/views/songs/get.zmpl | 1 - 6 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 5f3e0e4..1a3c799 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -55,8 +55,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); - const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int}); - try root.put("peak", peak); + //const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int}); + //try root.put("peak", peak); return request.render(.ok); } diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index d62af1c..7f40eb5 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -31,7 +31,6 @@ @else
{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
@end -
All-time peak: {{.peak.rank}} ({{.peak.date}})
{{.album.song_num}} songs
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 540ccd3..3136def 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -54,8 +54,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const timescale = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .timescale), .{id_int}); try root.put("yearly", timescale); - const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int}); - try root.put("peak", peak); + //const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int}); + //try root.put("peak", peak); return request.render(.ok); } diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 37a5379..89663f8 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -22,7 +22,6 @@ @else
{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
@end -
All-time peak: {{.peak.rank}} ({{.peak.date}})
{{.artist.song_num}} songs
{{.artist.album_num}} albums
diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index f3d4bf5..72d21ce 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -60,8 +60,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); - const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .peak), .{id_int}); - try root.put("peak", peak); + //const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .peak), .{id_int}); + //try root.put("peak", peak); return request.render(.ok); } diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 05e2313..a9416ce 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -30,7 +30,6 @@ @else
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
@end -
All-time peak: {{.peak.rank}} ({{.peak.date}})
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly) From c95ac51e0561f10d881c02f9fdd7ec5d23d06928 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 17:03:08 -0400 Subject: [PATCH 094/103] Prevent upload from crashing if 500 is received from LastFM --- src/app/views/upload.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index ac2c665..f22781d 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -66,6 +66,11 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); std.log.debug("{}: {}", .{ page, r }); + if (@intFromEnum(r.status) == 500) { + page -= 1; + std.time.sleep(3 * std.time.ns_per_s); + continue; + } const response_string = try lastfm_response_buffer.toOwnedSlice(); const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); From 8af6341f95898db84c369a26687163d40173391d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 17:07:14 -0400 Subject: [PATCH 095/103] Switch to defined constant when converting between s/ms/ns --- src/date_fmt.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 9746fe6..28e7b10 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -4,7 +4,7 @@ const Data = @import("types.zig"); pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, std.time.ns_per_s) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } From 902fcd4447fde65eafb518c35c0d8d63506e510e Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 17:13:48 -0400 Subject: [PATCH 096/103] Create own parsing function I have dreamt of this for a long time. It is a minor optimization to be honest, but I previously had to choose between ugly code (what I had just prior to this) or looping through the data twice (slow). This parses the data and puts it into my intermediary type directly, along with relevant information about whetehr or not the scrobble is valid, and then I only need to loop over it once with "nice enough" code. There is still more I can do. My ultimate goal is to remove the looping entirely, and verify the data as it's being parsed, queuing up much smaller jobs that handle the individual entities (albums, artistsongs, etc.) as they come, rather than collecting it all and running it at once. I can get over the problem of needing to wit for all the LastFM responses that way as well. Furthermore, the data checking in the function expects a rather rigid structure that I'm not certain I can guarantee, but I'm pretty sure it'll stay that way. In any case, it may behoove me to make it more dynamic at some point. In any case, I am very excited about this change, and I hope I can continue improving upon it. --- src/app/views/upload.zig | 331 ++++++++++----------------------------- src/date_fmt.zig | 230 +++++++++++++++++++++++++++ src/types.zig | 14 ++ 3 files changed, 328 insertions(+), 247 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index f22781d..15deee3 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -47,22 +47,21 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - const imported_scrobbles: Data.ImportedScrobbles = switch (source) { - 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles }, - 1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) }, + const imported_scrobbles: []Data.UnifiedScrobble = switch (source) { + 0, 1 => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), 2 => blk: { const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); - var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator); + var scrobble_buffer = std.ArrayList(Data.UnifiedScrobble).init(request.allocator); const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; var page: usize = 1; - var max_pages: ?usize = null; + //var max_pages: ?usize = null; while (true) : (page += 1) { - if (max_pages != null and page > max_pages.?) break; + if (page > 91) break; const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); std.log.debug("{}: {}", .{ page, r }); @@ -72,13 +71,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View { continue; } const response_string = try lastfm_response_buffer.toOwnedSlice(); - const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); - //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); - if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); - try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track); + const parsed_lastfm_response = try Utils.scrobbleIngest(request.allocator, response_string); + //const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); + //if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); + try scrobble_buffer.appendSlice(parsed_lastfm_response); } - break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items }; + break :blk try scrobble_buffer.toOwnedSlice(); }, else => unreachable, }; @@ -90,254 +89,92 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var albumsongs = try job.params.put("albumsongs", .object); var albumsongsartists = try job.params.put("albumsongsartists", .object); - // Not sure if I should be proud or feel sick - switch (imported_scrobbles) { - .LastFMStats => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { - limited_tracks += 1; - continue :appends; - } - const filtered_scrobble = Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date * 1_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + appends: for (imported_scrobbles) |scrobble| { + if (scrobble.date > latest_timestamp * std.time.ns_per_s or scrobble.date < earliest_timestamp * std.time.ns_per_s) { + limited_tracks += 1; + continue :appends; + } + if (scrobble.playtime != null and scrobble.playtime.? < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.track_artist == null or scrobble.album_artist == null or scrobble.track == null) { + skipped_tracks += 1; + continue :appends; + } - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + const filtered_scrobble = Data.Scrobble{ + .album = scrobble.album.?, + .artists_album = &.{scrobble.album_artist.?}, + .artists_track = &.{scrobble.track_artist.?}, + .date = scrobble.date, + .track = scrobble.track.?, + }; - try view_params.append(row); - //try job_params.append(complete_scrobble); - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + try view_params.append(row); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); } - }, - .LastFMWeb => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.date == null) continue :appends; - const filtered_scrobble = Data.Scrobble{ - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, - .track = scrobble.name, - .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + } - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - try view_params.append(row); - //try job_params.append(complete_scrobble); - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } - - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); } - }, - .Spotify => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { - limited_tracks += 1; - continue :appends; - } - - const filtered_scrobble = Data.Scrobble{ - .album = scrobble.master_metadata_album_album_name.?, - .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .track = scrobble.master_metadata_track_name.?, - .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; - - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - - try view_params.append(row); - //try job_params.append(complete_scrobble); - - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); - - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } - - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } - } - }, + } } std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 28e7b10..da677f9 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -50,3 +50,233 @@ pub fn urlDecode(allocator: std.mem.Allocator, str: []const u8) ![]const u8 { } return decoded.toOwnedSlice(); } + +const ScrobbleFields = enum { + date, // LastFM(Stats) timestamp + ts, // Spotify timestamp + name, // LastFM track name + track, // LastFMStats track name + master_metadata_track_name, // Spotify track name + artist, // LastFM(Stats) artist name + master_metadata_album_artist_name, // Spotify artist name + album, // LastFM(Stats) album name + master_metadata_album_album_name, // Spotify album name + ms_played, // Spotify playtime + reason_end, // Spotify reason end,1_000 + @"@attr", // LastFM now playing + irrelevant, +}; + +pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.UnifiedScrobble { + var scanner = std.json.Scanner.initCompleteInput(allocator, input); + defer scanner.deinit(); + + var out = std.ArrayList(Data.UnifiedScrobble).init(allocator); + + array: switch (try scanner.peekNextTokenType()) { + .array_begin => { + // Go into array + _ = try scanner.next(); + while (try scanner.peekNextTokenType() != .array_end) { + var r: Data.UnifiedScrobble = undefined; + // Go into object + _ = try scanner.next(); + while (try scanner.peekNextTokenType() != .object_end) { + const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const field_name = std.meta.stringToEnum(ScrobbleFields, switch (key_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }) orelse .irrelevant; + switch (field_name) { + .@"@attr" => { + freeAllocated(allocator, key_token); + r = undefined; + try scanner.skipUntilStackHeight(3); + }, + .ts, .date => |d| { + freeAllocated(allocator, key_token); + const date = switch (d) { + .date => blk: { + if (try scanner.peekNextTokenType() == .object_begin) { + // For now, try to just skip over the object_begin and assume the next field is uts + _ = try scanner.next(); + try scanner.skipValue(); + const lfw_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + try scanner.skipValue(); + try scanner.skipValue(); + _ = try scanner.next(); + const lfw_date = try std.fmt.parseInt(i64, switch (lfw_date_token) { + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }, 10) * std.time.ns_per_s; + freeAllocated(allocator, lfw_date_token); + break :blk lfw_date; + } else { + const lfs_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const lfs_date = try std.fmt.parseInt(i64, switch (lfs_date_token) { + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }, 10) * std.time.ns_per_ms; + freeAllocated(allocator, lfs_date_token); + break :blk lfs_date; + } + }, + .ts => blk: { + // This might need to be an alloc_always, but I'm gonna try if_needed first + const spotify_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const spotify_date = try zeit.instant(.{ .source = .{ .iso8601 = switch (spotify_date_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + } } }); + freeAllocated(allocator, spotify_date_token); + break :blk spotify_date.unixTimestamp() * std.time.ns_per_s; + }, + else => unreachable, + }; + @field(r, "date") = date; + }, + .ms_played => { + freeAllocated(allocator, key_token); + const spotify_ms_played = try scanner.nextAlloc(allocator, .alloc_if_needed); + @field(r, "playtime") = try std.fmt.parseInt(u64, switch (spotify_ms_played) { + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }, 10); + freeAllocated(allocator, spotify_ms_played); + }, + .master_metadata_track_name, .track, .name => { + freeAllocated(allocator, key_token); + const track = try scanner.nextAlloc(allocator, .alloc_always); + @field(r, "track") = switch (track) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }, + .master_metadata_album_artist_name, .artist => { + freeAllocated(allocator, key_token); + const artist = if (try scanner.peekNextTokenType() == .object_begin) blk: { + // Skip object_begin, mbid key, mbid, and #text key + _ = try scanner.next(); + try scanner.skipValue(); + try scanner.skipValue(); + try scanner.skipValue(); + const lfw_artist_token = try scanner.nextAlloc(allocator, .alloc_always); + // Leave object + _ = try scanner.next(); + break :blk switch (lfw_artist_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + } else blk: { + const artist_token = try scanner.nextAlloc(allocator, .alloc_always); + break :blk switch (artist_token) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }; + @field(r, "track_artist") = artist; + @field(r, "album_artist") = artist; + }, + .master_metadata_album_album_name, .album => { + freeAllocated(allocator, key_token); + const album = if (try scanner.peekNextTokenType() == .object_begin) blk: { + // Skip object_begin, mbid key, mbid, and #text key + _ = try scanner.next(); + try scanner.skipValue(); + try scanner.skipValue(); + try scanner.skipValue(); + const lfw_album_token = try scanner.nextAlloc(allocator, .alloc_always); + // Leave object + _ = try scanner.next(); + break :blk switch (lfw_album_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + } else blk: { + const album_token = try scanner.nextAlloc(allocator, .alloc_always); + break :blk switch (album_token) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }; + @field(r, "album") = album; + }, + .reason_end => { + freeAllocated(allocator, key_token); + const reason_end = try scanner.nextAlloc(allocator, .alloc_always); + @field(r, "reason_end") = switch (reason_end) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }, + else => { + freeAllocated(allocator, key_token); + try scanner.skipValue(); + }, + } + } + // Exit object + _ = try scanner.next(); + try out.append(r); + } + }, + // LastFM(stats) + .object_begin => { + _ = try scanner.next(); + find_array: while (true) { + const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const field_name = switch (key_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + if (!std.mem.eql(u8, field_name, "scrobbles") and !std.mem.eql(u8, field_name, "recenttracks")) { + freeAllocated(allocator, key_token); + try scanner.skipValue(); + } else { + freeAllocated(allocator, key_token); + break :find_array; + } + } + switch (try scanner.peekNextTokenType()) { + // LastFM Stats + .array_begin => continue :array .array_begin, + // LastFM + .object_begin => { + // Enter recenttracks + _ = try scanner.next(); + while (true) { + const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const field_name = switch (key_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + if (!std.mem.eql(u8, field_name, "track")) { + freeAllocated(allocator, key_token); + try scanner.skipValue(); + } else { + freeAllocated(allocator, key_token); + continue :array .array_begin; + } + } + }, + else => unreachable, + } + }, + else => return error.UnexpectedToken, + } + const scrobbles = try out.toOwnedSlice(); + return scrobbles; +} + +fn freeAllocated(allocator: std.mem.Allocator, token: std.json.Token) void { + switch (token) { + .allocated_number, .allocated_string => |slice| { + allocator.free(slice); + }, + else => {}, + } +} diff --git a/src/types.zig b/src/types.zig index 0a66ae7..df067e8 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,3 +1,17 @@ +const std = @import("std"); + +pub const UnifiedScrobble = struct { + track: ?[]const u8, + // These can be null per Spotify + track_artist: ?[]const u8, + album: ?[]const u8, + album_artist: ?[]const u8, + date: i64, + // Relevant Spotify data + playtime: ?u64 = null, + reason_end: ?[]const u8 = null, +}; + pub const ImportedScrobbles = union(ScrobbleSources) { LastFMStats: []IgnorantScrobble, LastFMWeb: []LastFMWebScrobble, From 6aac0bff2ba12a5e4c4c7b1ab6698e064f8157b9 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 9 Jul 2025 12:09:45 -0400 Subject: [PATCH 097/103] Remove .string branch if using .alloc_always --- src/date_fmt.zig | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/date_fmt.zig b/src/date_fmt.zig index da677f9..5a883b4 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -102,6 +102,7 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U _ = try scanner.next(); try scanner.skipValue(); const lfw_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + // Skip over human-readable date format and then leave try scanner.skipValue(); try scanner.skipValue(); _ = try scanner.next(); @@ -148,7 +149,7 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U freeAllocated(allocator, key_token); const track = try scanner.nextAlloc(allocator, .alloc_always); @field(r, "track") = switch (track) { - inline .string, .allocated_string => |slice| slice, + .allocated_string => |slice| slice, .null => null, else => return error.UnexpectedToken, }; @@ -165,13 +166,13 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U // Leave object _ = try scanner.next(); break :blk switch (lfw_artist_token) { - inline .string, .allocated_string => |slice| slice, + .allocated_string => |slice| slice, else => return error.UnexpectedToken, }; } else blk: { const artist_token = try scanner.nextAlloc(allocator, .alloc_always); break :blk switch (artist_token) { - inline .string, .allocated_string => |slice| slice, + .allocated_string => |slice| slice, .null => null, else => return error.UnexpectedToken, }; @@ -191,13 +192,13 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U // Leave object _ = try scanner.next(); break :blk switch (lfw_album_token) { - inline .string, .allocated_string => |slice| slice, + .allocated_string => |slice| slice, else => return error.UnexpectedToken, }; } else blk: { const album_token = try scanner.nextAlloc(allocator, .alloc_always); break :blk switch (album_token) { - inline .string, .allocated_string => |slice| slice, + .allocated_string => |slice| slice, .null => null, else => return error.UnexpectedToken, }; @@ -208,7 +209,7 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U freeAllocated(allocator, key_token); const reason_end = try scanner.nextAlloc(allocator, .alloc_always); @field(r, "reason_end") = switch (reason_end) { - inline .string, .allocated_string => |slice| slice, + .allocated_string => |slice| slice, .null => null, else => return error.UnexpectedToken, }; From cd8c798bd4eb1f4ef1c0e0d3b4db198295634814 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 14 Jul 2025 14:01:48 -0400 Subject: [PATCH 098/103] Fix incorrect pairing function --- src/app/views/upload.zig | 6 ++++-- src/date_fmt.zig | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 15deee3..3e9e8ba 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -177,12 +177,14 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } } - std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); + std.log.debug("Skipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); try job.schedule(); return request.render(.created); } +// Cantor Pairing Function +// https://en.wikipedia.org/wiki/Pairing_function fn pair(a: u64, b: u64) u64 { - return @divFloor((a +% b) *% (a +% b +% 1) +% b, 2); + return @mod(@divFloor((a +% b) *% (a +% b +% 1), 2) +% b, std.math.maxInt(u64)); } diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 5a883b4..e6914a4 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -64,7 +64,7 @@ const ScrobbleFields = enum { ms_played, // Spotify playtime reason_end, // Spotify reason end,1_000 @"@attr", // LastFM now playing - irrelevant, + irrelevant, // Not a field I care about }; pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.UnifiedScrobble { From 682eebc9514846983adf24913eb32505adbbc30a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 14 Jul 2025 14:03:20 -0400 Subject: [PATCH 099/103] Create buffer for signed hashes rather than using arraylist Also fixes bug with artistalbums hash. i64 will only ever take up 20 characters --- src/app/views/upload.zig | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 3e9e8ba..b21fa17 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -89,6 +89,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var albumsongs = try job.params.put("albumsongs", .object); var albumsongsartists = try job.params.put("albumsongsartists", .object); + var hash_buffer = [_]u8{undefined} ** 20; // A minimum i64 needs 19 digits + 1 negative sign + appends: for (imported_scrobbles) |scrobble| { if (scrobble.date > latest_timestamp * std.time.ns_per_s or scrobble.date < earliest_timestamp * std.time.ns_per_s) { limited_tracks += 1; @@ -124,18 +126,20 @@ pub fn post(request: *jetzig.Request) !jetzig.View { try album_hash_string.appendSlice(artist); const artist_hash = std.hash.Fnv1a_64.hash(artist); try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + //const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + const signed_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artist_hash))}); if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); } try album_hash_string.appendSlice(complete_scrobble.album); const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + const signed_album_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(album_hash))}); if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); for (stored_artist_hashes.items) |artist_hash| { const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + //const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + const signed_artistalbums_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); if (tracks.get(signed_artistalbums_hash_string) == null) { var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); @@ -147,28 +151,28 @@ pub fn post(request: *jetzig.Request) !jetzig.View { try track_hash_string.appendSlice(complete_scrobble.album); try track_hash_string.appendSlice(complete_scrobble.track); const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + const signed_track_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(track_hash))}); if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + const signed_albumsong_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(albumsong_hash))}); if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); + try albumsong_scrobbles.?.append(@divFloor(complete_scrobble.date, std.time.ns_per_us)); // MICROSECONDS } else { var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); try albumsong.put("album", @as(i64, @bitCast(album_hash))); try albumsong.put("song", @as(i64, @bitCast(track_hash))); var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); + try albumsong_scrobbles.append(@divFloor(complete_scrobble.date, std.time.ns_per_us)); // MICROSECONDS } for (complete_scrobble.artists_track) |artist| { const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + const signed_artist_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artist_hash))}); if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + const signed_albumsongsartist_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(albumsongsartist_hash))}); if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); From 280cba2f9a07e42c57c165e2ce206fadb0218503 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 14 Jul 2025 14:04:22 -0400 Subject: [PATCH 100/103] Switch to expectParams() rather than params() Makes some code nicer, particularly date parsing --- src/app/views/upload.zig | 47 ++++++++++++++++++--------------- src/app/views/upload/index.zmpl | 27 ++++++++++++++----- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index b21fa17..0beaf95 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -15,7 +15,17 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const params = try request.params(); + //const params = try request.params(); + + const UploadParams = struct { + source: enum { LFMW, LFMS, Spotify }, + earliest_date: ?[]const u8, + latest_date: ?[]const u8, + username: ?[]const u8, + }; + + const params = (try request.expectParams(UploadParams)).?; + const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), else => err, @@ -26,43 +36,39 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; //var job = try request.job("process_scrobbles"); var job = try request.job("process_scrobbles2"); - const source = params.getT(.integer, "t").?; // This param is required in HTML - const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { - const date = params.getT(.string, "latest-date").?; - break :blk try zeit.Time.fromISO8601(date); - } else (try zeit.instant(.{ .source = .now })).time(); + // We can parse the dates better + const latest_ts = if (params.latest_date) |ld| + (try zeit.instant(.{ .source = .{ .iso8601 = ld } })).timestamp + else + (try zeit.instant(.{ .source = .now })).timestamp; - const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { - const date = params.getT(.string, "earliest-date").?; - break :blk try zeit.Time.fromISO8601(date); - } else (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); - - const earliest_timestamp = earliest_date.instant().unixTimestamp(); - const latest_timestamp = latest_date.instant().unixTimestamp(); + const earliest_ts = if (params.earliest_date) |ed| + (try zeit.instant(.{ .source = .{ .iso8601 = ed } })).timestamp + else + (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).timestamp; var view_params = try root.put("scrobbles", .array); - //var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - const imported_scrobbles: []Data.UnifiedScrobble = switch (source) { - 0, 1 => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), - 2 => blk: { + const imported_scrobbles: []Data.UnifiedScrobble = switch (params.source) { + .LFMS, .Spotify => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), + .LFMW => blk: { const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); var scrobble_buffer = std.ArrayList(Data.UnifiedScrobble).init(request.allocator); - const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; + const username = if (params.username) |un| un else "VAOTM"; var page: usize = 1; //var max_pages: ?usize = null; while (true) : (page += 1) { if (page > 91) break; - const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); + const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, @divFloor(earliest_ts, std.time.ns_per_s), @divFloor(latest_ts, std.time.ns_per_s), page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); std.log.debug("{}: {}", .{ page, r }); if (@intFromEnum(r.status) == 500) { @@ -79,7 +85,6 @@ pub fn post(request: *jetzig.Request) !jetzig.View { break :blk try scrobble_buffer.toOwnedSlice(); }, - else => unreachable, }; var artists = try job.params.put("artists", .object); @@ -92,7 +97,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var hash_buffer = [_]u8{undefined} ** 20; // A minimum i64 needs 19 digits + 1 negative sign appends: for (imported_scrobbles) |scrobble| { - if (scrobble.date > latest_timestamp * std.time.ns_per_s or scrobble.date < earliest_timestamp * std.time.ns_per_s) { + if (scrobble.date > latest_ts or scrobble.date < earliest_ts) { limited_tracks += 1; continue :appends; } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 420e08c..bdeaacb 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -14,14 +14,29 @@
- Last.fm - Spotify - Last.fm (WebAuth) + Last.fm + Spotify + Last.fm (WebAuth) - Advanced Options - Limit to Scrobbles before: - Limit to Scrobbles after: + Advanced Options +
+ + From b0f7884f84b6dd1aaac70b5ed72131c095c08abc Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 14 Jul 2025 17:10:59 -0400 Subject: [PATCH 101/103] Move rule loading in upload.zig into a function --- src/app/views/upload.zig | 38 +++++++++----------------------------- src/date_fmt.zig | 12 ++++++++++++ 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 0beaf95..e351a91 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -13,10 +13,6 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { } pub fn post(request: *jetzig.Request) !jetzig.View { - var root = try request.data(.object); - - //const params = try request.params(); - const UploadParams = struct { source: enum { LFMW, LFMS, Spotify }, earliest_date: ?[]const u8, @@ -26,29 +22,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const params = (try request.expectParams(UploadParams)).?; - const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), - else => err, - }); - - defer rule_file.close(); - const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; - //var job = try request.job("process_scrobbles"); - var job = try request.job("process_scrobbles2"); - - // We can parse the dates better - const latest_ts = if (params.latest_date) |ld| - (try zeit.instant(.{ .source = .{ .iso8601 = ld } })).timestamp - else - (try zeit.instant(.{ .source = .now })).timestamp; - - const earliest_ts = if (params.earliest_date) |ed| - (try zeit.instant(.{ .source = .{ .iso8601 = ed } })).timestamp - else - (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).timestamp; - - var view_params = try root.put("scrobbles", .array); + const latest_ts = (try zeit.instant(.{ .source = if (params.latest_date) |ld| .{ .iso8601 = ld } else .now })).timestamp; + const earliest_ts = (try zeit.instant(.{ .source = if (params.earliest_date) |ed| .{ .iso8601 = ed } else .{ .unix_timestamp = 0 } })).timestamp; var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; @@ -87,6 +62,12 @@ pub fn post(request: *jetzig.Request) !jetzig.View { }, }; + var root = try request.data(.object); + var view_params = try root.put("scrobbles", .array); + + var job = try request.job("process_scrobbles2"); + const rule_list = try Utils.loadRules(request.allocator); + var artists = try job.params.put("artists", .object); var albums = try job.params.put("albums", .object); var tracks = try job.params.put("tracks", .object); @@ -121,9 +102,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - try view_params.append(row); - //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); var album_hash_string = std.ArrayList(u8).init(request.allocator); diff --git a/src/date_fmt.zig b/src/date_fmt.zig index e6914a4..cb6cee8 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -67,6 +67,18 @@ const ScrobbleFields = enum { irrelevant, // Not a field I care about }; +pub fn loadRules(allocator: std.mem.Allocator) !?[]Data.Rule { + const rule_file: std.fs.File = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), + else => err, + }); + + defer rule_file.close(); + const rule_file_size = try rule_file.stat().size; + const rule_file_content = try rule_file.readToEndAlloc(allocator, rule_file_size); + return std.json.parseFromSliceLeaky([]Data.Rule, allocator, rule_file_content, .{}) catch null; +} + pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.UnifiedScrobble { var scanner = std.json.Scanner.initCompleteInput(allocator, input); defer scanner.deinit(); From 2a42e07df09b2d9dbe94f26fcf56ddb894ff4a66 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 14 Jul 2025 18:04:46 -0400 Subject: [PATCH 102/103] Cleanup --- src/app/views/upload.zig | 4 ++-- src/date_fmt.zig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index e351a91..740fa8b 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -111,7 +111,6 @@ pub fn post(request: *jetzig.Request) !jetzig.View { try album_hash_string.appendSlice(artist); const artist_hash = std.hash.Fnv1a_64.hash(artist); try stored_artist_hashes.append(artist_hash); - //const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); const signed_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artist_hash))}); if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); } @@ -123,7 +122,6 @@ pub fn post(request: *jetzig.Request) !jetzig.View { for (stored_artist_hashes.items) |artist_hash| { const artistalbum_hash = pair(artist_hash, album_hash); - //const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); const signed_artistalbums_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); if (tracks.get(signed_artistalbums_hash_string) == null) { var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); @@ -132,6 +130,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } } + // The track hash does not currently include the track artists. Probably not necessary for it to, + // but easily can if need be var track_hash_string = std.ArrayList(u8).init(request.allocator); try track_hash_string.appendSlice(complete_scrobble.album); try track_hash_string.appendSlice(complete_scrobble.track); diff --git a/src/date_fmt.zig b/src/date_fmt.zig index cb6cee8..b8838ad 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -74,7 +74,7 @@ pub fn loadRules(allocator: std.mem.Allocator) !?[]Data.Rule { }); defer rule_file.close(); - const rule_file_size = try rule_file.stat().size; + const rule_file_size = (try rule_file.stat()).size; const rule_file_content = try rule_file.readToEndAlloc(allocator, rule_file_size); return std.json.parseFromSliceLeaky([]Data.Rule, allocator, rule_file_content, .{}) catch null; } From 5f451868af84602f607d3e9611620c4cdb1e693a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 19 Jul 2025 18:25:10 -0400 Subject: [PATCH 103/103] `scrobbleIngest` The code sucks. The functionality isn't quite what we hoped for. Dates are formatted incorrectly. There are practically no comments. It's not modular whatsoever. I lost several years of my life trying to make this work. LGTM --- src/app/jobs/add_album.zig | 31 +++++ src/app/jobs/add_artist.zig | 25 ++++ src/app/jobs/add_song.zig | 34 +++++ src/app/jobs/process_scrobbles.zig | 134 ------------------ src/app/views/upload.zig | 139 ++---------------- src/apply_rule.zig | 33 +++-- src/date_fmt.zig | 217 ++++++++++++++++++++++++++--- src/types.zig | 105 +++----------- 8 files changed, 346 insertions(+), 372 deletions(-) create mode 100644 src/app/jobs/add_album.zig create mode 100644 src/app/jobs/add_artist.zig create mode 100644 src/app/jobs/add_song.zig delete mode 100644 src/app/jobs/process_scrobbles.zig diff --git a/src/app/jobs/add_album.zig b/src/app/jobs/add_album.zig new file mode 100644 index 0000000..a5d46a5 --- /dev/null +++ b/src/app/jobs/add_album.zig @@ -0,0 +1,31 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +// The `run` function for a job is invoked every time the job is processed by a queue worker +// (or by the Jetzig server if the job is processed in-line). +// +// Arguments: +// * allocator: Arena allocator for use during the job execution process. +// * params: Params assigned to a job (from a request, values added to response data). +// * env: Provides the following fields: +// - logger: Logger attached to the same stream as the Jetzig server. +// - environment: Enum of `{ production, development }`. +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + const artists = params.getT(.array, "artists").?.items(); + const album_id = try (params.get("album_hash").?).coerce(u64); + + for (artists) |artist| { + const artist_name = try artist.coerce([]const u8); + const artist_id = std.hash.Fnv1a_64.hash(artist_name); + const paired = @as(i64, @bitCast(@mod(@divFloor((artist_id +% album_id) *% (artist_id +% album_id +% 1), 2) +% album_id, std.math.maxInt(u64)))); + const aa_query = try jetzig.database.Query(.Artistalbum) + .find(paired).execute(env.repo); + + if (aa_query == null) { + try jetzig.database.Query(.Artistalbum) + .insert(.{ .id = paired, .artist_id = @as(i64, @bitCast(artist_id)), .album_id = @as(i64, @bitCast(album_id)) }) + .execute(env.repo); + } + } +} diff --git a/src/app/jobs/add_artist.zig b/src/app/jobs/add_artist.zig new file mode 100644 index 0000000..06b5b25 --- /dev/null +++ b/src/app/jobs/add_artist.zig @@ -0,0 +1,25 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +// The `run` function for a job is invoked every time the job is processed by a queue worker +// (or by the Jetzig server if the job is processed in-line). +// +// Arguments: +// * allocator: Arena allocator for use during the job execution process. +// * params: Params assigned to a job (from a request, values added to response data). +// * env: Provides the following fields: +// - logger: Logger attached to the same stream as the Jetzig server. +// - environment: Enum of `{ production, development }`. +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + const artist = params.getT(.string, "artist").?; + const id = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist))); + const artist_query = try jetzig.database.Query(.Artist) + .find(id).execute(env.repo); + + if (artist_query == null) { + try jetzig.database.Query(.Artist) + .insert(.{ .id = id, .name = artist }) + .execute(env.repo); + } +} diff --git a/src/app/jobs/add_song.zig b/src/app/jobs/add_song.zig new file mode 100644 index 0000000..6f7c504 --- /dev/null +++ b/src/app/jobs/add_song.zig @@ -0,0 +1,34 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +// The `run` function for a job is invoked every time the job is processed by a queue worker +// (or by the Jetzig server if the job is processed in-line). +// +// Arguments: +// * allocator: Arena allocator for use during the job execution process. +// * params: Params assigned to a job (from a request, values added to response data). +// * env: Provides the following fields: +// - logger: Logger attached to the same stream as the Jetzig server. +// - environment: Enum of `{ production, development }`. +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + //const album = params.getT(.string, "album").?; + const as_id = try (params.get("as_hash").?).coerce(u64); + const album_artists = params.getT(.array, "album_artists").?.items(); + // Will use this eventually, but not now + // const track_artists = params.getT(.array,"track_artists"); + + for (album_artists) |artist| { + const artist_name = try artist.coerce([]const u8); + const artist_id = std.hash.Fnv1a_64.hash(artist_name); + const asa_id = @as(i64, @bitCast(@mod(@divFloor((as_id +% artist_id) *% (as_id +% artist_id +% 1), 2) +% artist_id, std.math.maxInt(u64)))); + const asa_query = try jetzig.database.Query(.Albumsongsartist) + .find(asa_id).execute(env.repo); + + if (asa_query == null) { + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ .id = asa_id, .albumsong_id = @as(i64, @bitCast(as_id)), .artist_id = @as(i64, @bitCast(artist_id)) }) + .execute(env.repo); + } + } +} diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig deleted file mode 100644 index 770bde7..0000000 --- a/src/app/jobs/process_scrobbles.zig +++ /dev/null @@ -1,134 +0,0 @@ -const std = @import("std"); -const jetzig = @import("jetzig"); -const jetquery = @import("jetzig").jetquery; -const Data = @import("../../types.zig"); -const rules = @import("../../apply_rule.zig"); - -// The `run` function for a job is invoked every time the job is processed by a queue worker -// (or by the Jetzig server if the job is processed in-line). -// -// Arguments: -// * allocator: Arena allocator for use during the job execution process. -// * params: Params assigned to a job (from a request, values added to response data). -// * env: Provides the following fields: -// - logger: Logger attached to the same stream as the Jetzig server. -// - environment: Enum of `{ production, development }`. -pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { - //_ = env; - if (params.getT(.array, "scrobbles")) |scrobbles| { - for (scrobbles.items()) |item| { - - // Probably want to include artist name here, but not sure how to yet - - const track_artists = item.getT(.array, "artists_track").?.items(); - const album_artists = item.getT(.array, "artists_album").?.items(); - - var track_artist_name_buffer = try allocator.alloc([]const u8, track_artists.len); - var album_artist_name_buffer = try allocator.alloc([]const u8, album_artists.len); - var track_artist_id_buffer = try allocator.alloc(i64, track_artists.len); - var album_artist_id_buffer = try allocator.alloc(i64, album_artists.len); - - const scrobble: Data.Scrobble = .{ - .track = item.getT(.string, "track").?, - .artists_track = track_artist_name_buffer, - .album = item.getT(.string, "album") orelse "", - .artists_album = album_artist_name_buffer, - .date = @as(i64, @truncate(item.getT(.integer, "date").?)), - }; - - var album_hash_string = std.ArrayList(u8).init(allocator); - var track_hash_string = std.ArrayList(u8).init(allocator); - - // I theoretically don't need this for loop - for (track_artists, 0..track_artists.len) |artist, i| { - const artist_name = try artist.coerce([]const u8); - track_artist_name_buffer[i] = artist_name; - track_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); - } - - for (album_artists, 0..album_artists.len) |artist, i| { - const artist_name = try artist.coerce([]const u8); - album_artist_name_buffer[i] = artist_name; - album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); - try album_hash_string.appendSlice(artist_name); - } - - try album_hash_string.appendSlice(scrobble.album); - try track_hash_string.appendSlice(scrobble.album); - const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(album_hash_string.items))); - try track_hash_string.appendSlice(scrobble.track); - const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(track_hash_string.items))); - - var albumsong_id = try jetzig.database.Query(.Albumsong) - .find(album_hash ^ track_hash) - .select(.{.id}).execute(env.repo); - - var album_id = try jetzig.database.Query(.Album) - .find(album_hash) - .select(.{.id}).execute(env.repo); - - for (track_artist_name_buffer, track_artist_id_buffer) |scrobble_track_artist, track_artist_hash| { - var artist_id = try jetzig.database.Query(.Artist) - .find(track_artist_hash) - .select(.{.id}).execute(env.repo); - - if (artist_id == null) - artist_id = try jetzig.database.Query(.Artist) - .insert(.{ .id = track_artist_hash, .name = scrobble_track_artist, .disambiguation = null }) - .returning(.{.id}).execute(env.repo); - - if (albumsong_id == null) { - var track_id = try jetzig.database.Query(.Song) - .find(track_hash) - .select(.{.id}).execute(env.repo); - - if (track_id == null) - track_id = try jetzig.database.Query(.Song) - .insert(.{ .id = track_hash, .name = scrobble.track, .length = null, .hidden = false }) - .returning(.{.id}).execute(env.repo); - - if (album_id == null) - album_id = try jetzig.database.Query(.Album) - .insert(.{ .id = album_hash, .name = scrobble.album, .length = null }) - .returning(.{.id}).execute(env.repo); - - albumsong_id = try jetzig.database.Query(.Albumsong) - .insert(.{ .song_id = track_id.?.id, .album_id = album_id.?.id }) - .returning(.{.id}).execute(env.repo); - - try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); - } else { - const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) - .findBy(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }) - .select(.{.id}).execute(env.repo); - - if (ins_albumsongartist == null) - try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); - } - } - - for (album_artist_name_buffer, album_artist_id_buffer) |scrobble_album_artist, album_artist_hash| { - const artistalbum_id = try jetzig.database.Query(.Artistalbum) - .findBy(.{ .album_id = album_id.?.id, .artist_id = album_artist_hash }) - .select(.{.id}).execute(env.repo); - - if (artistalbum_id == null) { - var artist_id = try jetzig.database.Query(.Artist) - .find(album_artist_hash) - .select(.{.id}).execute(env.repo); - if (artist_id == null) - artist_id = try jetzig.database.Query(.Artist) - .insert(.{ .id = album_artist_hash, .name = scrobble_album_artist, .disambiguation = null }) - .returning(.{.id}).execute(env.repo); - try jetzig.database.Query(.Artistalbum) - .insert(.{ .album_id = album_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); - } - } - - try jetzig.database.Query(.Scrobble) - .insert(.{ .albumsong = albumsong_id.?.id, .datetime = scrobble.date }).execute(env.repo); - } - } -} diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 740fa8b..27b2e78 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -13,6 +13,8 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { } pub fn post(request: *jetzig.Request) !jetzig.View { + var root = try request.data(.object); + var view_params = try root.put("scrobbles", .array); const UploadParams = struct { source: enum { LFMW, LFMS, Spotify }, earliest_date: ?[]const u8, @@ -25,21 +27,21 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const latest_ts = (try zeit.instant(.{ .source = if (params.latest_date) |ld| .{ .iso8601 = ld } else .now })).timestamp; const earliest_ts = (try zeit.instant(.{ .source = if (params.earliest_date) |ed| .{ .iso8601 = ed } else .{ .unix_timestamp = 0 } })).timestamp; - var skipped_tracks: u64 = 0; - var limited_tracks: u64 = 0; - - const imported_scrobbles: []Data.UnifiedScrobble = switch (params.source) { - .LFMS, .Spotify => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), - .LFMW => blk: { + switch (params.source) { + .LFMS, .Spotify => { + const ctx = try Utils.scrobbleIngest(request, if (try request.file("upload")) |file| file.content else unreachable, .{}, null); + for (ctx.rows) |row| try view_params.append(row); + }, + .LFMW => { const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); - var scrobble_buffer = std.ArrayList(Data.UnifiedScrobble).init(request.allocator); const username = if (params.username) |un| un else "VAOTM"; var page: usize = 1; - //var max_pages: ?usize = null; + + var ctx: ?Utils.IngestContext = null; while (true) : (page += 1) { if (page > 91) break; @@ -52,128 +54,11 @@ pub fn post(request: *jetzig.Request) !jetzig.View { continue; } const response_string = try lastfm_response_buffer.toOwnedSlice(); - const parsed_lastfm_response = try Utils.scrobbleIngest(request.allocator, response_string); - //const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); - //if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); - try scrobble_buffer.appendSlice(parsed_lastfm_response); + ctx = try Utils.scrobbleIngest(request, response_string, .{}, ctx); } - break :blk try scrobble_buffer.toOwnedSlice(); + for (ctx.?.rows) |row| try view_params.append(row); }, - }; - - var root = try request.data(.object); - var view_params = try root.put("scrobbles", .array); - - var job = try request.job("process_scrobbles2"); - const rule_list = try Utils.loadRules(request.allocator); - - var artists = try job.params.put("artists", .object); - var albums = try job.params.put("albums", .object); - var tracks = try job.params.put("tracks", .object); - var artistalbums = try job.params.put("artistalbums", .object); - var albumsongs = try job.params.put("albumsongs", .object); - var albumsongsartists = try job.params.put("albumsongsartists", .object); - - var hash_buffer = [_]u8{undefined} ** 20; // A minimum i64 needs 19 digits + 1 negative sign - - appends: for (imported_scrobbles) |scrobble| { - if (scrobble.date > latest_ts or scrobble.date < earliest_ts) { - limited_tracks += 1; - continue :appends; - } - if (scrobble.playtime != null and scrobble.playtime.? < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.track_artist == null or scrobble.album_artist == null or scrobble.track == null) { - skipped_tracks += 1; - continue :appends; - } - - const filtered_scrobble = Data.Scrobble{ - .album = scrobble.album.?, - .artists_album = &.{scrobble.album_artist.?}, - .artists_track = &.{scrobble.track_artist.?}, - .date = scrobble.date, - .track = scrobble.track.?, - }; - - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; - - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - try view_params.append(row); - - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); - - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } - - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - // The track hash does not currently include the track artists. Probably not necessary for it to, - // but easily can if need be - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(@divFloor(complete_scrobble.date, std.time.ns_per_us)); // MICROSECONDS - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(@divFloor(complete_scrobble.date, std.time.ns_per_us)); // MICROSECONDS - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(albumsongsartist_hash))}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } } - - std.log.debug("Skipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); - try job.schedule(); - return request.render(.created); } - -// Cantor Pairing Function -// https://en.wikipedia.org/wiki/Pairing_function -fn pair(a: u64, b: u64) u64 { - return @mod(@divFloor((a +% b) *% (a +% b +% 1), 2) +% b, std.math.maxInt(u64)); -} diff --git a/src/apply_rule.zig b/src/apply_rule.zig index fc4c0bc..0d357a6 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -27,14 +27,25 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble, switch (rule.cond_req) { .any => switch (cond.match_on) { inline .album, .track => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), - inline .artists_album, .artists_track => |on| { - for (@field(scrobble, @tagName(on))) |artist| match_found = match_found or match_fn(artist, cond.match_txt); + .album_artists => { + for (scrobble.album_artists) |artist| match_found = match_found or match_fn(artist, cond.match_txt); + }, + .track_artists => { + if (scrobble.track_artists) |ta| { + for (ta) |a| match_found = match_found or match_fn(a, cond.match_txt); + } else match_found = false; }, }, .all => switch (cond.match_on) { inline .album, .track => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), - inline .artists_album, .artists_track => |on| { - for (@field(scrobble, @tagName(on))) |artist| match_found = match_found and match_fn(artist, cond.match_txt); + .album_artists => { + for (scrobble.album_artists) |artist| match_found = match_found and match_fn(artist, cond.match_txt); + }, + + .track_artists => { + if (scrobble.track_artists) |ta| { + for (ta) |a| match_found = match_found and match_fn(a, cond.match_txt); + } else match_found = false; }, }, } @@ -46,17 +57,21 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble, var al = std.ArrayList([]const u8).init(allocator); switch (act.action_on) { .album, .track => unreachable, - inline else => |on| { - try al.appendSlice(@field(output_scrobble, @tagName(on))); + .album_artists => { + try al.appendSlice(scrobble.album_artists); try al.append(act.action_txt); - const list = try al.toOwnedSlice(); - @field(output_scrobble, @tagName(on)) = list; + output_scrobble.album_artists = try al.toOwnedSlice(); + }, + .track_artists => { + if (scrobble.track_artists) |ta| try al.appendSlice(ta); + try al.append(act.action_txt); + output_scrobble.track_artists = try al.toOwnedSlice(); }, } }, .replace => switch (act.action_on) { inline .album, .track => |on| @field(output_scrobble, @tagName(on)) = act.action_txt, - inline .artists_album, .artists_track => |on| { + inline .album_artists, .track_artists => |on| { const artist = try allocator.alloc([]const u8, 1); artist[0] = act.action_txt; @field(output_scrobble, @tagName(on)) = artist; diff --git a/src/date_fmt.zig b/src/date_fmt.zig index b8838ad..4c40a25 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -1,10 +1,12 @@ const std = @import("std"); const zeit = @import("zeit"); const Data = @import("types.zig"); +const jetzig = @import("jetzig"); +const applyRule = @import("apply_rule.zig").applyScrobbleRule; pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, std.time.ns_per_s) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, std.time.ns_per_us) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } @@ -23,7 +25,7 @@ pub fn dateCompare(self: *[]const u8, earliest: []const u8, latest: []const u8) pub fn scrobbleToRow(allocator: std.mem.Allocator, scrobble: Data.Scrobble) !Data.TableRow { var artistlist = std.ArrayList(Data.HyperlinkData).init(allocator); - for (scrobble.artists_track) |a| { + for (scrobble.track_artists orelse scrobble.album_artists) |a| { try artistlist.append(Data.HyperlinkData{ .name = a, .id = 0 }); } return Data.TableRow{ @@ -79,17 +81,46 @@ pub fn loadRules(allocator: std.mem.Allocator) !?[]Data.Rule { return std.json.parseFromSliceLeaky([]Data.Rule, allocator, rule_file_content, .{}) catch null; } -pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.UnifiedScrobble { +/// Configuration for scrobble ingestion +const IngestConfig = struct { + /// The earliest date (a unix timestamp in nanoseconds) to accept a scrobble + earliest: ?i128 = null, + /// The latest date (a unix timestamp in nanoseconds) to accept a scrobble + latest: ?i128 = null, + /// The minimum number of milliseconds needed to accept a scrobble + /// Only affects Spotify scrobbles + minimum_playtime: i128 = 30_000, + /// The amount of metadata required to accept a scrobble. A track name is always required + /// - need_artist: Only an artist name is required to accept a scrobble + /// - need_album: Only an album name is required to accept a scrobble + /// - need_both: Both an artist name and an album name are required to accept a scrobble + /// - need_neither: No extra metadata is required to accept a scrobble + null_tolerance: enum { need_artist, need_album, need_both, need_neither } = .need_both, +}; + +pub const IngestContext = struct { + rows: []Data.TableRow, + map: ?std.StringHashMap(std.StringHashMap(std.StringHashMap(?std.BufSet))), +}; + +/// +pub fn scrobbleIngest(request: *jetzig.Request, input: []const u8, config: IngestConfig, context: ?IngestContext) !IngestContext { + const allocator = request.allocator; + var out = std.ArrayList(Data.TableRow).init(allocator); + var artists = if (context) |ctx| blk: { + try out.appendSlice(ctx.rows); + break :blk ctx.map.?; + } else std.StringHashMap(std.StringHashMap(std.StringHashMap(?std.BufSet))).init(allocator); var scanner = std.json.Scanner.initCompleteInput(allocator, input); defer scanner.deinit(); - var out = std.ArrayList(Data.UnifiedScrobble).init(allocator); + const rule_list = try loadRules(allocator); array: switch (try scanner.peekNextTokenType()) { .array_begin => { // Go into array _ = try scanner.next(); - while (try scanner.peekNextTokenType() != .array_end) { + scrobble_array: while (try scanner.peekNextTokenType() != .array_end) { var r: Data.UnifiedScrobble = undefined; // Go into object _ = try scanner.next(); @@ -97,18 +128,22 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed); const field_name = std.meta.stringToEnum(ScrobbleFields, switch (key_token) { inline .string, .allocated_string => |slice| slice, - else => return error.UnexpectedToken, + else => { + std.log.debug("{any}", .{key_token}); + return error.UnexpectedToken; + }, }) orelse .irrelevant; switch (field_name) { - .@"@attr" => { - freeAllocated(allocator, key_token); + .@"@attr" => |d| { r = undefined; - try scanner.skipUntilStackHeight(3); + try skipScrobble(allocator, &scanner, key_token, d); }, .ts, .date => |d| { freeAllocated(allocator, key_token); - const date = switch (d) { + const date: i64 = switch (d) { .date => blk: { + // We can filter by date via the API, so we will always have results in the + // specified timeframe through LFMW if (try scanner.peekNextTokenType() == .object_begin) { // For now, try to just skip over the object_begin and assume the next field is uts _ = try scanner.next(); @@ -121,7 +156,7 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U const lfw_date = try std.fmt.parseInt(i64, switch (lfw_date_token) { inline .number, .allocated_number, .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, - }, 10) * std.time.ns_per_s; + }, 10) * std.time.us_per_s; freeAllocated(allocator, lfw_date_token); break :blk lfw_date; } else { @@ -129,7 +164,11 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U const lfs_date = try std.fmt.parseInt(i64, switch (lfs_date_token) { inline .number, .allocated_number, .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, - }, 10) * std.time.ns_per_ms; + }, 10) * std.time.us_per_ms; + if ((config.earliest != null and lfs_date < config.earliest.?) or (config.latest != null and lfs_date > config.latest.?)) { + r = undefined; + try skipScrobble(allocator, &scanner, lfs_date_token, d); + } freeAllocated(allocator, lfs_date_token); break :blk lfs_date; } @@ -137,12 +176,16 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U .ts => blk: { // This might need to be an alloc_always, but I'm gonna try if_needed first const spotify_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); - const spotify_date = try zeit.instant(.{ .source = .{ .iso8601 = switch (spotify_date_token) { + const spotify_date = (try zeit.instant(.{ .source = .{ .iso8601 = switch (spotify_date_token) { inline .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, - } } }); + } } })).timestamp; + if ((config.earliest != null and spotify_date < config.earliest.?) or (config.latest != null and spotify_date > config.latest.?)) { + r = undefined; + try skipScrobble(allocator, &scanner, spotify_date_token, d); + } freeAllocated(allocator, spotify_date_token); - break :blk spotify_date.unixTimestamp() * std.time.ns_per_s; + break :blk @as(i64, @truncate(@divFloor(spotify_date, std.time.us_per_ms))); }, else => unreachable, }; @@ -157,7 +200,7 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U }, 10); freeAllocated(allocator, spotify_ms_played); }, - .master_metadata_track_name, .track, .name => { + .master_metadata_track_name, .track, .name => |d| { freeAllocated(allocator, key_token); const track = try scanner.nextAlloc(allocator, .alloc_always); @field(r, "track") = switch (track) { @@ -165,6 +208,11 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U .null => null, else => return error.UnexpectedToken, }; + if (r.track == null) { + r = undefined; + try skipScrobble(allocator, &scanner, track, d); + _ = try scanner.next(); + } }, .master_metadata_album_artist_name, .artist => { freeAllocated(allocator, key_token); @@ -189,7 +237,6 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U else => return error.UnexpectedToken, }; }; - @field(r, "track_artist") = artist; @field(r, "album_artist") = artist; }, .master_metadata_album_album_name, .album => { @@ -234,7 +281,118 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U } // Exit object _ = try scanner.next(); - try out.append(r); + + // Final checks + if (r.playtime != null and r.playtime.? < config.minimum_playtime and (r.reason_end == null or !std.mem.eql(u8, r.reason_end.?, "trackdone"))) continue :scrobble_array; + switch (config.null_tolerance) { + .need_neither => {}, + .need_both => if (r.album == null and r.album_artist == null) continue :scrobble_array, + .need_album => if (r.album == null) continue :scrobble_array, + .need_artist => if (r.album_artist == null) continue :scrobble_array, + } + var scr = Data.Scrobble{ + .track = r.track.?, + .album = r.album orelse "Unknown Album", + .track_artists = null, + .album_artists = &.{r.album_artist orelse "Unknown Artist"}, + .date = r.date, + }; + + if (rule_list) |rules| scr = try applyRule(allocator, scr, rules); + + // Try not to have an aneurysm (impossible challenge 2025) + const artist: []const u8, const single_artist_flag: bool = if (scr.album_artists.len == 1) .{ scr.album_artists[0], true } else blk: { + var combined = try std.ArrayListUnmanaged(u8).initCapacity(allocator, 0); + for (scr.album_artists) |aa| try combined.appendSlice(allocator, aa); + break :blk .{ try combined.toOwnedSlice(allocator), false }; + }; + + const premade_hashes = try scr.asHash(allocator); + + // I'm doing all the hashing in the jobs, meaning we hash more than we need to + // If I get bored maybe I'll work on storing them instead + const hm_artist_info = try artists.getOrPut(artist); + if (!hm_artist_info.found_existing) { + hm_artist_info.value_ptr.* = std.StringHashMap(std.StringHashMap(?std.BufSet)).init(allocator); + if (single_artist_flag) { + var add_artist = try request.job("add_artist"); + try add_artist.params.put("artist", artist); + try add_artist.schedule(); + } else { + for (scr.album_artists) |a| { + const hm_ind_artist_info = try artists.getOrPut(a); + if (!hm_ind_artist_info.found_existing) { + hm_ind_artist_info.value_ptr.* = std.StringHashMap(std.StringHashMap(?std.BufSet)).init(allocator); + var add_artist = try request.job("add_artist"); + try add_artist.params.put("artist", a); + try add_artist.schedule(); + } + } + } + } + const hm_album_info = try hm_artist_info.value_ptr.*.getOrPut(scr.album); + if (!hm_album_info.found_existing) { + const album_query = try jetzig.database.Query(.Album) + .find(@as(i64, @bitCast(premade_hashes[0]))).execute(request.repo); + + if (album_query == null) { + try jetzig.database.Query(.Album) + .insert(.{ .id = @as(i64, @bitCast(premade_hashes[0])), .name = scr.album, .length = null }) + .execute(request.repo); + } + hm_album_info.value_ptr.* = std.StringHashMap(?std.BufSet).init(allocator); + var add_album = try request.job("add_album"); + try add_album.params.put("album_hash", premade_hashes[0]); + try add_album.params.put("artists", scr.album_artists); + //if (!single_artist_flag) add_album.put("combined", artist); + try add_album.schedule(); + } + + const hm_song_info = try hm_album_info.value_ptr.*.getOrPut(scr.track); + if (!hm_song_info.found_existing) { + const track_query = try jetzig.database.Query(.Song) + .find(@as(i64, @bitCast(premade_hashes[1]))).execute(request.repo); + + if (track_query == null) { + try jetzig.database.Query(.Song) + .insert(.{ .id = @as(i64, @bitCast(premade_hashes[1])), .name = scr.track, .length = null, .hidden = false }) + .execute(request.repo); + } + const as_query = try jetzig.database.Query(.Albumsong) + .find(@as(i64, @bitCast(premade_hashes[2]))).execute(request.repo); + + if (as_query == null) { + try jetzig.database.Query(.Albumsong) + .insert(.{ .id = @as(i64, @bitCast(premade_hashes[2])), .song_id = @as(i64, @bitCast(premade_hashes[1])), .album_id = @as(i64, @bitCast(premade_hashes[0])) }) + .execute(request.repo); + } + hm_song_info.value_ptr.* = null; + var add_song = try request.job("add_song"); + try add_song.params.put("as_hash", premade_hashes[2]); + //add_song.put("album", scr.album); + if (scr.track_artists) |track_artists| { + try add_song.params.put("track_artists", track_artists); + if (hm_song_info.value_ptr.* == null) hm_song_info.value_ptr.* = std.BufSet.init(allocator); + for (track_artists) |ta| { + try hm_song_info.value_ptr.*.?.insert(ta); + const hm_ta_info = try artists.getOrPut(ta); + if (!hm_ta_info.found_existing) { + hm_ta_info.value_ptr.* = std.StringHashMap(std.StringHashMap(?std.BufSet)).init(allocator); + var add_artist = try request.job("add_artist"); + try add_artist.params.put("artist", ta); + try add_artist.schedule(); + } + } + } + try add_song.params.put("album_artists", scr.album_artists); + //if (!single_artist_flag) add_song.put("combined", artist); + try add_song.schedule(); + } + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = @as(i64, @bitCast(premade_hashes[2])), .datetime = scr.date }) + .execute(request.repo); + + const b = try scrobbleToRow(allocator, scr); + try out.append(b); } }, // LastFM(stats) @@ -282,7 +440,7 @@ pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.U else => return error.UnexpectedToken, } const scrobbles = try out.toOwnedSlice(); - return scrobbles; + return IngestContext{ .map = artists, .rows = scrobbles }; } fn freeAllocated(allocator: std.mem.Allocator, token: std.json.Token) void { @@ -293,3 +451,24 @@ fn freeAllocated(allocator: std.mem.Allocator, token: std.json.Token) void { else => {}, } } + +// Cantor Pairing Function +// https://en.wikipedia.org/wiki/Pairing_function + +fn skipScrobble(allocator: std.mem.Allocator, scanner: *std.json.Scanner, token: std.json.Token, field: ScrobbleFields) !void { + freeAllocated(allocator, token); + try scanner.skipUntilStackHeight(switch (field) { + // Spotify specific fields + .ts, .master_metadata_album_album_name, .master_metadata_album_artist_name, .master_metadata_track_name, .ms_played, .reason_end => 1, + // LastFM Stats specific field + .track => 2, + // LastFM Web specific fields + .name, .@"@attr" => 3, + // Fields shared by LastFM Stats and LastFM Web: album, artist, date (although date is never invalid for LastFM Web) + else => switch (scanner.stackHeight()) { + 5 => 3, // Five levels deep => LastFM Web (all of those fields are fortunately objects / same stack height) + 3 => 2, // Three levels deep => LastFM stats + else => unreachable, + }, + }); +} diff --git a/src/types.zig b/src/types.zig index df067e8..dd5fd20 100644 --- a/src/types.zig +++ b/src/types.zig @@ -3,7 +3,7 @@ const std = @import("std"); pub const UnifiedScrobble = struct { track: ?[]const u8, // These can be null per Spotify - track_artist: ?[]const u8, + //track_artist: ?[]const u8, // As far as I'm aware, there are no services that provide separate track/album artist lists album: ?[]const u8, album_artist: ?[]const u8, date: i64, @@ -12,114 +12,53 @@ pub const UnifiedScrobble = struct { reason_end: ?[]const u8 = null, }; -pub const ImportedScrobbles = union(ScrobbleSources) { - LastFMStats: []IgnorantScrobble, - LastFMWeb: []LastFMWebScrobble, - Spotify: []SpotifyScrobble, -}; - const ScrobbleSources = enum { LastFMStats, LastFMWeb, Spotify, }; -pub const IgnorantScrobble = struct { - track: []const u8, - artist: []const u8, - album: []const u8 = "Not Provided", - //albumId: []const u8, - date: i64, -}; +fn hashAndSign(a: []const u8) !i64 { + return @as(i64, @bitCast(std.hash.Fnv1a_64.hash(a))); +} + +fn pair(a: u64, b: u64) u64 { + return @mod(@divFloor((a +% b) *% (a +% b +% 1), 2) +% b, std.math.maxInt(u64)); +} pub const Scrobble = struct { track: []const u8, - artists_track: []const []const u8, - album: []const u8 = "", - artists_album: []const []const u8, + track_artists: ?[]const []const u8, + album: []const u8 = "Unknown Album", + album_artists: []const []const u8, date: i64, -}; -// From lastfmstats.com -pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble }; + pub fn asHash(self: *Scrobble, allocator: std.mem.Allocator) ![3]u64 { + var string_buf = try std.ArrayListUnmanaged(u8).initCapacity(allocator, 0); -// I derived whether or not these values were optional from searching -// the respective fields for null in Vim, so there may be some fields -// that can be optional that I haven't run into yet -pub const SpotifyScrobble = struct { - ts: []const u8, - //username: []const u8, - //platform: []const u8, - ms_played: u64, - //conn_country: []const u8, - //ip_addr_decrypted: ?[]const u8, - //user_agent_decrypted: ?[]const u8, - master_metadata_track_name: ?[]const u8, - master_metadata_album_artist_name: ?[]const u8, - master_metadata_album_album_name: ?[]const u8, - //spotify_track_uri: ?[]const u8, - //episode_name: ?[]const u8, - //episode_show_name: ?[]const u8, - //spotify_episode_uri: ?[]const u8, - reason_start: []const u8, - reason_end: ?[]const u8, - //shuffle: bool, - skipped: ?bool, - //offline: bool, - offline_timestamp: u64, - //incognito_mode: ?bool, -}; + for (self.album_artists) |artist| try string_buf.appendSlice(allocator, artist); -pub const LastFMWeb = struct { - recenttracks: struct { - track: []LastFMWebScrobble, - @"@attr": LastFMWebQueryInfo, - }, -}; + try string_buf.appendSlice(allocator, self.album); + const a = std.hash.Fnv1a_64.hash(string_buf.items); -pub const LastFMWebHyperlinkData = struct { - mbid: []const u8, - @"#text": []const u8, -}; + try string_buf.appendSlice(allocator, self.track); + const s = std.hash.Fnv1a_64.hash(try string_buf.toOwnedSlice(allocator)); -pub const LastFMWebScrobble = struct { - artist: LastFMWebHyperlinkData, - album: ?LastFMWebHyperlinkData = null, - name: []const u8, - mbid: ?[]const u8 = null, - image: []struct { - size: []const u8, - @"#text": []const u8, - }, - date: ?struct { - uts: []const u8, - @"#text": []const u8, - } = null, - @"@attr": ?struct { - nowplaying: []const u8, - } = null, - url: []const u8, -}; - -pub const LastFMWebQueryInfo = struct { - perPage: []const u8, - totalPages: []const u8, - page: []const u8, - user: []const u8, - total: []const u8, + return .{ a, s, pair(a, s) }; + } }; pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, conditionals: []struct { - match_on: enum { artists_album, artists_track, album, track }, + match_on: enum { album_artists, track_artists, album, track }, match_cond: enum { is, contains }, match_txt: []const u8, }, actions: []struct { action: enum { replace, add }, - action_on: enum { artists_album, album, artists_track, track }, + action_on: enum { album_artists, album, track_artists, track }, action_txt: []const u8, }, };