From 0537ef7db211ec3c7b693d65e1cd549cb3f5e5ee Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Apr 2025 10:44:28 -0400 Subject: [PATCH 01/71] she QUERY on my DATA so she's BASEd --- config/database.zig | 2 +- .../2025-02-17_22-38-45_create_ratings.zig | 21 ------------------- ...025-02-19_18-03-51_create_albumartists.zig | 20 ------------------ ...025-02-19_18-04-22_create_songsartists.zig | 20 ------------------ .../2025-02-19_18-46-48_create_albumsongs.zig | 20 ------------------ ...-02-21_14-24-31_create_scrobbleartists.zig | 20 ------------------ ...g => 2025-04-07_14-31-14_create_songs.zig} | 0 ... => 2025-04-07_14-31-45_create_albums.zig} | 0 ...2025-04-07_14-34-39_create_albumsongs.zig} | 8 +++---- ... 2025-04-07_14-35-53_create_scrobbles.zig} | 5 ++--- ...=> 2025-04-07_14-38-02_create_artists.zig} | 2 +- ...-07_14-39-09_create_albumsongsartists.zig} | 7 ++++--- ...25-04-07_14-40-17_create_artistalbums.zig} | 7 ++++--- 13 files changed, 16 insertions(+), 116 deletions(-) delete mode 100644 src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig delete mode 100644 src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig delete mode 100644 src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig delete mode 100644 src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig delete mode 100644 src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig rename src/app/database/migrations/{2025-02-17_22-38-47_create_songs.zig => 2025-04-07_14-31-14_create_songs.zig} (100%) rename src/app/database/migrations/{2025-02-17_22-38-40_create_albums.zig => 2025-04-07_14-31-45_create_albums.zig} (100%) rename src/app/database/migrations/{2025-02-17_22-38-41_create_aliases.zig => 2025-04-07_14-34-39_create_albumsongs.zig} (57%) rename src/app/database/migrations/{2025-02-17_22-38-46_create_scrobbles.zig => 2025-04-07_14-35-53_create_scrobbles.zig} (72%) rename src/app/database/migrations/{2025-02-17_22-38-42_create_artists.zig => 2025-04-07_14-38-02_create_artists.zig} (85%) rename src/app/database/migrations/{2025-02-17_22-38-44_create_mastersongs.zig => 2025-04-07_14-39-09_create_albumsongsartists.zig} (54%) rename src/app/database/migrations/{2025-02-17_22-38-43_create_masteralbums.zig => 2025-04-07_14-40-17_create_artistalbums.zig} (56%) diff --git a/config/database.zig b/config/database.zig index 361129d..4042ff0 100644 --- a/config/database.zig +++ b/config/database.zig @@ -15,7 +15,7 @@ pub const database = .{ .port = 5432, .username = "postgres", .password = "postgres", - .database = "zuletzt_dev", + .database = "zuletzt_rsql", .pool_size = 16, }, diff --git a/src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig b/src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig deleted file mode 100644 index d7ac939..0000000 --- a/src/app/database/migrations/2025-02-17_22-38-45_create_ratings.zig +++ /dev/null @@ -1,21 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "ratings", - &.{ - t.primaryKey("id", .{}), - t.column("reference_id", .integer, .{}), - t.column("score", .float, .{}), - t.column("date", .datetime, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("ratings", .{}); -} diff --git a/src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig b/src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig deleted file mode 100644 index b0e4f54..0000000 --- a/src/app/database/migrations/2025-02-19_18-03-51_create_albumartists.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Albumartists", - &.{ - t.primaryKey("id", .{}), - t.column("album_id", .integer, .{}), - t.column("artist_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Albumartists", .{}); -} diff --git a/src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig b/src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig deleted file mode 100644 index a509d7a..0000000 --- a/src/app/database/migrations/2025-02-19_18-04-22_create_songsartists.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Songartists", - &.{ - t.primaryKey("id", .{}), - t.column("song_id", .integer, .{}), - t.column("artist_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Songartists", .{}); -} diff --git a/src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig b/src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig deleted file mode 100644 index 1865c2e..0000000 --- a/src/app/database/migrations/2025-02-19_18-46-48_create_albumsongs.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Albumsongs", - &.{ - t.primaryKey("id", .{}), - t.column("album_id", .integer, .{}), - t.column("song_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Albumsongs", .{}); -} diff --git a/src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig b/src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig deleted file mode 100644 index 2125a87..0000000 --- a/src/app/database/migrations/2025-02-21_14-24-31_create_scrobbleartists.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); -const jetquery = @import("jetquery"); -const t = jetquery.schema.table; - -pub fn up(repo: anytype) !void { - try repo.createTable( - "Scrobbleartists", - &.{ - t.primaryKey("id", .{}), - t.column("scrobble_id", .integer, .{}), - t.column("artist_id", .integer, .{}), - t.timestamps(.{}), - }, - .{}, - ); -} - -pub fn down(repo: anytype) !void { - try repo.dropTable("Scrobbleartists", .{}); -} diff --git a/src/app/database/migrations/2025-02-17_22-38-47_create_songs.zig b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig similarity index 100% rename from src/app/database/migrations/2025-02-17_22-38-47_create_songs.zig rename to src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig diff --git a/src/app/database/migrations/2025-02-17_22-38-40_create_albums.zig b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig similarity index 100% rename from src/app/database/migrations/2025-02-17_22-38-40_create_albums.zig rename to src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig diff --git a/src/app/database/migrations/2025-02-17_22-38-41_create_aliases.zig b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig similarity index 57% rename from src/app/database/migrations/2025-02-17_22-38-41_create_aliases.zig rename to src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig index 11cbb70..4381a5b 100644 --- a/src/app/database/migrations/2025-02-17_22-38-41_create_aliases.zig +++ b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig @@ -4,11 +4,11 @@ const t = jetquery.schema.table; pub fn up(repo: anytype) !void { try repo.createTable( - "aliases", + "albumsongs", &.{ t.primaryKey("id", .{}), - t.column("reference_id", .integer, .{}), - t.column("alias", .string, .{}), + t.column("song_id", .integer, .{ .reference = .{ "songs", "id" } }), + t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), t.timestamps(.{}), }, .{}, @@ -16,5 +16,5 @@ pub fn up(repo: anytype) !void { } pub fn down(repo: anytype) !void { - try repo.dropTable("aliases", .{}); + try repo.dropTable("albumsongs", .{}); } diff --git a/src/app/database/migrations/2025-02-17_22-38-46_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig similarity index 72% rename from src/app/database/migrations/2025-02-17_22-38-46_create_scrobbles.zig rename to src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index 9764d99..088a43c 100644 --- a/src/app/database/migrations/2025-02-17_22-38-46_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -7,9 +7,8 @@ pub fn up(repo: anytype) !void { "scrobbles", &.{ t.primaryKey("id", .{}), - t.column("song_id", .integer, .{}), - t.column("album_id", .integer, .{}), - t.column("date", .datetime, .{}), + t.column("albumsong", .integer, .{ .reference = .{ "albumsongs", "id" } }), + t.column("datetime", .datetime, .{}), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-02-17_22-38-42_create_artists.zig b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig similarity index 85% rename from src/app/database/migrations/2025-02-17_22-38-42_create_artists.zig rename to src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig index 2c92de4..ed0d3d4 100644 --- a/src/app/database/migrations/2025-02-17_22-38-42_create_artists.zig +++ b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig @@ -8,7 +8,7 @@ pub fn up(repo: anytype) !void { &.{ t.primaryKey("id", .{}), t.column("name", .string, .{}), - t.column("descriptive_string", .string, .{}), + t.column("disambiguation", .string, .{ .optional = true }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-02-17_22-38-44_create_mastersongs.zig b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig similarity index 54% rename from src/app/database/migrations/2025-02-17_22-38-44_create_mastersongs.zig rename to src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig index 050a467..55c7d84 100644 --- a/src/app/database/migrations/2025-02-17_22-38-44_create_mastersongs.zig +++ b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig @@ -4,10 +4,11 @@ const t = jetquery.schema.table; pub fn up(repo: anytype) !void { try repo.createTable( - "mastersongs", + "albumsongsartists", &.{ t.primaryKey("id", .{}), - t.column("name", .string, .{}), + t.column("albumsong_id", .integer, .{ .reference = .{ "albumsongs", "id" } }), + t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, @@ -15,5 +16,5 @@ pub fn up(repo: anytype) !void { } pub fn down(repo: anytype) !void { - try repo.dropTable("mastersongs", .{}); + try repo.dropTable("albumsongsartists", .{}); } diff --git a/src/app/database/migrations/2025-02-17_22-38-43_create_masteralbums.zig b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig similarity index 56% rename from src/app/database/migrations/2025-02-17_22-38-43_create_masteralbums.zig rename to src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig index 69e82de..76bac1e 100644 --- a/src/app/database/migrations/2025-02-17_22-38-43_create_masteralbums.zig +++ b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig @@ -4,10 +4,11 @@ const t = jetquery.schema.table; pub fn up(repo: anytype) !void { try repo.createTable( - "masteralbums", + "artistalbums", &.{ t.primaryKey("id", .{}), - t.column("name", .string, .{}), + t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), + t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, @@ -15,5 +16,5 @@ pub fn up(repo: anytype) !void { } pub fn down(repo: anytype) !void { - try repo.dropTable("masteralbums", .{}); + try repo.dropTable("artistalbums", .{}); } From 64038079d8d49f2354c3bac3603a1f3c3dfc2b0b Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Apr 2025 15:44:52 -0400 Subject: [PATCH 02/71] Update process_scrobbles.zig to fit new db --- src/app/jobs/process_scrobbles.zig | 65 +++++++++++++----------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 1314465..6481d2e 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -54,55 +54,48 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig // artists have an album of the same name, then // the IDs also depend on the hash of the artist // name. As far as I can tell, this is only an - // issue for Weezer. + // issue for Weezer and Peter Gabriel, but their + // albums go by unique names anyways. // Artist: Artist hash. If two artists have the same name, - // then a descriptive string can be provided to + // then a disambiguating string can be provided to // differentiate after the fact, or in a rule. //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); //const song_id = (song_hash ^ artist_hash ^ album_hash); - // Inserts - const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }); - const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .descriptive_string = "" }); - const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }); + var albumsong_id = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); + var ins_album_id = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); - // Checks - const album_check = try jetzig.database.Query(.Album).find(album_id).execute(env.repo); - const artist_check = try jetzig.database.Query(.Artist).find(artist_id).execute(env.repo); - const song_check = try jetzig.database.Query(.Song).find(song_id).execute(env.repo); + var ins_artist_id = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); + if (ins_artist_id == null) ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); - // I think there must be a better way to do this next part - // There are very few situations where artist_check is null - // but song_check/album is not. Also yes, the order of these - // checks is weird, I didn't put a lot of thought into it - var associative_table_flags: [3]bool = [3]bool{ true, true, true }; + if (albumsong_id == null) { + const ins_song_id: []const u8 = + jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo) catch + jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); - if (album_check == null) { - try env.repo.execute(album_insert); - try jetzig.database.Query(.Albumartist).insert(.{ .album_id = album_id, .artist_id = artist_id }).execute(env.repo); - associative_table_flags[0] = false; - try jetzig.database.Query(.Albumsong).insert(.{ .album_id = album_id, .song_id = song_id }).execute(env.repo); - associative_table_flags[1] = false; + if (ins_album_id == null) ins_album_id = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); + + albumsong_id = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song_id, .album_id = ins_album_id }).execute(env.repo); + + try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); + try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); + } else { + const ins_albumsongartist = try jetzig.database.Query(.Albumsongartist).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); + if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); + + const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); + if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); } - if (artist_check == null) { - try env.repo.execute(artist_insert); - if (associative_table_flags[0]) try jetzig.database.Query(.Albumartist).insert(.{ .album_id = album_id, .artist_id = artist_id }).execute(env.repo); - try jetzig.database.Query(.Songartist).insert(.{ .song_id = song_id, .artist_id = artist_id }).execute(env.repo); - associative_table_flags[2] = false; - } + //if (ins_artist_id == null) { + // ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); + // try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); + // try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); + //} - if (song_check == null) { - try env.repo.execute(song_insert); - if (associative_table_flags[1]) try jetzig.database.Query(.Albumsong).insert(.{ .album_id = album_id, .song_id = song_id }).execute(env.repo); - if (associative_table_flags[2]) try jetzig.database.Query(.Songartist).insert(.{ .song_id = song_id, .artist_id = artist_id }).execute(env.repo); - } - - const scr_id = try jetzig.database.Query(.Scrobble).insert(.{ .song_id = song_id, .album_id = album_id, .date = scrobble.date }).returning(.{.id}).execute(env.repo); - defer env.repo.free(scr_id); - try jetzig.database.Query(.Scrobbleartist).insert(.{ .scrobble_id = scr_id.?.id, .artist_id = artist_id }).execute(env.repo); + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong_id = albumsong_id, .date = scrobble.date }).execute(env.repo); } } From 3f69183b6f082ade92a17b58ee779e98d8d218c8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:23:12 -0400 Subject: [PATCH 03/71] Create new Schema from migrations --- src/app/database/Schema.zig | 211 ++++++++++-------------------------- 1 file changed, 59 insertions(+), 152 deletions(-) diff --git a/src/app/database/Schema.zig b/src/app/database/Schema.zig index 231c283..eeec984 100644 --- a/src/app/database/Schema.zig +++ b/src/app/database/Schema.zig @@ -12,27 +12,68 @@ pub const Album = jetquery.Model( }, .{ .relations = .{ - .masteralbum = jetquery.belongsTo(.Masteralbum, .{}), - .scrobbles = jetquery.hasMany(.Scrobble, .{}), - .ratings = jetquery.hasMany(.Rating, .{}), - .aliases = jetquery.hasMany(.Alias, .{}), .albumsongs = jetquery.hasMany(.Albumsong, .{}), - .albumartists = jetquery.hasMany(.Albumartist, .{}), + .artistalbums = jetquery.hasMany(.Artistalbum, .{}), }, }, ); -pub const Alias = jetquery.Model( +pub const Albumsong = jetquery.Model( @This(), - "aliases", + "albumsongs", struct { id: i32, - reference_id: i32, - alias: []const u8, + song_id: i32, + album_id: i32, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, - .{}, + .{ + .relations = .{ + .song = jetquery.belongsTo(.Song, .{}), + .album = jetquery.belongsTo(.Album, .{}), + .scrobbles = jetquery.hasMany(.Scrobble, .{ + .foreign_key = "albumsong", + }), + .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), + }, + }, +); + +pub const Albumsongsartist = jetquery.Model( + @This(), + "albumsongsartists", + struct { + id: i32, + albumsong_id: i32, + artist_id: i32, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .albumsong = jetquery.belongsTo(.Albumsong, .{}), + .artist = jetquery.belongsTo(.Artist, .{}), + }, + }, +); + +pub const Artistalbum = jetquery.Model( + @This(), + "artistalbums", + struct { + id: i32, + album_id: i32, + artist_id: i32, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .album = jetquery.belongsTo(.Album, .{}), + .artist = jetquery.belongsTo(.Artist, .{}), + }, + }, ); pub const Artist = jetquery.Model( @@ -41,70 +82,14 @@ pub const Artist = jetquery.Model( struct { id: i32, name: []const u8, - descriptive_string: []const u8, + disambiguation: ?[]const u8, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .scrobbleartists = jetquery.hasMany(.Scrobbleartist, .{}), - .aliases = jetquery.hasMany(.Alias, .{}), - .artistsongs = jetquery.hasMany(.Songartist, .{}), - .mastersongs = jetquery.hasMany(.Mastersong, .{}), - .artistalbums = jetquery.hasMany(.Albumartist, .{}), - .masteralbums = jetquery.hasMany(.Masteralbum, .{}), - }, - }, -); - -pub const Masteralbum = jetquery.Model( - @This(), - "masteralbums", - struct { - id: i32, - name: []const u8, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .albums = jetquery.hasMany(.Album, .{}), - }, - }, -); - -pub const Mastersong = jetquery.Model( - @This(), - "mastersongs", - struct { - id: i32, - name: []const u8, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .songs = jetquery.hasMany(.Song, .{}), - }, - }, -); - -pub const Rating = jetquery.Model( - @This(), - "ratings", - struct { - id: i32, - reference_id: i32, - score: f32, - date: jetquery.DateTime, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .song = jetquery.belongsTo(.Song, .{}), - .album = jetquery.belongsTo(.Album, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), + .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), + .artistalbums = jetquery.hasMany(.Artistalbum, .{}), }, }, ); @@ -114,17 +99,16 @@ pub const Scrobble = jetquery.Model( "scrobbles", struct { id: i32, - song_id: i32, - album_id: i32, - date: jetquery.DateTime, + albumsong: i32, + datetime: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .song = jetquery.belongsTo(.Song, .{}), - .album = jetquery.belongsTo(.Album, .{}), - .scrobbleartists = jetquery.hasMany(.Scrobbleartist, .{}), + .albumsong = jetquery.belongsTo(.Albumsong, .{ + .foreign_key = "albumsong", + }), }, }, ); @@ -142,84 +126,7 @@ pub const Song = jetquery.Model( }, .{ .relations = .{ - .mastersong = jetquery.belongsTo(.Mastersong, .{}), - .scrobbles = jetquery.hasMany(.Scrobble, .{}), - .ratings = jetquery.hasMany(.Rating, .{}), - .aliases = jetquery.hasMany(.Alias, .{}), - .songartists = jetquery.hasMany(.Songartist, .{}), .albumsongs = jetquery.hasMany(.Albumsong, .{}), }, }, ); - -pub const Albumartist = jetquery.Model( - @This(), - "Albumartists", - struct { - id: i32, - album_id: i32, - artist_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .album = jetquery.belongsTo(.Album, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), - }, - }, -); - -pub const Songartist = jetquery.Model( - @This(), - "Songartists", - struct { - id: i32, - song_id: i32, - artist_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .song = jetquery.belongsTo(.Song, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), - }, - }, -); - -pub const Albumsong = jetquery.Model( - @This(), - "Albumsongs", - struct { - id: i32, - album_id: i32, - song_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .album = jetquery.belongsTo(.Album, .{}), - .song = jetquery.belongsTo(.Song, .{}), - }, - }, -); - -pub const Scrobbleartist = jetquery.Model( - @This(), - "Scrobbleartists", - struct { - id: i32, - scrobble_id: i32, - artist_id: i32, - created_at: jetquery.DateTime, - updated_at: jetquery.DateTime, - }, - .{ - .relations = .{ - .scrobble = jetquery.belongsTo(.Scrobble, .{}), - .artist = jetquery.belongsTo(.Artist, .{}), - }, - }, -); From 09d4453665b654f5be6d428ed798b72113a9442a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:24:16 -0400 Subject: [PATCH 04/71] Fix various issues with process_scrobbles I use the ins_ variables an unnecessary amount I think, I need to take a closer look at it, and give them better names --- src/app/jobs/process_scrobbles.zig | 36 ++++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 6481d2e..70c2976 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -64,29 +64,31 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); //const song_id = (song_hash ^ artist_hash ^ album_hash); - var albumsong_id = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); - var ins_album_id = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); + var albumsong = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); + var ins_album = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); - var ins_artist_id = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); - if (ins_artist_id == null) ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); + var ins_artist = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); + if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); - if (albumsong_id == null) { - const ins_song_id: []const u8 = - jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo) catch - jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); + if (albumsong == null) { + var ins_song = try jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); + if (ins_song == null) ins_song = try jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo); - if (ins_album_id == null) ins_album_id = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); + if (ins_album == null) { + ins_album = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); + // I think there's still technically a bug here when you have a different artist but I'm not sure + try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist.?.id, .album_id = ins_album.?.id }).execute(env.repo); + } - albumsong_id = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song_id, .album_id = ins_album_id }).execute(env.repo); + albumsong = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }).returning(.{.id}).execute(env.repo); - try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); - try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); + try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } else { - const ins_albumsongartist = try jetzig.database.Query(.Albumsongartist).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); - if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); + const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist).findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); + if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); - const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).select(.{.id}).execute(env.repo); - if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }).execute(env.repo); + const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); + if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } //if (ins_artist_id == null) { @@ -95,7 +97,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig // try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); //} - try jetzig.database.Query(.Scrobble).insert(.{ .albumsong_id = albumsong_id, .date = scrobble.date }).execute(env.repo); + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); } } From 27358fe2179e6b8ad04e9ae3423d5b6380f2a6ed Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:24:48 -0400 Subject: [PATCH 05/71] Implement db searches using raw sql --- src/app/views/albums.zig | 111 +++++++++++++++++++++++------------- src/app/views/artists.zig | 91 ++++++++++++++++++++--------- src/app/views/scrobbles.zig | 73 ++++++++++++++++-------- src/app/views/songs.zig | 60 +++++++++++++------ 4 files changed, 226 insertions(+), 109 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 634f414..9f363f5 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -5,60 +5,91 @@ const jetquery = @import("jetzig").jetquery; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var albums_view = try root.put("albums", .array); - const albums = try jetzig.database.Query(.Album) - .select(.{ .id, .name }) - .include(.albumartists, .{ .select = .{.artist_id} }) - .include(.scrobbles, .{ .select = .{.id} }) - .orderBy(.{ .name = .asc }) - .all(request.repo); - //const albums = try request.repo.all(query); + //const albums = try jetzig.database.Query(.Album) + // .select(.{ .id, .name }) + // .include(.albumartists, .{ .select = .{.artist_id} }) + // .include(.scrobbles, .{ .select = .{.id} }) + // .orderBy(.{ .name = .asc }) + // .all(request.repo); + ////const albums = try request.repo.all(query); - for (albums) |album| { + const query = + \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN albums ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + ; + + var albums_jq_result = try request.repo.executeSql(query, .{}); + defer albums_jq_result.deinit(); + + const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try albums_jq_result.postgresql.result.next()) |album_row| { + const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); var album_view = try albums_view.append(.object); - - var artist_infos = try album_view.put("artist_info", .array); - for (album.albumartists) |artist| { - var artist_info = try artist_infos.append(.object); - const artist_data = try jetzig.database.Query(.Artist) - .find(artist.artist_id) - .select(.{ .id, .name }) - .execute(request.repo); - try artist_info.put("name", artist_data.?.name); - try artist_info.put("id", artist_data.?.id); - } - try album_view.put("name", album.name); try album_view.put("url", album.id); - try album_view.put("scrobbles", (album.scrobbles).len); + try album_view.put("scrobbles", album.scrobbles); } return request.render(.ok); } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - const album = try jetzig.database.Query(.Album) - .find(id) - .select(.{ .id, .name }) - .execute(request.repo); var root = try request.data(.object); - try root.put("album", album.?.name); var songs_view = try root.put("songs", .array); - const query = jetzig.database.Query(.Albumsong) - .select(.{.id}) - .include(.song, .{ .select = .{ .name, .id } }) - .join(.inner, .album) - .where(.{ .album = .{ .id = id } }); + const album_name = try jetzig.database.Query(.Album).find(id).select(.{ .id, .name }).execute(request.repo); + _ = try root.put("album", album_name.?.name); - const songs = try request.repo.all(query); - for (songs) |song| { - const scrobbles = try jetzig.database.Query(.Scrobble) - .where(.{ .song_id = song.song.id }) - .count() - .execute(request.repo); + const query = + \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albumsongs.album_id = $1 + \\GROUP BY songs.id + \\ORDER BY scrobbles DESC + ; + + var songs_js_result = try request.repo.executeSql(query, .{id}); + defer songs_js_result.deinit(); + + const Song = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try songs_js_result.postgresql.result.next()) |song_row| { + const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); var song_view = try songs_view.append(.object); - try song_view.put("name", song.song.name); - try song_view.put("url", song.song.id); - try song_view.put("scrobbles", scrobbles); + try song_view.put("name", song.name); + try song_view.put("url", song.id); + try song_view.put("scrobbles", song.scrobbles); } + + //const album = try jetzig.database.Query(.Album) + // .find(id) + // .select(.{ .id, .name }) + // .execute(request.repo); + //var root = try request.data(.object); + //try root.put("album", album.?.name); + //var songs_view = try root.put("songs", .array); + //const query = jetzig.database.Query(.Albumsong) + // .select(.{.id}) + // .include(.song, .{ .select = .{ .name, .id } }) + // .join(.inner, .album) + // .where(.{ .album = .{ .id = id } }); + + //const songs = try request.repo.all(query); + //for (songs) |song| { + // const scrobbles = try jetzig.database.Query(.Scrobble) + // .where(.{ .song_id = song.song.id }) + // .count() + // .execute(request.repo); + // var song_view = try songs_view.append(.object); + // try song_view.put("name", song.song.name); + // try song_view.put("url", song.song.id); + // try song_view.put("scrobbles", scrobbles); + //} return request.render(.ok); } diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 78058a7..b8d6bc3 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -5,46 +5,85 @@ const jetquery = @import("jetzig").jetquery; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var artists_view = try root.put("artists", .array); - const artists = try jetzig.database.Query(.Artist) - .select(.{ .id, .name }) - .include(.scrobbleartists, .{ .select = .{.id} }) - .orderBy(.{ .name = .asc }) - .all(request.repo); - for (artists) |artist| { + + const query = + \\SELECT artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongsartists + \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id + \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY artists.id + \\ORDER BY scrobbles DESC; + ; + + var artists_jq_result = try request.repo.executeSql(query, .{}); + defer artists_jq_result.deinit(); + + const Artist = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try artists_jq_result.postgresql.result.next()) |artist_row| { + const artist = try artist_row.to(Artist, .{ .dupe = true, .allocator = request.allocator }); var artist_view = try artists_view.append(.object); try artist_view.put("name", artist.name); try artist_view.put("url", artist.id); - try artist_view.put("scrobbles", (artist.scrobbleartists).len); + try artist_view.put("scrobbles", artist.scrobbles); } return request.render(.ok); } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - const artist = try jetzig.database.Query(.Artist) - .find(id) - .select(.{ .id, .name }) - .execute(request.repo); var root = try request.data(.object); - try root.put("artist", artist.?.name); var albums_view = try root.put("albums", .array); - const query = jetzig.database.Query(.Albumartist) - .select(.{.id}) - .include(.album, .{ .select = .{ .name, .id } }) - .join(.inner, .artist) - .where(.{ .artist = .{ .id = id } }); + const artist_name = try jetzig.database.Query(.Artist).find(id).select(.{ .id, .name }).execute(request.repo); + _ = try root.put("artist", artist_name.?.name); + const query = + \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE artistalbums.artist_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + ; - const albums = try request.repo.all(query); - for (albums) |album| { - const scrobbles = try jetzig.database.Query(.Scrobble) - .where(.{ .album_id = album.album.id }) - .count() - .execute(request.repo); + var albums_jq_result = try request.repo.executeSql(query, .{id}); + defer albums_jq_result.deinit(); + + const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try albums_jq_result.postgresql.result.next()) |album_row| { + const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); var album_view = try albums_view.append(.object); - try album_view.put("name", album.album.name); - try album_view.put("url", album.album.id); - try album_view.put("scrobbles", scrobbles); + try album_view.put("name", album.name); + try album_view.put("url", album.id); + try album_view.put("scrobbles", album.scrobbles); } + //const artist = try jetzig.database.Query(.Artist) + // .find(id) + // .select(.{ .id, .name }) + // .execute(request.repo); + //var root = try request.data(.object); + //try root.put("artist", artist.?.name); + //var albums_view = try root.put("albums", .array); + //const query = jetzig.database.Query(.Albumartist) + // .select(.{.id}) + // .include(.album, .{ .select = .{ .name, .id } }) + // .join(.inner, .artist) + // .where(.{ .artist = .{ .id = id } }); + + //const albums = try request.repo.all(query); + //for (albums) |album| { + // const scrobbles = try jetzig.database.Query(.Scrobble) + // .where(.{ .album_id = album.album.id }) + // .count() + // .execute(request.repo); + // var album_view = try albums_view.append(.object); + // try album_view.put("name", album.album.name); + // try album_view.put("url", album.album.id); + // try album_view.put("scrobbles", scrobbles); + //} return request.render(.ok); } diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index ebda828..48e5f2b 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -4,33 +4,58 @@ const jetzig = @import("jetzig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var scrobbles_view = try root.put("scrobbles", .array); - const query = jetzig.database.Query(.Scrobble) - .select(.{ .id, .date }) - .include(.song, .{ .select = .{ .id, .name } }) - .include(.album, .{ .select = .{ .id, .name } }) - .include(.scrobbleartists, .{ .select = .{.artist_id} }) - .orderBy(.{ .date = .desc }); - const scrobbles = try request.repo.all(query); - for (scrobbles) |scrobble| { + //const query = jetzig.database.Query(.Scrobble) + // .select(.{ .id, .date }) + // .include(.song, .{ .select = .{ .id, .name } }) + // .include(.album, .{ .select = .{ .id, .name } }) + // .include(.scrobbleartists, .{ .select = .{.artist_id} }) + // .orderBy(.{ .date = .desc }); + //const scrobbles = try request.repo.all(query); + + const query = + \\SELECT songs.name, songs.id, albums.name, albums.id, scrobbles.datetime + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\ORDER BY scrobbles.datetime ASC + ; + + var scrobbles_js_result = try request.repo.executeSql(query, .{}); + defer scrobbles_js_result.deinit(); + + const Scrobble = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, date: i64 }; + + while (try scrobbles_js_result.postgresql.result.next()) |scrobble_row| { + const scrobble = try scrobble_row.to(Scrobble, .{ .dupe = true, .allocator = request.allocator }); var scrobble_view = try scrobbles_view.append(.object); - - var artist_infos = try scrobble_view.put("artist_info", .array); - for (scrobble.scrobbleartists) |artist| { - var artist_info = try artist_infos.append(.object); - const artist_data = try jetzig.database.Query(.Artist) - .find(artist.artist_id) - .select(.{ .id, .name }) - .execute(request.repo); - try artist_info.put("name", artist_data.?.name); - try artist_info.put("id", artist_data.?.id); - } - - try scrobble_view.put("song_name", scrobble.song.name); - try scrobble_view.put("song_id", scrobble.song.id); - try scrobble_view.put("album_name", scrobble.album.name); - try scrobble_view.put("album_id", scrobble.album.id); + try scrobble_view.put("song_name", scrobble.song_name); + try scrobble_view.put("song_id", scrobble.song_id); + try scrobble_view.put("album_name", scrobble.album_name); + try scrobble_view.put("album_id", scrobble.album_id); try scrobble_view.put("date", scrobble.date); } + + //for (scrobbles) |scrobble| { + // var scrobble_view = try scrobbles_view.append(.object); + + // var artist_infos = try scrobble_view.put("artist_info", .array); + // for (scrobble.scrobbleartists) |artist| { + // var artist_info = try artist_infos.append(.object); + // const artist_data = try jetzig.database.Query(.Artist) + // .find(artist.artist_id) + // .select(.{ .id, .name }) + // .execute(request.repo); + // try artist_info.put("name", artist_data.?.name); + // try artist_info.put("id", artist_data.?.id); + // } + + // try scrobble_view.put("song_name", scrobble.song.name); + // try scrobble_view.put("song_id", scrobble.song.id); + // try scrobble_view.put("album_name", scrobble.album.name); + // try scrobble_view.put("album_id", scrobble.album.id); + // try scrobble_view.put("date", scrobble.date); + //} return request.render(.ok); } diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 6c134b1..fe2bec4 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -4,30 +4,52 @@ const jetzig = @import("jetzig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var songs_view = try root.put("songs", .array); - const songs = try jetzig.database.Query(.Song) - .select(.{ .id, .name }) - .include(.songartists, .{ .select = .{.artist_id} }) - .include(.scrobbles, .{ .select = .{.id} }) - .orderBy(.{ .name = .asc }) - .all(request.repo); + //const songs = try jetzig.database.Query(.Song) + // .select(.{ .id, .name }) + // .include(.songartists, .{ .select = .{.artist_id} }) + // .include(.scrobbles, .{ .select = .{.id} }) + // .orderBy(.{ .name = .asc }) + // .all(request.repo); - for (songs) |song| { + const query = + \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY songs.id + \\ORDER BY scrobbles DESC + ; + + var songs_js_result = try request.repo.executeSql(query, .{}); + defer songs_js_result.deinit(); + + const Song = struct { name: []const u8, id: i32, scrobbles: i64 }; + + while (try songs_js_result.postgresql.result.next()) |song_row| { + const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); var song_view = try songs_view.append(.object); - - var artist_infos = try song_view.put("artist_info", .array); - for (song.songartists) |artist| { - var artist_info = try artist_infos.append(.object); - const artist_data = try jetzig.database.Query(.Artist) - .find(artist.artist_id) - .select(.{ .id, .name }) - .execute(request.repo); - try artist_info.put("name", artist_data.?.name); - try artist_info.put("id", artist_data.?.id); - } try song_view.put("name", song.name); try song_view.put("url", song.id); - try song_view.put("scrobbles", (song.scrobbles).len); + try song_view.put("scrobbles", song.scrobbles); } + + //for (songs) |song| { + // var song_view = try songs_view.append(.object); + + // var artist_infos = try song_view.put("artist_info", .array); + // for (song.songartists) |artist| { + // var artist_info = try artist_infos.append(.object); + // const artist_data = try jetzig.database.Query(.Artist) + // .find(artist.artist_id) + // .select(.{ .id, .name }) + // .execute(request.repo); + // try artist_info.put("name", artist_data.?.name); + // try artist_info.put("id", artist_data.?.id); + // } + // try song_view.put("name", song.name); + // try song_view.put("url", song.id); + // try song_view.put("scrobbles", (song.scrobbles).len); + //} return request.render(.ok); } From 41ab0dc88844acf0b36b0f7e5f45c938878ec062 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 00:26:56 -0400 Subject: [PATCH 06/71] Remove artists column from views I kinda just didn't want to deal with it while implementing the raw sql. Bringing it back is my next priority, but I want to do the searching in a nice way, and I'm not sure how to do that yet --- src/app/views/albums/index.zmpl | 6 ------ src/app/views/scrobbles/index.zmpl | 6 ------ src/app/views/songs/index.zmpl | 6 ------ 3 files changed, 18 deletions(-) diff --git a/src/app/views/albums/index.zmpl b/src/app/views/albums/index.zmpl index 2c259a6..7de3b87 100644 --- a/src/app/views/albums/index.zmpl +++ b/src/app/views/albums/index.zmpl @@ -10,7 +10,6 @@ Name -Artist(s) Scrobbles @@ -18,11 +17,6 @@ @for (.albums) |album| { {{album.name}} - - @for (album.get("artist_info").?) |ai| { - {{ai.name}} - } - {{album.scrobbles}} } diff --git a/src/app/views/scrobbles/index.zmpl b/src/app/views/scrobbles/index.zmpl index 377f50f..26657ff 100644 --- a/src/app/views/scrobbles/index.zmpl +++ b/src/app/views/scrobbles/index.zmpl @@ -10,7 +10,6 @@ Song -Artist(s) Album Date @@ -19,11 +18,6 @@ @for (.scrobbles) |scrobble| { {{scrobble.song_name}} - - @for (scrobble.get("artist_info").?) |ai| { - {{ai.name}} - } - {{scrobble.album_name}} {{scrobble.date}} diff --git a/src/app/views/songs/index.zmpl b/src/app/views/songs/index.zmpl index a0a1128..85eaa25 100644 --- a/src/app/views/songs/index.zmpl +++ b/src/app/views/songs/index.zmpl @@ -10,7 +10,6 @@ Name -Artists(s) Scrobbles @@ -18,11 +17,6 @@ @for (.songs) |song| { {{song.name}} - - @for (song.get("artist_info").?) |ai| { - {{ai.name}} - } - {{song.scrobbles}} } From 2c4af0b37893d84cfd0061cc42f7d3d22be7d355 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 14:05:15 -0400 Subject: [PATCH 07/71] Include artist column for albums I'm convinced there's a better way of doing this, but this is all I can think of right now --- src/app/views/albums.zig | 18 +++++++++++++++++- src/app/views/albums/index.zmpl | 6 ++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 9f363f5..3fbb8ab 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -22,7 +22,10 @@ pub fn index(request: *jetzig.Request) !jetzig.View { \\ORDER BY scrobbles DESC ; - var albums_jq_result = try request.repo.executeSql(query, .{}); + var inter_conn = try request.repo.connect(); + defer inter_conn.release(); + + var albums_jq_result = try inter_conn.executeSql(query, .{}, null, request.repo); defer albums_jq_result.deinit(); const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; @@ -30,6 +33,19 @@ pub fn index(request: *jetzig.Request) !jetzig.View { while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); var album_view = try albums_view.append(.object); + var artist_infos = try album_view.put("artist_info", .array); + const artist_data = try jetzig.database.Query(.Artistalbum) + .select(.{.id}) + .where(.{ .album_id = album.id }) + .include(.artist, .{ .select = .{ .name, .id } }) + .all(request.repo); + + for (artist_data) |artist| { + var artist_info = try artist_infos.append(.object); + try artist_info.put("name", artist.artist.name); + try artist_info.put("url", artist.artist.id); + } + try album_view.put("name", album.name); try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); diff --git a/src/app/views/albums/index.zmpl b/src/app/views/albums/index.zmpl index 7de3b87..e5536eb 100644 --- a/src/app/views/albums/index.zmpl +++ b/src/app/views/albums/index.zmpl @@ -10,6 +10,7 @@ Name +Artist(s) Scrobbles @@ -17,6 +18,11 @@ @for (.albums) |album| { {{album.name}} + + @for (album.get("artist_info").?) |ai| { + {{ai.name}} + } + {{album.scrobbles}} } From 4d63844def269fc78f2d50bcac9b31e694c16257 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 15:05:44 -0400 Subject: [PATCH 08/71] Make artist retrieval apart of main query This feels bad or wrong somehow, but it do be working tho --- src/app/views/albums.zig | 45 +++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 3fbb8ab..2100051 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -14,41 +14,52 @@ pub fn index(request: *jetzig.Request) !jetzig.View { ////const albums = try request.repo.all(query); const query = - \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\GROUP BY albums.id + \\INNER JOIN artistalbums ON artistalbums.album_id = albums.id + \\INNER JOIN artists ON artists.id = artistalbums.artist_id + \\GROUP BY albums.id, artists.id \\ORDER BY scrobbles DESC ; - var inter_conn = try request.repo.connect(); - defer inter_conn.release(); + //var inter_conn = try request.repo.connect(); + //defer inter_conn.release(); - var albums_jq_result = try inter_conn.executeSql(query, .{}, null, request.repo); + var albums_jq_result = try request.repo.executeSql(query, .{}); defer albums_jq_result.deinit(); - const Album = struct { name: []const u8, id: i32, scrobbles: i64 }; + const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; - while (try albums_jq_result.postgresql.result.next()) |album_row| { + var prev_album_id: ?i32 = null; + var prev_artist_infos = try root.put("test", .array); + + blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); + if (album.id == prev_album_id) { + var artist_info = try prev_artist_infos.append(.object); + try artist_info.put("name", album.artist_name); + try artist_info.put("url", album.artist_id); + continue :blk; + } var album_view = try albums_view.append(.object); var artist_infos = try album_view.put("artist_info", .array); - const artist_data = try jetzig.database.Query(.Artistalbum) - .select(.{.id}) - .where(.{ .album_id = album.id }) - .include(.artist, .{ .select = .{ .name, .id } }) - .all(request.repo); + //const artist_data = try jetzig.database.Query(.Artistalbum) + // .select(.{.id}) + // .where(.{ .album_id = album.id }) + // .include(.artist, .{ .select = .{ .name, .id } }) + // .all(request.repo); - for (artist_data) |artist| { - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", artist.artist.name); - try artist_info.put("url", artist.artist.id); - } + var artist_info = try artist_infos.append(.object); + try artist_info.put("name", album.artist_name); + try artist_info.put("url", album.artist_id); try album_view.put("name", album.name); try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); + prev_artist_infos = artist_infos; + prev_album_id = album.id; } return request.render(.ok); } From 387493d3c0c339613c2cf232620aaaeba4ec918d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 15:17:10 -0400 Subject: [PATCH 09/71] Change typedef of prev_artist_infos Feeling much better about my choices this time around --- src/app/views/albums.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 2100051..00b2338 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -33,12 +33,12 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; var prev_album_id: ?i32 = null; - var prev_artist_infos = try root.put("test", .array); + var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); if (album.id == prev_album_id) { - var artist_info = try prev_artist_infos.append(.object); + var artist_info = try prev_artist_infos.?.append(.object); try artist_info.put("name", album.artist_name); try artist_info.put("url", album.artist_id); continue :blk; From ff8cdabbf1e2b3b8706d71fb43a89e1b73f315b9 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 17 Apr 2025 15:28:00 -0400 Subject: [PATCH 10/71] Cleanup --- src/app/views/albums.zig | 43 +--------------------------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 00b2338..57d2bb1 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -5,14 +5,6 @@ const jetquery = @import("jetzig").jetquery; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var albums_view = try root.put("albums", .array); - //const albums = try jetzig.database.Query(.Album) - // .select(.{ .id, .name }) - // .include(.albumartists, .{ .select = .{.artist_id} }) - // .include(.scrobbles, .{ .select = .{.id} }) - // .orderBy(.{ .name = .asc }) - // .all(request.repo); - ////const albums = try request.repo.all(query); - const query = \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs @@ -24,9 +16,6 @@ pub fn index(request: *jetzig.Request) !jetzig.View { \\ORDER BY scrobbles DESC ; - //var inter_conn = try request.repo.connect(); - //defer inter_conn.release(); - var albums_jq_result = try request.repo.executeSql(query, .{}); defer albums_jq_result.deinit(); @@ -45,12 +34,6 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } var album_view = try albums_view.append(.object); var artist_infos = try album_view.put("artist_info", .array); - //const artist_data = try jetzig.database.Query(.Artistalbum) - // .select(.{.id}) - // .where(.{ .album_id = album.id }) - // .include(.artist, .{ .select = .{ .name, .id } }) - // .all(request.repo); - var artist_info = try artist_infos.append(.object); try artist_info.put("name", album.artist_name); try artist_info.put("url", album.artist_id); @@ -58,6 +41,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { try album_view.put("name", album.name); try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); + prev_artist_infos = artist_infos; prev_album_id = album.id; } @@ -92,31 +76,6 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { try song_view.put("url", song.id); try song_view.put("scrobbles", song.scrobbles); } - - //const album = try jetzig.database.Query(.Album) - // .find(id) - // .select(.{ .id, .name }) - // .execute(request.repo); - //var root = try request.data(.object); - //try root.put("album", album.?.name); - //var songs_view = try root.put("songs", .array); - //const query = jetzig.database.Query(.Albumsong) - // .select(.{.id}) - // .include(.song, .{ .select = .{ .name, .id } }) - // .join(.inner, .album) - // .where(.{ .album = .{ .id = id } }); - - //const songs = try request.repo.all(query); - //for (songs) |song| { - // const scrobbles = try jetzig.database.Query(.Scrobble) - // .where(.{ .song_id = song.song.id }) - // .count() - // .execute(request.repo); - // var song_view = try songs_view.append(.object); - // try song_view.put("name", song.song.name); - // try song_view.put("url", song.song.id); - // try song_view.put("scrobbles", scrobbles); - //} return request.render(.ok); } From 18cdb48b5326b819783dcfbe73dcfab67bd6ab58 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 18 Apr 2025 21:29:00 -0400 Subject: [PATCH 11/71] Begin rules --- src/app/jobs/process_rule.zig | 32 +++++++++++ src/app/views/rules.zig | 21 ++++++- src/app/views/rules/index.zmpl | 102 ++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 src/app/jobs/process_rule.zig diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig new file mode 100644 index 0000000..fb0d78b --- /dev/null +++ b/src/app/jobs/process_rule.zig @@ -0,0 +1,32 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + _ = env; + //_ = params; + + const rule = try params.toJson(); + + //const rule = struct { + // name: []const u8 = params, + // conditions: []struct { + // match_on: []const u8, + // match_cond: []const u8, + // match_text: []const u8, + // }, + // actions: []struct { + // action: []const u8, + // action_cond: []const u8, + // action_text: []const u8, + // }, + //}; + + //var file = try std.fs.cwd().openFile("rules.json", .{}); + + //_ = try file.write(rule); + try std.fs.cwd().writeFile(.{ "rules.json", rule, .{} }); + + // Job execution code goes here. Add any code that you would like to run in the background. + //try env.logger.INFO("Running a job.", .{}); +} diff --git a/src/app/views/rules.zig b/src/app/views/rules.zig index a2a222e..bb8ea4b 100644 --- a/src/app/views/rules.zig +++ b/src/app/views/rules.zig @@ -20,6 +20,26 @@ pub fn edit(id: []const u8, request: *jetzig.Request) !jetzig.View { } pub fn post(request: *jetzig.Request) !jetzig.View { + const params = try request.params(); + + var job = try request.job("process_rule"); + + _ = try job.params.put("name", params.get("rule-title")); + + var conditionals = try job.params.put("conditionals", .array); + var cond0 = try conditionals.append(.object); + try cond0.put("match_on", params.get("match-on")); + try cond0.put("match_cond", params.get("match-cond")); + try cond0.put("match_txt", params.get("match-txt")); + + var actions = try job.params.put("actions", .array); + var act0 = try actions.append(.object); + try act0.put("action", params.get("action")); + try act0.put("action_on", params.get("action-on")); + try act0.put("action_txt", params.get("action-txt")); + + try job.schedule(); + return request.render(.created); } @@ -38,7 +58,6 @@ pub fn delete(id: []const u8, request: *jetzig.Request) !jetzig.View { return request.render(.ok); } - test "index" { var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); defer app.deinit(); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index 76457d0..4d15188 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -1,3 +1,99 @@ -
- Content goes here -
+ + + + + +@partial partials/header +

Rules

+Rules allow you change the default Scrobble import behavior based on provided criteria. +Add a rule below. +

+
+ + +
+If + + + + + + + +
+then + + +with + + +
+ +Current rules: + + \ No newline at end of file From 5383b69eb66c72aaf31a6ee6e1d057a708843d26 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 19 Apr 2025 15:01:30 -0400 Subject: [PATCH 12/71] Allow reading and writing rules.json I like the idea of letting the user write to a file themselves for rules, but I think this is going to significantly slow things down. Will probably switch to SQL table at some point. Also very hardcoded for my purposes. ALSO the code looks bad, I think there must be a better way... --- .gitignore | 3 +- src/app/jobs/process_rule.zig | 63 ++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 70b0239..76f5b26 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ static/ src/app/database/data.db-journal src/app/database/old_migrations/ src/lib -src/app/scripts/ \ No newline at end of file +src/app/scripts/ +rules.json \ No newline at end of file diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index fb0d78b..93d1279 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -2,31 +2,54 @@ const std = @import("std"); const jetzig = @import("jetzig"); pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { - _ = allocator; _ = env; //_ = params; - const rule = try params.toJson(); + const Rule = struct { + name: []const u8, + conditionals: []struct { + match_on: []const u8, + match_cond: []const u8, + match_txt: []const u8, + }, + actions: []struct { + action: []const u8, + action_on: []const u8, + action_txt: []const u8, + }, + }; - //const rule = struct { - // name: []const u8 = params, - // conditions: []struct { - // match_on: []const u8, - // match_cond: []const u8, - // match_text: []const u8, - // }, - // actions: []struct { - // action: []const u8, - // action_cond: []const u8, - // action_text: []const u8, - // }, - //}; + const Rules = struct { + rules: []Rule, + }; - //var file = try std.fs.cwd().openFile("rules.json", .{}); + const rule = try std.json.parseFromSliceLeaky(Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); - //_ = try file.write(rule); - try std.fs.cwd().writeFile(.{ "rules.json", rule, .{} }); + const file_read: std.fs.File = std.fs.cwd().openFile("rules.json", .{}) catch |read_err| switch (read_err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true, .exclusive = true }) catch |write_err| switch (write_err) { + error.PathAlreadyExists => unreachable, + else => { + std.log.debug("{any} while writing file", .{write_err}); + return; + }, + }, + else => { + std.log.debug("{any} while reading file", .{read_err}); + return; + }, + }; - // Job execution code goes here. Add any code that you would like to run in the background. - //try env.logger.INFO("Running a job.", .{}); + var rules = std.ArrayList(Rule).init(allocator); + const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); + if (file_content.len != 0) { + const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + try rules.appendSlice(content.rules); + } + try rules.append(rule); + file_read.close(); + + const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); + const out_rules = Rules{ .rules = rules.items }; + const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file_write.writeAll(out); } From baf9ef38a4c83fb4588d7c12f883cec294f1bd95 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 19 Apr 2025 15:36:51 -0400 Subject: [PATCH 13/71] Simplify file creation branch of process_rule.zig Still not quite where I want it, but definitely better than what it was --- src/app/jobs/process_rule.zig | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 93d1279..7cb1a71 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -20,18 +20,25 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }; const Rules = struct { - rules: []Rule, + rules: []const Rule, }; const rule = try std.json.parseFromSliceLeaky(Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); const file_read: std.fs.File = std.fs.cwd().openFile("rules.json", .{}) catch |read_err| switch (read_err) { - error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true, .exclusive = true }) catch |write_err| switch (write_err) { - error.PathAlreadyExists => unreachable, - else => { - std.log.debug("{any} while writing file", .{write_err}); - return; - }, + error.FileNotFound => { + const file = std.fs.cwd().createFile("rules.json", .{ .read = true, .exclusive = true }) catch |write_err| switch (write_err) { + error.PathAlreadyExists => unreachable, + else => { + std.log.debug("{any} while writing file", .{write_err}); + return; + }, + }; + const out_rules = Rules{ .rules = &[_]Rule{rule} }; + const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file.writeAll(out); + file.close(); + return; }, else => { std.log.debug("{any} while reading file", .{read_err}); @@ -40,16 +47,19 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }; var rules = std.ArrayList(Rule).init(allocator); + defer rules.deinit(); + const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); - if (file_content.len != 0) { - const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); - try rules.appendSlice(content.rules); - } + const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + try rules.appendSlice(content.rules); try rules.append(rule); file_read.close(); const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); const out_rules = Rules{ .rules = rules.items }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file_write.writeAll(out); + file_write.close(); + return; } From 445ca45fa96576fe4e8ada1729cb40fa570e074f Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 21 Apr 2025 00:17:16 -0400 Subject: [PATCH 14/71] Begin rule application The more I think about this, the more I think it's gonna be super slow and bad. There must bve a good way of doing this, but I'm not sure how... --- src/app/jobs/process_scrobbles.zig | 7 ++++++- src/apply_rule.zig | 11 +++++++++++ src/types.zig | 26 ++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 src/apply_rule.zig diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 70c2976..bbfc9d4 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -14,14 +14,19 @@ const lastfm = @import("../../types.zig").LastFM; // - logger: Logger attached to the same stream as the Jetzig server. // - environment: Enum of `{ production, development }`. pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { - _ = allocator; //_ = env; + const file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); + const file_content = try file.readToEndAlloc(allocator, 16_000_000); + + const rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { //const fixed_date: u32 = @as(u32, item.getT(.integer, "date").?); const scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; + for (rules) |rule| {} + // Make hashes //const album_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.album))); //const artist_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.artist))); diff --git a/src/apply_rule.zig b/src/apply_rule.zig new file mode 100644 index 0000000..871c996 --- /dev/null +++ b/src/apply_rule.zig @@ -0,0 +1,11 @@ +const Scrobble = @import("types").LastFMScrobble; +const Rules = @import("types").Rules; + +pub fn applyRule(scrobble: Scrobble, rules: Rules) !Scrobble { + var output_scrobble: Scrobble = scrobble; + for (rules) |rule| { + for (rule.conditionals) |cond| { + switch (cond.match_cond) {} + } + } +} diff --git a/src/types.zig b/src/types.zig index 55bca1c..00e063c 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,5 +1,3 @@ -const zeit = @import("zeit"); - pub const LastFMScrobble = struct { track: []const u8, artist: []const u8, @@ -36,3 +34,27 @@ pub const SpotifyScrobble = struct { offline_timestamp: u64, incognito_mode: ?bool, }; + +const Rule = struct { + name: []const u8, + conditionals: []struct { + match_on: MatchOn, + match_cond: enum { is, contains }, + match_txt: []const u8, + }, + actions: []struct { + action: []const u8, + action_on: enum { is, contains }, + action_txt: []const u8, + }, +}; + +const Rules = struct { + rules: []const Rule, +}; + +const MatchOn = enum { + artist, + album, + song, +}; From 87a2fe2d3479ace74d3d11439bd6ad5e9f184e80 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 21 Apr 2025 12:23:20 -0400 Subject: [PATCH 15/71] Complete preliminary find and replace rules Tested by replacing AJR with John Van Derwood. Need to test on albums and artists, as well as matching on one piece of metadata, and replacing another --- src/app/jobs/process_scrobbles.zig | 9 +++++--- src/app/views/rules/index.zmpl | 2 +- src/apply_rule.zig | 33 +++++++++++++++++++++++++----- src/types.zig | 12 +++++------ 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index bbfc9d4..06d7365 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -3,6 +3,9 @@ const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; const Scrobble = @import("../../types.zig").LastFMScrobble; const lastfm = @import("../../types.zig").LastFM; +//const Rules = @import("../../types.zig").Rules; +const Data = @import("../../types.zig"); +const rules = @import("../../apply_rule.zig"); // The `run` function for a job is invoked every time the job is processed by a queue worker // (or by the Jetzig server if the job is processed in-line). @@ -18,14 +21,14 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig const file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); const file_content = try file.readToEndAlloc(allocator, 16_000_000); - const rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + const rule_list = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { //const fixed_date: u32 = @as(u32, item.getT(.integer, "date").?); - const scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; + const pre_scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; - for (rules) |rule| {} + const scrobble = rules.applyScrobbleRule(pre_scrobble, rule_list); // Make hashes //const album_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.album))); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index 4d15188..b9a1b17 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -21,7 +21,7 @@ If @@ -87,7 +91,7 @@ then with diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 53f3b27..032da9d 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -3,6 +3,8 @@ const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; const ScrobbleTypes = @import("../../types.zig"); const zeit = @import("zeit"); +const rules = @import("../../apply_rule.zig"); +const Data = @import("../../types.zig"); pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { _ = data; @@ -31,6 +33,11 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; + const rule_file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); + defer rule_file.close(); + const file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); + const rule_list = try std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}); + // The only difference between a LastFM scrobble and a Spotify scrobble is the format. // I've made a branches for each, because doing it all in one made the readability terrible, // and formatting the date in particular was challenging. I could probably pull out the @@ -46,12 +53,14 @@ pub fn post(request: *jetzig.Request) !jetzig.View { if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; var value = try scrobbles_data.append(.object); + const formatted_scrobble = rules.applyScrobbleRule(scrobble, rule_list); + // This is so unnecessary, probably useful once I start doing Spotify integration though inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| { - try value.put(f.name, @as(f.type, @field(scrobble, f.name))); + try value.put(f.name, @as(f.type, @field(formatted_scrobble, f.name))); } // Note sure why this works for ZMPL, but not for jobs. - try scrobbles_view.append(scrobble); + try scrobbles_view.append(formatted_scrobble); } }, 1 => { @@ -88,7 +97,9 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } // Turn SpotifyScrobble into a LastFM scrobble - const formatted_scrobble: ScrobbleTypes.LastFMScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const pre_formatted_scrobble: ScrobbleTypes.LastFMScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + + const formatted_scrobble = rules.applyScrobbleRule(pre_formatted_scrobble, rule_list); var value = try scrobbles_data.append(.object); diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 769d062..92d7598 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -12,7 +12,7 @@ pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { inline else => |on| match_found = match_found and std.mem.eql(u8, @field(scrobble, @tagName(on)), cond.match_txt), }, .contains => switch (cond.match_on) { - inline else => |on| match_found = match_found and (std.mem.count(u8, @field(scrobble, @tagName(on)), cond.match_txt) > 0), + inline else => |on| match_found = match_found and std.mem.containsAtLeast(u8, @field(scrobble, @tagName(on)), 1, cond.match_txt), }, } } From e9c72041a530cf283178d9fcefe3eee693daff03 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 22 Apr 2025 13:50:39 -0400 Subject: [PATCH 17/71] Allow multiple conditions in rules. Scrobble processing appears noticeably slower (according to the logs), so I think rules are going to be something to optimize later. Fortunately, they shouldn't need to be applied too often --- build.zig.zon | 4 +- src/app/jobs/process_rule.zig | 29 +++--------- src/app/views/rules.zig | 13 +++-- src/app/views/rules/index.zmpl | 87 ++++++++++------------------------ src/apply_rule.zig | 28 ++++++++--- src/types.zig | 3 +- 6 files changed, 67 insertions(+), 97 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 823b42c..0839430 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -17,8 +17,8 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/2c52792217b9441ed5e91d67e7ec5a8959285307.tar.gz", - .hash = "jetzig-0.0.0-IpAgLbMeDwDYRqWu0OJixGbcDUoUbfAN0xGe1xsYRHTj", + .url = "https://github.com/jetzig-framework/jetzig/archive/86d82026ab574d4e5c3c6cc3817dda84b510001a.tar.gz", + .hash = "jetzig-0.0.0-IpAgLTkzDwDKmsY9MqM41EHDXWGkViiECa0lzV8xl17x", }, .zeit = .{ .url = "https://github.com/rockorager/zeit/archive/refs/heads/main.tar.gz", diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 7cb1a71..f211dbe 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -1,29 +1,14 @@ const std = @import("std"); const jetzig = @import("jetzig"); +const Data = @import("../../types.zig"); pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { _ = env; //_ = params; - const Rule = struct { - name: []const u8, - conditionals: []struct { - match_on: []const u8, - match_cond: []const u8, - match_txt: []const u8, - }, - actions: []struct { - action: []const u8, - action_on: []const u8, - action_txt: []const u8, - }, - }; + std.log.debug("{s}", .{try params.toJson()}); - const Rules = struct { - rules: []const Rule, - }; - - const rule = try std.json.parseFromSliceLeaky(Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); + const rule = try std.json.parseFromSliceLeaky(Data.Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); const file_read: std.fs.File = std.fs.cwd().openFile("rules.json", .{}) catch |read_err| switch (read_err) { error.FileNotFound => { @@ -34,7 +19,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig return; }, }; - const out_rules = Rules{ .rules = &[_]Rule{rule} }; + const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file.writeAll(out); file.close(); @@ -46,17 +31,17 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }, }; - var rules = std.ArrayList(Rule).init(allocator); + var rules = std.ArrayList(Data.Rule).init(allocator); defer rules.deinit(); const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); - const content: Rules = try std.json.parseFromSliceLeaky(Rules, allocator, file_content, .{}); + const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); try rules.appendSlice(content.rules); try rules.append(rule); file_read.close(); const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); - const out_rules = Rules{ .rules = rules.items }; + const out_rules = Data.Rules{ .rules = rules.items }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file_write.writeAll(out); diff --git a/src/app/views/rules.zig b/src/app/views/rules.zig index bb8ea4b..957f5d2 100644 --- a/src/app/views/rules.zig +++ b/src/app/views/rules.zig @@ -25,12 +25,17 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var job = try request.job("process_rule"); _ = try job.params.put("name", params.get("rule-title")); + _ = try job.params.put("cond_req", params.get("cond-req")); var conditionals = try job.params.put("conditionals", .array); - var cond0 = try conditionals.append(.object); - try cond0.put("match_on", params.get("match-on")); - try cond0.put("match_cond", params.get("match-cond")); - try cond0.put("match_txt", params.get("match-txt")); + inline for (0..5) |i| { + if (!std.mem.eql(u8, "", params.getT(.string, comptime std.fmt.comptimePrint("match-txt{}", .{i})).?)) { + var cond = try conditionals.append(.object); + try cond.put("match_on", params.get(comptime std.fmt.comptimePrint("match-on{}", .{i}))); + try cond.put("match_cond", params.get(comptime std.fmt.comptimePrint("match-cond{}", .{i}))); + try cond.put("match_txt", params.get(comptime std.fmt.comptimePrint("match-txt{}", .{i}))); + } + } var actions = try job.params.put("actions", .array); var act0 = try actions.append(.object); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index ab55c96..038ce71 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -12,74 +12,37 @@ Add a rule below.
+Match + +conditonals. +
If - - - - - - - +@for (0..5) |i| { + + + + + + + +
+}
diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 92d7598..db6eb42 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -2,17 +2,33 @@ const std = @import("std"); const Scrobble = @import("./types.zig").LastFMScrobble; const Rules = @import("./types.zig").Rules; +// Wrapper for containsAtLeast to make the switch below to work +fn containsAtLeastOne(haystack: []const u8, needle: []const u8) bool { + return std.mem.containsAtLeast(u8, haystack, 1, needle); +} + +fn eqlDecomped(haystack: []const u8, needle: []const u8) bool { + return std.mem.eql(u8, haystack, needle); +} + pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { - var match_found: bool = true; var output_scrobble: Scrobble = scrobble; for (rules.rules) |rule| { + var match_found: bool = switch (rule.cond_req) { + .any => false, + .all => true, + }; for (rule.conditionals) |cond| { - switch (cond.match_cond) { - .is => switch (cond.match_on) { - inline else => |on| match_found = match_found and std.mem.eql(u8, @field(scrobble, @tagName(on)), cond.match_txt), + const match_fn: *const fn ([]const u8, []const u8) bool = switch (cond.match_cond) { + .is => eqlDecomped, + .contains => containsAtLeastOne, + }; + switch (rule.cond_req) { + .any => switch (cond.match_on) { + inline else => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), }, - .contains => switch (cond.match_on) { - inline else => |on| match_found = match_found and std.mem.containsAtLeast(u8, @field(scrobble, @tagName(on)), 1, cond.match_txt), + .all => switch (cond.match_on) { + inline else => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), }, } } diff --git a/src/types.zig b/src/types.zig index 61b0e88..0e385a5 100644 --- a/src/types.zig +++ b/src/types.zig @@ -35,8 +35,9 @@ pub const SpotifyScrobble = struct { incognito_mode: ?bool, }; -const Rule = struct { +pub const Rule = struct { name: []const u8, + cond_req: enum { any, all }, conditionals: []struct { match_on: ScrobbleFields, match_cond: enum { is, contains }, From 0631ded1155a98df046617f08b3c5a5c7333f0ea Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 23 Apr 2025 19:32:32 -0400 Subject: [PATCH 18/71] Work on add artist action in rules Really close to having it work, but there seems to be an error when uploading files, which causes particularly annoying problems on WSL when testing, so I'm commiting and trying on my desktop. --- src/app/jobs/process_rule.zig | 14 +- src/app/jobs/process_scrobbles.zig | 217 ++++++++++++++++++++--------- src/app/views/partials/_table.zmpl | 6 +- src/app/views/rules/index.zmpl | 3 +- src/app/views/upload.zig | 114 +++++++++------ src/apply_rule.zig | 45 ++++-- src/types.zig | 46 +++--- 7 files changed, 300 insertions(+), 145 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index f211dbe..217826a 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -35,12 +35,20 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig defer rules.deinit(); const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); - const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); - try rules.appendSlice(content.rules); - try rules.append(rule); file_read.close(); const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); + if (file_content.len == 0) { + const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; + const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + try file_write.writeAll(out); + file_write.close(); + return; + } + const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); + try rules.appendSlice(content.rules); + try rules.append(rule); + const out_rules = Data.Rules{ .rules = rules.items }; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 4262fe5..87871c7 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -1,9 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const Scrobble = @import("../../types.zig").LastFMScrobble; const lastfm = @import("../../types.zig").LastFM; -//const Rules = @import("../../types.zig").Rules; const Data = @import("../../types.zig"); const rules = @import("../../apply_rule.zig"); @@ -18,26 +16,50 @@ const rules = @import("../../apply_rule.zig"); // - environment: Enum of `{ production, development }`. pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { //_ = env; - _ = allocator; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { - //const fixed_date: u32 = @as(u32, item.getT(.integer, "date").?); - const scrobble: Scrobble = .{ .track = item.getT(.string, "track").?, .artist = item.getT(.string, "artist").?, .album = item.getT(.string, "album") orelse "", .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))) }; + //var buffer: [256**4]u8 = undefined; + //var fba = std.heap.FixedBufferAllocator.init(&buffer); + //const alloc = fba.allocator(); - // Make hashes - //const album_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.album))); - //const artist_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.artist))); - //const song_hash = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.track))); + var alssu8 = std.ArrayList([]const u8).init(allocator); + defer alssu8.deinit(); - // Create a buffer to hold the metadata to hash. Numbers based on the title of a - // particularly long Sufjan Stevens song title, and we're gonna pray the metadata - // does not exceed three times it's length. - var buffer = [_]u8{undefined} ** (288 * 3); - const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(scrobble.artist))); - const album_prehash = try std.fmt.bufPrint(&buffer, "{s}{s}", .{ scrobble.artist, scrobble.album }); - const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(album_prehash))); - const song_prehash = try std.fmt.bufPrint(&buffer, "{s}{s}{s}", .{ scrobble.artist, scrobble.album, scrobble.track }); - const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(song_prehash))); + for (item.getT(.array, "artists_track").?.items()) |artist| { + try alssu8.append(try artist.coerce([]const u8)); + } + + const track_artists = try alssu8.toOwnedSlice(); + + for (item.getT(.array, "artists_album").?.items()) |artist| { + try alssu8.append(try artist.coerce([]const u8)); + } + + const album_artists = try alssu8.toOwnedSlice(); + + const scrobble: Data.Scrobble = .{ + .track = item.getT(.string, "track").?, + .artists_track = track_artists, + .album = item.getT(.string, "album") orelse "", + .artists_album = album_artists, + .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), + }; + + var id_prehash = std.ArrayList(u8).init(allocator); + defer id_prehash.deinit(); + + var alartist = std.ArrayList(struct { name: []const u8, id: i32 }).init(allocator); + defer alartist.deinit(); + + for (scrobble.artists_track) |artist| { + //try id_prehash.appendSlice(artist); + try alartist.append(.{ .name = artist, .id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist))) }); + } + //const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); + try id_prehash.appendSlice(scrobble.album); + const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); + try id_prehash.appendSlice(scrobble.track); + const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); // Make IDs // Song: Song hash XOR artist hash XOR album hash @@ -65,64 +87,121 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); //const song_id = (song_hash ^ artist_hash ^ album_hash); + var albumsong = try jetzig.database.Query(.Albumsong) + .findBy(.{ + .album_id = album_id, + .song_id = song_id, + }) + .select(.{.id}) + .execute(env.repo); - var albumsong = try jetzig.database.Query(.Albumsong).findBy(.{ .album_id = album_id, .song_id = song_id }).select(.{.id}).execute(env.repo); - var ins_album = try jetzig.database.Query(.Album).find(album_id).select(.{.id}).execute(env.repo); + var ins_album = try jetzig.database.Query(.Album) + .find(album_id) + .select(.{.id}) + .execute(env.repo); - var ins_artist = try jetzig.database.Query(.Artist).find(artist_id).select(.{.id}).execute(env.repo); - if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); + for (alartist.items) |artist| { + var ins_artist = try jetzig.database.Query(.Artist) + .find(artist.id) + .select(.{.id}) + .execute(env.repo); - if (albumsong == null) { - var ins_song = try jetzig.database.Query(.Song).find(song_id).select(.{.id}).execute(env.repo); - if (ins_song == null) ins_song = try jetzig.database.Query(.Song).insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }).returning(.{.id}).execute(env.repo); + if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist) + .insert(.{ + .id = artist.id, + .name = artist.name, + .disambiguation = null, + }) + .returning(.{.id}) + .execute(env.repo); - if (ins_album == null) { - ins_album = try jetzig.database.Query(.Album).insert(.{ .id = album_id, .name = scrobble.album, .length = null }).returning(.{.id}).execute(env.repo); - // I think there's still technically a bug here when you have a different artist but I'm not sure - try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist.?.id, .album_id = ins_album.?.id }).execute(env.repo); + if (albumsong == null) { + var ins_song = try jetzig.database.Query(.Song) + .find(song_id) + .select(.{.id}) + .execute(env.repo); + + if (ins_song == null) ins_song = try jetzig.database.Query(.Song) + .insert(.{ + .id = song_id, + .name = scrobble.track, + .length = null, + .hidden = false, + }) + .returning(.{.id}) + .execute(env.repo); + + if (ins_album == null) { + ins_album = try jetzig.database.Query(.Album) + .insert(.{ + .id = album_id, + .name = scrobble.album, + .length = null, + }) + .returning(.{.id}) + .execute(env.repo); + // I think there's still technically a bug here when you have a different artist but I'm not sure + try jetzig.database.Query(.Artistalbum) + .insert(.{ + .artist_id = ins_artist.?.id, + .album_id = ins_album.?.id, + }) + .execute(env.repo); + } + + albumsong = try jetzig.database.Query(.Albumsong) + .insert(.{ + .song_id = ins_song.?.id, + .album_id = ins_album.?.id, + }) + .returning(.{.id}) + .execute(env.repo); + + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ + .albumsong_id = albumsong.?.id, + .artist_id = ins_artist.?.id, + }) + .execute(env.repo); + } else { + const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) + .findBy(.{ + .albumsong_id = albumsong.?.id, + .artist_id = ins_artist.?.id, + }) + .select(.{.id}) + .execute(env.repo); + + if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist) + .insert(.{ + .albumsong_id = albumsong.?.id, + .artist_id = ins_artist.?.id, + }) + .execute(env.repo); + + const ins_artistalbum = try jetzig.database.Query(.Artistalbum) + .findBy(.{ + .album_id = ins_album.?.id, + .artist_id = ins_artist.?.id, + }) + .select(.{.id}) + .execute(env.repo); + + if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum) + .insert(.{ + .album_id = ins_album.?.id, + .artist_id = ins_artist.?.id, + }) + .execute(env.repo); } - - albumsong = try jetzig.database.Query(.Albumsong).insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }).returning(.{.id}).execute(env.repo); - - try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); - } else { - const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist).findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); - if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist).insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); - - const ins_artistalbum = try jetzig.database.Query(.Artistalbum).findBy(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).select(.{.id}).execute(env.repo); - if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum).insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } - //if (ins_artist_id == null) { - // ins_artist_id = try jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .disambiguation = null }).returning(.{.id}).execute(env.repo); - // try jetzig.database.Query(.Albumsongartist).insert(.{ .albumsong_id = albumsong_id, .artist_id = ins_artist_id }); - // try jetzig.database.Query(.Artistalbum).insert(.{ .artist_id = ins_artist_id, .album_id = ins_album_id }); - //} - - try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); + try jetzig.database.Query(.Scrobble) + .insert(.{ + .albumsong = albumsong.?.id, + .datetime = scrobble.date, + }) + .execute(env.repo); } } - - // I would like to replicate this kind of functionality for several kinds of queries - // This one gives me all albums by Dream Theater (it also returns Dream Theater for - // each entry, but removing artists.name from the SELECT would remove that) - // - // SELECT - // artists.name, albums.name - // FROM - // "Albumartists" - // INNER JOIN artists - // ON "Albumartists".artist_id = artists.id - // INNER JOIN albums - // ON "Albumartists".album_id = albums.id - // WHERE artists.name = 'Dream Theater'; - - //const query = jetzig.database.Query(.Artist).include(.artistalbums, .{}); - //const results = try env.repo.all(query); - //defer env.repo.free(results); - //for (results) |result| { - // for (result.artistalbums) |artistalbum| { - // std.log.debug("{s}: {any}", .{ result.name, artistalbum.album_id }); - // } - //} } diff --git a/src/app/views/partials/_table.zmpl b/src/app/views/partials/_table.zmpl index 8ea394d..83c1cd4 100644 --- a/src/app/views/partials/_table.zmpl +++ b/src/app/views/partials/_table.zmpl @@ -10,7 +10,11 @@ @for (table_data) |value| { {{value.track}} - {{value.artist}} + + @for (value.get("artists").?) |artist| { + {{artist}} + } + {{value.album}} {{value.date}} diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index 038ce71..f97ef35 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -52,7 +52,8 @@ then diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 032da9d..98f0009 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -1,7 +1,6 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const ScrobbleTypes = @import("../../types.zig"); const zeit = @import("zeit"); const rules = @import("../../apply_rule.zig"); const Data = @import("../../types.zig"); @@ -33,58 +32,70 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - const rule_file = try std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }); + const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), + else => err, + }); + defer rule_file.close(); const file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = try std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}); + const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}) catch null; - // The only difference between a LastFM scrobble and a Spotify scrobble is the format. - // I've made a branches for each, because doing it all in one made the readability terrible, - // and formatting the date in particular was challenging. I could probably pull out the - // actual appending at some point, since that's the same process for each, but I'm not - // sure how to do that yet. switch (source) { 0 => { - const content: ScrobbleTypes.LastFM = try std.json.parseFromSliceLeaky(ScrobbleTypes.LastFM, request.allocator, file.content, .{}); + const content: Data.LastFM = try std.json.parseFromSliceLeaky(Data.LastFM, request.allocator, file.content, .{}); const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; appends: for (content.scrobbles) |scrobble| { // We can short-circuit on the limiter bools if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; - var value = try scrobbles_data.append(.object); - const formatted_scrobble = rules.applyScrobbleRule(scrobble, rule_list); + const formatted_scrobble = if (rule_list) |rl| + rules.applyScrobbleRule(request.allocator, scrobble, rl) + else + Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date, + }; - // This is so unnecessary, probably useful once I start doing Spotify integration though - inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| { - try value.put(f.name, @as(f.type, @field(formatted_scrobble, f.name))); + var scrobble_view = try scrobbles_view.append(.object); + var artists = try scrobble_view.put("artists", .array); + + try scrobble_view.put("track", formatted_scrobble.track); + try scrobble_view.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_track) |artist| { + try artists.append(artist); } - // Note sure why this works for ZMPL, but not for jobs. - try scrobbles_view.append(formatted_scrobble); + try scrobble_view.put("date", formatted_scrobble.date); + + var scrobble_data = try scrobbles_data.append(.object); + var artists_album = try scrobble_data.put("artists_album", .array); + var artists_track = try scrobble_data.put("artists_track", .array); + + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_album) |artist| { + try artists_album.append(artist); + } + + for (formatted_scrobble.artists_track) |artist| { + try artists_track.append(artist); + } + try scrobble_data.put("date", formatted_scrobble.date); } }, 1 => { - const content: []ScrobbleTypes.SpotifyScrobble = try std.json.parseFromSliceLeaky([]ScrobbleTypes.SpotifyScrobble, request.allocator, file.content, .{}); + const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else (try zeit.instant(.{})).time(); const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); appends: for (content) |scrobble| { - // A LastFM Scrobble occurs when half a song has been played - // or the song plays for 4 minutes, whichever happens first. - // Spotify considers a song played if it plays for 30 seconds. - // Ideally, I would go with the LastFM convention, but Spotify - // history data only gives us so much information. I'm okay - // with the 30 second convention, but eventually I would prefer - // to get the song length from MusicBrainz and check if it meets - // the requirement. Until then, if it goes 30 seconds, or the - // reason_end field reads "trackdone", then it counts as a Scrobble. - // May consider giving user control to the minimum millisecond requirement. if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { skipped_tracks += 1; continue :appends; } - // In the case where the artist is null, but there's other metadata, I could - // probably let the user edit it in themselves, although I'm not sure if that - // situation happens. if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { skipped_tracks += 1; continue :appends; @@ -96,19 +107,42 @@ pub fn post(request: *jetzig.Request) !jetzig.View { continue :appends; } - // Turn SpotifyScrobble into a LastFM scrobble - const pre_formatted_scrobble: ScrobbleTypes.LastFMScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const pre_formatted_scrobble: Data.ImportedScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const formatted_scrobble = if (rule_list) |rl| + rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; - const formatted_scrobble = rules.applyScrobbleRule(pre_formatted_scrobble, rule_list); + var scrobble_view = try scrobbles_view.append(.object); + var artists = try scrobble_view.put("artists", .array); - var value = try scrobbles_data.append(.object); - - // This is so unnecessary, probably useful once I start doing Spotify integration though - inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| { - try value.put(f.name, @as(f.type, @field(formatted_scrobble, f.name))); + try scrobble_view.put("track", formatted_scrobble.track); + try scrobble_view.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_track) |artist| { + try artists.append(artist); } - // Note sure why this works for ZMPL, but not for jobs. - try scrobbles_view.append(formatted_scrobble); + try scrobble_view.put("date", formatted_scrobble.date); + + var scrobble_data = try scrobbles_data.append(.object); + var artists_album = try scrobble_data.put("artists_album", .array); + var artists_track = try scrobble_data.put("artists_track", .array); + + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("album", formatted_scrobble.album); + for (formatted_scrobble.artists_album) |artist| { + try artists_album.append(artist); + } + + for (formatted_scrobble.artists_track) |artist| { + try artists_track.append(artist); + } + try scrobble_data.put("date", formatted_scrobble.date); } }, else => unreachable, diff --git a/src/apply_rule.zig b/src/apply_rule.zig index db6eb42..dd9f8bb 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -1,18 +1,32 @@ const std = @import("std"); -const Scrobble = @import("./types.zig").LastFMScrobble; const Rules = @import("./types.zig").Rules; +const Data = @import("./types.zig"); // Wrapper for containsAtLeast to make the switch below to work -fn containsAtLeastOne(haystack: []const u8, needle: []const u8) bool { +fn containsWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.containsAtLeast(u8, haystack, 1, needle); } -fn eqlDecomped(haystack: []const u8, needle: []const u8) bool { +fn eqlWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.eql(u8, haystack, needle); } -pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { - var output_scrobble: Scrobble = scrobble; +pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) Data.Scrobble { + var output_scrobble = Data.Scrobble{ + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .date = scrobble.date, + }; + + //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + //const gpalloc = gpa.allocator(); + + //var arena = std.heap.ArenaAllocator.init(gpalloc); + //defer arena.deinit(); + //const allocator = arena.allocator(); + for (rules.rules) |rule| { var match_found: bool = switch (rule.cond_req) { .any => false, @@ -20,8 +34,8 @@ pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { }; for (rule.conditionals) |cond| { const match_fn: *const fn ([]const u8, []const u8) bool = switch (cond.match_cond) { - .is => eqlDecomped, - .contains => containsAtLeastOne, + .is => eqlWrapper, + .contains => containsWrapper, }; switch (rule.cond_req) { .any => switch (cond.match_on) { @@ -35,9 +49,22 @@ pub fn applyScrobbleRule(scrobble: Scrobble, rules: Rules) Scrobble { if (match_found) { for (rule.actions) |act| { switch (act.action) { - .add => {}, + .add => { + var al = std.ArrayList([]const u8).init(allocator); + switch (act.action_on) { + .album, .track => unreachable, + inline else => |on| { + // I have decided an error won't happen :) + al.appendSlice(@field(output_scrobble, @tagName(on))) catch unreachable; + al.append(act.action_txt) catch unreachable; + const artists = al.toOwnedSlice() catch unreachable; + @field(output_scrobble, @tagName(on)) = artists; + }, + } + }, .replace => switch (act.action_on) { - inline else => |on| @field(output_scrobble, @tagName(on)) = act.action_txt, + inline .album, .track => |on| @field(output_scrobble, @tagName(on)) = act.action_txt, + inline .artists_album, .artists_track => |on| @field(output_scrobble, @tagName(on)) = &[_][]const u8{act.action_txt}, }, } } diff --git a/src/types.zig b/src/types.zig index 0e385a5..f0268ed 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,51 +1,59 @@ -pub const LastFMScrobble = struct { +pub const ImportedScrobble = struct { track: []const u8, artist: []const u8, album: []const u8 = "", date: i128, }; +pub const Scrobble = struct { + track: []const u8, + artists_track: []const []const u8, + album: []const u8 = "", + artists_album: []const []const u8, + date: i128, +}; + // From lastfmstats.com -pub const LastFM = struct { username: []const u8, scrobbles: []LastFMScrobble }; +pub const LastFM = struct { username: []const u8, scrobbles: []ImportedScrobble }; // I derived whether or not these values were optional from searching // the respective fields for null in Vim, so there may be some fields // that can be optional that I haven't run into yet pub const SpotifyScrobble = struct { ts: []const u8, - username: []const u8, - platform: []const u8, + //username: []const u8, + //platform: []const u8, ms_played: u64, - conn_country: []const u8, - ip_addr_decrypted: ?[]const u8, - user_agent_decrypted: ?[]const u8, + //conn_country: []const u8, + //ip_addr_decrypted: ?[]const u8, + //user_agent_decrypted: ?[]const u8, master_metadata_track_name: ?[]const u8, master_metadata_album_artist_name: ?[]const u8, master_metadata_album_album_name: ?[]const u8, - spotify_track_uri: ?[]const u8, - episode_name: ?[]const u8, - episode_show_name: ?[]const u8, - spotify_episode_uri: ?[]const u8, + //spotify_track_uri: ?[]const u8, + //episode_name: ?[]const u8, + //episode_show_name: ?[]const u8, + //spotify_episode_uri: ?[]const u8, reason_start: []const u8, reason_end: ?[]const u8, - shuffle: bool, + //shuffle: bool, skipped: ?bool, - offline: bool, + //offline: bool, offline_timestamp: u64, - incognito_mode: ?bool, + //incognito_mode: ?bool, }; pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, conditionals: []struct { - match_on: ScrobbleFields, + match_on: enum { artist, album, track }, match_cond: enum { is, contains }, match_txt: []const u8, }, actions: []struct { action: enum { replace, add }, - action_on: ScrobbleFields, + action_on: enum { artists_album, album, artists_track, track }, action_txt: []const u8, }, }; @@ -53,9 +61,3 @@ pub const Rule = struct { pub const Rules = struct { rules: []const Rule, }; - -pub const ScrobbleFields = enum { - artist, - album, - track, -}; From be8c1191b05e2fb8b941fc383c1b7f66b06425f4 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 24 Apr 2025 09:34:34 -0400 Subject: [PATCH 19/71] Clean --- src/app/jobs/process_scrobbles.zig | 23 ++++++----------------- src/apply_rule.zig | 15 ++++----------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index 87871c7..fc30e58 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -18,41 +18,30 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //_ = env; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { - //var buffer: [256**4]u8 = undefined; - //var fba = std.heap.FixedBufferAllocator.init(&buffer); - //const alloc = fba.allocator(); - - var alssu8 = std.ArrayList([]const u8).init(allocator); - defer alssu8.deinit(); + var track_artists_al = std.ArrayList([]const u8).init(allocator); + var album_artists_al = std.ArrayList([]const u8).init(allocator); for (item.getT(.array, "artists_track").?.items()) |artist| { - try alssu8.append(try artist.coerce([]const u8)); + try track_artists_al.append(try artist.coerce([]const u8)); } - const track_artists = try alssu8.toOwnedSlice(); - for (item.getT(.array, "artists_album").?.items()) |artist| { - try alssu8.append(try artist.coerce([]const u8)); + try album_artists_al.append(try artist.coerce([]const u8)); } - const album_artists = try alssu8.toOwnedSlice(); - const scrobble: Data.Scrobble = .{ .track = item.getT(.string, "track").?, - .artists_track = track_artists, + .artists_track = track_artists_al.items, .album = item.getT(.string, "album") orelse "", - .artists_album = album_artists, + .artists_album = album_artists_al.items, .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), }; var id_prehash = std.ArrayList(u8).init(allocator); - defer id_prehash.deinit(); var alartist = std.ArrayList(struct { name: []const u8, id: i32 }).init(allocator); - defer alartist.deinit(); for (scrobble.artists_track) |artist| { - //try id_prehash.appendSlice(artist); try alartist.append(.{ .name = artist, .id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist))) }); } //const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); diff --git a/src/apply_rule.zig b/src/apply_rule.zig index dd9f8bb..c63ad4a 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -20,13 +20,6 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc .date = scrobble.date, }; - //var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - //const gpalloc = gpa.allocator(); - - //var arena = std.heap.ArenaAllocator.init(gpalloc); - //defer arena.deinit(); - //const allocator = arena.allocator(); - for (rules.rules) |rule| { var match_found: bool = switch (rule.cond_req) { .any => false, @@ -57,9 +50,11 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc // I have decided an error won't happen :) al.appendSlice(@field(output_scrobble, @tagName(on))) catch unreachable; al.append(act.action_txt) catch unreachable; - const artists = al.toOwnedSlice() catch unreachable; - @field(output_scrobble, @tagName(on)) = artists; + @field(output_scrobble, @tagName(on)) = al.items; }, + //else => { + // std.log.debug("Adding artists doesn't work yet", .{}); + //}, } }, .replace => switch (act.action_on) { @@ -73,5 +68,3 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc return output_scrobble; } - -//pub fn applyAlbumRule() !Album {} From 9df8f9ea12f14bced8fc220fb6f4827e6cce39e4 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 10:41:42 -0400 Subject: [PATCH 20/71] Fix segfault in applyScrobbleRule Thanks bob :) --- src/app/jobs/process_rule.zig | 1 - src/app/views/upload.zig | 16 +++++++++++----- src/apply_rule.zig | 16 +++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 217826a..ef0fc09 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -32,7 +32,6 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig }; var rules = std.ArrayList(Data.Rule).init(allocator); - defer rules.deinit(); const file_content = try file_read.readToEndAlloc(allocator, 16_000_000); file_read.close(); diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 98f0009..59442a8 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -38,8 +38,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { }); defer rule_file.close(); - const file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, file_content, .{}) catch null; + const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); + const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; switch (source) { 0 => { @@ -51,7 +51,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; const formatted_scrobble = if (rule_list) |rl| - rules.applyScrobbleRule(request.allocator, scrobble, rl) + try rules.applyScrobbleRule(request.allocator, scrobble, rl) else Data.Scrobble{ .album = scrobble.album, @@ -107,9 +107,15 @@ pub fn post(request: *jetzig.Request) !jetzig.View { continue :appends; } - const pre_formatted_scrobble: Data.ImportedScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 }; + const pre_formatted_scrobble: Data.ImportedScrobble = .{ + .track = scrobble.master_metadata_track_name.?, + .album = scrobble.master_metadata_album_album_name.?, + .artist = scrobble.master_metadata_album_artist_name.?, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000, + }; + const formatted_scrobble = if (rule_list) |rl| - rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) else Data.Scrobble{ .album = pre_formatted_scrobble.album, diff --git a/src/apply_rule.zig b/src/apply_rule.zig index c63ad4a..6c03b38 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -11,12 +11,14 @@ fn eqlWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.eql(u8, haystack, needle); } -pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) Data.Scrobble { +pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) !Data.Scrobble { + const artists = try allocator.alloc([]const u8, 1); + artists[0] = scrobble.artist; var output_scrobble = Data.Scrobble{ .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, + .artists_track = artists, .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, + .artists_album = artists, .date = scrobble.date, }; @@ -47,10 +49,10 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc switch (act.action_on) { .album, .track => unreachable, inline else => |on| { - // I have decided an error won't happen :) - al.appendSlice(@field(output_scrobble, @tagName(on))) catch unreachable; - al.append(act.action_txt) catch unreachable; - @field(output_scrobble, @tagName(on)) = al.items; + try al.appendSlice(@field(output_scrobble, @tagName(on))); + try al.append(act.action_txt); + const list = try al.toOwnedSlice(); + @field(output_scrobble, @tagName(on)) = list; }, //else => { // std.log.debug("Adding artists doesn't work yet", .{}); From 5e58e81ca7ef162c1af4ceed412bd124a3948809 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 14:28:39 -0400 Subject: [PATCH 21/71] Fix album artist parsing in process_scrobbles --- src/app/jobs/process_rule.zig | 3 - src/app/jobs/process_scrobbles.zig | 189 +++++++++-------------------- 2 files changed, 60 insertions(+), 132 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index ef0fc09..7b60d54 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -4,9 +4,6 @@ const Data = @import("../../types.zig"); pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { _ = env; - //_ = params; - - std.log.debug("{s}", .{try params.toJson()}); const rule = try std.json.parseFromSliceLeaky(Data.Rule, allocator, try params.toJson(), .{ .ignore_unknown_fields = true }); diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index fc30e58..ae3cfa8 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -18,179 +18,110 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //_ = env; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { - var track_artists_al = std.ArrayList([]const u8).init(allocator); - var album_artists_al = std.ArrayList([]const u8).init(allocator); + const track_artist_count = item.getT(.array, "artists_track").?.count(); + const album_artist_count = item.getT(.array, "artists_album").?.count(); + var track_artist_name_buffer = try allocator.alloc([]const u8, track_artist_count); + var album_artist_name_buffer = try allocator.alloc([]const u8, album_artist_count); + var track_artist_id_buffer = try allocator.alloc(i32, track_artist_count); + var album_artist_id_buffer = try allocator.alloc(i32, album_artist_count); - for (item.getT(.array, "artists_track").?.items()) |artist| { - try track_artists_al.append(try artist.coerce([]const u8)); + for (item.getT(.array, "artists_track").?.items(), 0..track_artist_count) |artist, i| { + const artist_name = try artist.coerce([]const u8); + track_artist_name_buffer[i] = artist_name; + track_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); } - for (item.getT(.array, "artists_album").?.items()) |artist| { - try album_artists_al.append(try artist.coerce([]const u8)); + for (item.getT(.array, "artists_album").?.items(), 0..album_artist_count) |artist, i| { + const artist_name = try artist.coerce([]const u8); + album_artist_name_buffer[i] = artist_name; + album_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); } const scrobble: Data.Scrobble = .{ .track = item.getT(.string, "track").?, - .artists_track = track_artists_al.items, + .artists_track = track_artist_name_buffer, .album = item.getT(.string, "album") orelse "", - .artists_album = album_artists_al.items, + .artists_album = album_artist_name_buffer, .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), }; + // Probably want to include artist name here, but not sure how to yet var id_prehash = std.ArrayList(u8).init(allocator); - var alartist = std.ArrayList(struct { name: []const u8, id: i32 }).init(allocator); - - for (scrobble.artists_track) |artist| { - try alartist.append(.{ .name = artist, .id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist))) }); - } - //const artist_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); try id_prehash.appendSlice(scrobble.album); const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); try id_prehash.appendSlice(scrobble.track); const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); - // Make IDs - // Song: Song hash XOR artist hash XOR album hash - // This way, if two songs share a name, then - // the IDs also depend on the hash of the album - // they're on, as well as the artist name. As far - // as I can tell, this is only as issue for Sufjan - // Steven's `Songs for Christmas`. (In practice. - // In reality, there are albums with several untitled - // songs (Selected Ambient Works Vol. II by Aphex Twin, - // ( ) by Sigur Ros, ...) that have working titles - // in their place.) - - // Album: If the album is not self-titled, then - // album hash XOR artist hash. This way, if two - // artists have an album of the same name, then - // the IDs also depend on the hash of the artist - // name. As far as I can tell, this is only an - // issue for Weezer and Peter Gabriel, but their - // albums go by unique names anyways. - - // Artist: Artist hash. If two artists have the same name, - // then a disambiguating string can be provided to - // differentiate after the fact, or in a rule. - - //var album_id: i32 = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(formed))); - //const song_id = (song_hash ^ artist_hash ^ album_hash); var albumsong = try jetzig.database.Query(.Albumsong) - .findBy(.{ - .album_id = album_id, - .song_id = song_id, - }) - .select(.{.id}) - .execute(env.repo); + .findBy(.{ .album_id = album_id, .song_id = song_id }) + .select(.{.id}).execute(env.repo); var ins_album = try jetzig.database.Query(.Album) .find(album_id) - .select(.{.id}) - .execute(env.repo); + .select(.{.id}).execute(env.repo); - for (alartist.items) |artist| { + for (track_artist_name_buffer, track_artist_id_buffer) |artist_name, artist_id| { var ins_artist = try jetzig.database.Query(.Artist) - .find(artist.id) - .select(.{.id}) - .execute(env.repo); + .find(artist_id) + .select(.{.id}).execute(env.repo); - if (ins_artist == null) ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ - .id = artist.id, - .name = artist.name, - .disambiguation = null, - }) - .returning(.{.id}) - .execute(env.repo); + if (ins_artist == null) + ins_artist = try jetzig.database.Query(.Artist) + .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + .returning(.{.id}).execute(env.repo); if (albumsong == null) { var ins_song = try jetzig.database.Query(.Song) .find(song_id) - .select(.{.id}) - .execute(env.repo); + .select(.{.id}).execute(env.repo); - if (ins_song == null) ins_song = try jetzig.database.Query(.Song) - .insert(.{ - .id = song_id, - .name = scrobble.track, - .length = null, - .hidden = false, - }) - .returning(.{.id}) - .execute(env.repo); + if (ins_song == null) + ins_song = try jetzig.database.Query(.Song) + .insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }) + .returning(.{.id}).execute(env.repo); - if (ins_album == null) { + if (ins_album == null) ins_album = try jetzig.database.Query(.Album) - .insert(.{ - .id = album_id, - .name = scrobble.album, - .length = null, - }) - .returning(.{.id}) - .execute(env.repo); - // I think there's still technically a bug here when you have a different artist but I'm not sure - try jetzig.database.Query(.Artistalbum) - .insert(.{ - .artist_id = ins_artist.?.id, - .album_id = ins_album.?.id, - }) - .execute(env.repo); - } + .insert(.{ .id = album_id, .name = scrobble.album, .length = null }) + .returning(.{.id}).execute(env.repo); albumsong = try jetzig.database.Query(.Albumsong) - .insert(.{ - .song_id = ins_song.?.id, - .album_id = ins_album.?.id, - }) - .returning(.{.id}) - .execute(env.repo); + .insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }) + .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Albumsongsartist) - .insert(.{ - .albumsong_id = albumsong.?.id, - .artist_id = ins_artist.?.id, - }) - .execute(env.repo); + .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } else { const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) - .findBy(.{ - .albumsong_id = albumsong.?.id, - .artist_id = ins_artist.?.id, - }) - .select(.{.id}) - .execute(env.repo); + .findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }) + .select(.{.id}).execute(env.repo); - if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist) - .insert(.{ - .albumsong_id = albumsong.?.id, - .artist_id = ins_artist.?.id, - }) - .execute(env.repo); + if (ins_albumsongartist == null) + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + } + } + for (album_artist_name_buffer, album_artist_id_buffer) |artist_name, artist_id| { + const ins_artistalbum = try jetzig.database.Query(.Artistalbum) + .findBy(.{ .album_id = ins_album.?.id, .artist_id = artist_id }) + .select(.{.id}).execute(env.repo); - const ins_artistalbum = try jetzig.database.Query(.Artistalbum) - .findBy(.{ - .album_id = ins_album.?.id, - .artist_id = ins_artist.?.id, - }) - .select(.{.id}) - .execute(env.repo); - - if (ins_artistalbum == null) try jetzig.database.Query(.Artistalbum) - .insert(.{ - .album_id = ins_album.?.id, - .artist_id = ins_artist.?.id, - }) - .execute(env.repo); + if (ins_artistalbum == null) { + var ins_artist = try jetzig.database.Query(.Artist) + .find(artist_id) + .select(.{.id}).execute(env.repo); + if (ins_artist == null) + ins_artist = try jetzig.database.Query(.Artist) + .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + .returning(.{.id}).execute(env.repo); + try jetzig.database.Query(.Artistalbum) + .insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); } } try jetzig.database.Query(.Scrobble) - .insert(.{ - .albumsong = albumsong.?.id, - .datetime = scrobble.date, - }) - .execute(env.repo); + .insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); } } } From 18d4df0a5c2d404e9ffe2752e9b74aff5948c6be Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 15:48:47 -0400 Subject: [PATCH 22/71] Fix albums not being hashed correctly Also provides more actions for rules, but they don't seem to work... --- src/app/jobs/process_scrobbles.zig | 25 ++++++++++--------- src/app/views/rules.zig | 17 +++++++++---- src/app/views/rules/index.zmpl | 39 +++++++++++++----------------- 3 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index ae3cfa8..b2ae22e 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -18,6 +18,9 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig //_ = env; if (params.getT(.array, "scrobbles")) |scrobbles| { for (scrobbles.items()) |item| { + + // Probably want to include artist name here, but not sure how to yet + const track_artist_count = item.getT(.array, "artists_track").?.count(); const album_artist_count = item.getT(.array, "artists_album").?.count(); var track_artist_name_buffer = try allocator.alloc([]const u8, track_artist_count); @@ -25,6 +28,16 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig var track_artist_id_buffer = try allocator.alloc(i32, track_artist_count); var album_artist_id_buffer = try allocator.alloc(i32, album_artist_count); + const scrobble: Data.Scrobble = .{ + .track = item.getT(.string, "track").?, + .artists_track = track_artist_name_buffer, + .album = item.getT(.string, "album") orelse "", + .artists_album = album_artist_name_buffer, + .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), + }; + + var id_prehash = std.ArrayList(u8).init(allocator); + for (item.getT(.array, "artists_track").?.items(), 0..track_artist_count) |artist, i| { const artist_name = try artist.coerce([]const u8); track_artist_name_buffer[i] = artist_name; @@ -35,19 +48,9 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; album_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + try id_prehash.appendSlice(artist_name); } - const scrobble: Data.Scrobble = .{ - .track = item.getT(.string, "track").?, - .artists_track = track_artist_name_buffer, - .album = item.getT(.string, "album") orelse "", - .artists_album = album_artist_name_buffer, - .date = @as(u64, @bitCast(@as(i64, @truncate(item.getT(.integer, "date").? * 1000)))), - }; - - // Probably want to include artist name here, but not sure how to yet - var id_prehash = std.ArrayList(u8).init(allocator); - try id_prehash.appendSlice(scrobble.album); const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); try id_prehash.appendSlice(scrobble.track); diff --git a/src/app/views/rules.zig b/src/app/views/rules.zig index 957f5d2..7a2df91 100644 --- a/src/app/views/rules.zig +++ b/src/app/views/rules.zig @@ -22,6 +22,8 @@ pub fn edit(id: []const u8, request: *jetzig.Request) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View { const params = try request.params(); + std.log.debug("{s}", .{try params.toJson()}); + var job = try request.job("process_rule"); _ = try job.params.put("name", params.get("rule-title")); @@ -30,6 +32,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var conditionals = try job.params.put("conditionals", .array); inline for (0..5) |i| { if (!std.mem.eql(u8, "", params.getT(.string, comptime std.fmt.comptimePrint("match-txt{}", .{i})).?)) { + //if (params.getT(.string, comptime std.fmt.comptimePrint("match-txt{}", .{i})) != null) { var cond = try conditionals.append(.object); try cond.put("match_on", params.get(comptime std.fmt.comptimePrint("match-on{}", .{i}))); try cond.put("match_cond", params.get(comptime std.fmt.comptimePrint("match-cond{}", .{i}))); @@ -38,11 +41,15 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } var actions = try job.params.put("actions", .array); - var act0 = try actions.append(.object); - try act0.put("action", params.get("action")); - try act0.put("action_on", params.get("action-on")); - try act0.put("action_txt", params.get("action-txt")); - + inline for (0..5) |i| { + if (!std.mem.eql(u8, "", params.getT(.string, comptime std.fmt.comptimePrint("action-txt{}", .{i})).?)) { + //if (params.getT(.string, comptime std.fmt.comptimePrint("action-txt{}", .{i})) != null) { + var act = try actions.append(.object); + try act.put("action", params.get(comptime std.fmt.comptimePrint("action{}", .{i}))); + try act.put("action_on", params.get(comptime std.fmt.comptimePrint("action-on{}", .{i}))); + try act.put("action_txt", params.get(comptime std.fmt.comptimePrint("action-txt{}", .{i}))); + } + } try job.schedule(); return request.render(.created); diff --git a/src/app/views/rules/index.zmpl b/src/app/views/rules/index.zmpl index f97ef35..5661f97 100644 --- a/src/app/views/rules/index.zmpl +++ b/src/app/views/rules/index.zmpl @@ -14,8 +14,8 @@ Add a rule below.
Match conditonals.
@@ -38,27 +38,22 @@ If
} - - - - -
then - - -with - +@for (0..5) |i| { + + + with + +
+} From 01fe10f045909597e892b828dec06135bba3b8e1 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 16:27:03 -0400 Subject: [PATCH 23/71] Fix limit on rule parameters and fix segfault in applyScrobbleRule For sure this time --- src/apply_rule.zig | 6 +++++- src/main.zig | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 6c03b38..4f99ca9 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -61,7 +61,11 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc }, .replace => switch (act.action_on) { inline .album, .track => |on| @field(output_scrobble, @tagName(on)) = act.action_txt, - inline .artists_album, .artists_track => |on| @field(output_scrobble, @tagName(on)) = &[_][]const u8{act.action_txt}, + inline .artists_album, .artists_track => |on| { + const artist = try allocator.alloc([]const u8, 1); + artist[0] = act.action_txt; + @field(output_scrobble, @tagName(on)) = artist; + }, }, } } diff --git a/src/main.zig b/src/main.zig index 1cab920..cdc9bca 100644 --- a/src/main.zig +++ b/src/main.zig @@ -14,10 +14,13 @@ pub const jetzig_options = struct { // htmx middleware skips layouts when `HX-Target` header is present and issues // `HX-Redirect` instead of a regular HTTP redirect when `request.redirect` is called. jetzig.middleware.HtmxMiddleware, - // Demo middleware included with new projects. Remove once you are familiar with Jetzig's - // middleware system. + // Demo middleware included with new projects. Remove once you are familiar with Jetzig's + // middleware system. }; + // This is currently the largest number of parameters one can have in a rule + pub const max_multipart_form_fields = 42; + // Maximum bytes to allow in request body. pub const max_bytes_request_body: usize = std.math.pow(usize, 2, 24); From 65136a44d652dbda40b76597ddc59ee34dc83a1a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 27 Apr 2025 23:58:50 -0400 Subject: [PATCH 24/71] Add more information to artists view, songs view, and format dates correctly in scrobbles view --- README.md | 5 +++- src/app/views/albums.zig | 1 + src/app/views/artists.zig | 47 +++++++++++++++--------------- src/app/views/artists/get.zmpl | 17 +++++++++++ src/app/views/scrobbles.zig | 5 +++- src/app/views/scrobbles/index.zmpl | 2 +- src/app/views/songs.zig | 30 +++++++++++++++---- src/app/views/songs/index.zmpl | 6 ++++ src/apply_rule.zig | 3 -- 9 files changed, 82 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index b62b576..f18e7e0 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,14 @@ Licensed under MIT. - [ ] List all songs on artist page, with respective album - [x] List all albums on artist page - [x] Include number of plays for each + - [x] List albums features on - [x] See all albums under "/albums" - [x] See all songs from album - [x] Include number of plays + - [x] Include name of artist(s) + - [ ] Include artists features on each song - [x] See all songs under "/songs" - - [ ] Include respective artist(s) + - [x] Include respective artist(s) - [ ] Include respective album[^10] - [x] Include number of plays - [ ] Create disambiguation pages diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 57d2bb1..cc6d765 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -33,6 +33,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { continue :blk; } var album_view = try albums_view.append(.object); + var artist_infos = try album_view.put("artist_info", .array); var artist_info = try artist_infos.append(.object); try artist_info.put("name", album.artist_name); diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index b8d6bc3..9f707f0 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -35,6 +35,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var albums_view = try root.put("albums", .array); + var appears = try root.put("appears", .array); const artist_name = try jetzig.database.Query(.Artist).find(id).select(.{ .id, .name }).execute(request.repo); _ = try root.put("artist", artist_name.?.name); const query = @@ -48,6 +49,18 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { \\ORDER BY scrobbles DESC ; + const appears_query = + \\SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 AND artistalbums.artist_id != $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC; + ; + var albums_jq_result = try request.repo.executeSql(query, .{id}); defer albums_jq_result.deinit(); @@ -60,30 +73,18 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { try album_view.put("url", album.id); try album_view.put("scrobbles", album.scrobbles); } - //const artist = try jetzig.database.Query(.Artist) - // .find(id) - // .select(.{ .id, .name }) - // .execute(request.repo); - //var root = try request.data(.object); - //try root.put("artist", artist.?.name); - //var albums_view = try root.put("albums", .array); - //const query = jetzig.database.Query(.Albumartist) - // .select(.{.id}) - // .include(.album, .{ .select = .{ .name, .id } }) - // .join(.inner, .artist) - // .where(.{ .artist = .{ .id = id } }); - //const albums = try request.repo.all(query); - //for (albums) |album| { - // const scrobbles = try jetzig.database.Query(.Scrobble) - // .where(.{ .album_id = album.album.id }) - // .count() - // .execute(request.repo); - // var album_view = try albums_view.append(.object); - // try album_view.put("name", album.album.name); - // try album_view.put("url", album.album.id); - // try album_view.put("scrobbles", scrobbles); - //} + var appears_jq_result = try request.repo.executeSql(appears_query, .{id}); + defer appears_jq_result.deinit(); + + while (try appears_jq_result.postgresql.result.next()) |album_row| { + const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); + var album_view = try appears.append(.object); + try album_view.put("name", album.name); + try album_view.put("url", album.id); + try album_view.put("scrobbles", album.scrobbles); + } + return request.render(.ok); } diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 911d2c3..2cc50cd 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -6,6 +6,7 @@ @partial partials/header

{{.artist}}

+

Albums

@@ -21,6 +22,22 @@ }
+

Albums Featured On

+ + + + + + + + @for (.appears) |album| { + + + + + } + +
NameScrobbles
{{album.name}}{{album.scrobbles}}
- +@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) \ No newline at end of file diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 4164fc1..34cea3e 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -138,19 +138,23 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { defer first_last_songs_jq_result.deinit(); // These look backwards, but it's correct this way + var last_song_view = try root.put("last", .object); + if (try first_last_songs_jq_result.postgresql.result.next()) |last_song_row| { const last_song = try last_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try root.put("last_song_name", last_song.name); - try root.put("last_song_id", last_song.id); - try root.put("last_song_date", (try dateFmt(request.allocator, last_song.date))); - } + try last_song_view.put("name", last_song.name); + try last_song_view.put("id", last_song.id); + try last_song_view.put("date", (try dateFmt(request.allocator, last_song.date))); + } else unreachable; + + var first_song_view = try root.put("first", .object); if (try first_last_songs_jq_result.postgresql.result.next()) |first_song_row| { const first_song = try first_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try root.put("first_song_name", first_song.name); - try root.put("first_song_id", first_song.id); - try root.put("first_song_date", (try dateFmt(request.allocator, first_song.date))); - } + try first_song_view.put("name", first_song.name); + try first_song_view.put("id", first_song.id); + try first_song_view.put("date", (try dateFmt(request.allocator, first_song.date))); + } else unreachable; try first_last_songs_jq_result.drain(); diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index adf6158..48c1c25 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -1,53 +1,22 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.album, .scrobbles}; +} + - @partial partials/header

{{.artist.name}}

-{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place) -
-First listen: {{.first_song_name}} ({{.first_song_date}}) -
-Most recent listen: {{.last_song_name}} ({{.last_song_date}}) +@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, last_song: .last, first_song: .first) +

Albums

-
- - - - - - -@for (.albums) |album| { - - - - -} - -
NameScrobbles
{{album.name}}{{album.scrobbles}}
+@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) +

Albums Featured On

- - - - - - - - @for (.appears) |album| { - - - - - } - -
NameScrobbles
{{album.name}}{{album.scrobbles}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .appears, columns: columns) + \ No newline at end of file diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl new file mode 100644 index 0000000..322a0e8 --- /dev/null +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -0,0 +1,9 @@ +@args scrobbles: i64, rank: []const u8, last_song: *ZmplValue, first_song: *ZmplValue + +
+{{scrobbles}} scrobbles ({{rank}} place) +
+First listen: {{first_song.name}} ({{first_song.date}}) +
+Most recent listen: {{last_song.name}} ({{last_song.date}}) +
\ No newline at end of file diff --git a/src/app/views/partials/_newtable.zmpl b/src/app/views/partials/_newtable.zmpl index bee43c1..8960e21 100644 --- a/src/app/views/partials/_newtable.zmpl +++ b/src/app/views/partials/_newtable.zmpl @@ -1,18 +1,29 @@ -@args table_data *ZmplValue, table_headers: []enum{Song, Album, Artist, Scrobbles, Date} +@args T: type, table_data: *ZmplValue, columns: T
@zig { - for (table_headers) |header| { + for (columns) |header| { switch (header) { - .Artist => { + .song => { + + }, + .album => { + + }, + .artist => { + + }, + .artistlist => { }, - inline else => |other| { - const h = @tagName(other); - + .scrobbles => { + }, + .date => { + + } } } } @@ -20,35 +31,38 @@ @zig { - for (table_data) |data| { + const array = table_data.items(.array); + for (array) |ent| { - for (table_header) |header| { + for (columns) |header| { switch (header) { - .Song => { + .song, .album, .artist => { + const path = switch (header) { + .song => "songs", + .album => "albums", + .artist => "artists", + else => unreachable + }; }, - .Album => { + .artistlist => { - }, - .Artist => { - }, - .Scrobbles => { - + .scrobbles => { + }, - .Date =>{ - + .date =>{ + } - }; + } } + } } diff --git a/src/types.zig b/src/types.zig index 2574610..455f282 100644 --- a/src/types.zig +++ b/src/types.zig @@ -62,10 +62,13 @@ pub const Rules = struct { rules: []const Rule, }; -pub const Headers = []enum { - Song, - Album, - Artist, - Scrobbles, - Date, -}; +// Can't import types in .zmpl files, so defining this here +// doesn't really do much (except maybe in the .zig file for views?) +//pub const HeaderTypes = []enum { +// song, +// album, +// artist, +// artistlist, +// scrobbles, +// date, +//}; From 365b9dbf11345af987960b92d58148769ccdbe43 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 13 May 2025 14:24:14 -0400 Subject: [PATCH 36/71] Switch to using newtable partial for all tables Will be renamed eventually, don't care right now. Also cleans up a lot of code I wasn't particularly happy about --- build.zig.zon | 4 +-- src/app/views/albums.zig | 42 ++++++++++++++------------- src/app/views/artists.zig | 25 ++++++++++++---- src/app/views/artists/index.zmpl | 31 ++++---------------- src/app/views/partials/_newtable.zmpl | 28 ++++++++++-------- src/app/views/scrobbles.zig | 41 +++++++++++++------------- src/app/views/scrobbles/index.zmpl | 39 ++++--------------------- src/app/views/songs.zig | 32 ++++++++++---------- src/app/views/songs/index.zmpl | 37 ++++------------------- src/types.zig | 15 ++++++++++ 10 files changed, 128 insertions(+), 166 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index f2e5778..d7fb785 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -17,8 +17,8 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/a298192bb0cddf9c45d7d0d976a9852804457de2.tar.gz", - .hash = "jetzig-0.0.0-IpAgLf5aDwB4UOKMhIjtK22zBsPfbWEwCYgHT0hayyT-", + .url = "https://github.com/jetzig-framework/jetzig/archive/7be1d137fcab5c422e05f12092f6e04a02900d6f.tar.gz", + .hash = "jetzig-0.0.0-IpAgLURbDwB1NlywUH7lnQ3zptNvSQWVosaA1k7l1cNz", }, .zeit = .{ .url = "https://github.com/rockorager/zeit/archive/refs/tags/v0.6.0.tar.gz", diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 1264dac..95f5b86 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -1,6 +1,8 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; +const TableRow = @import("../../types.zig").TableRows; +const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -22,28 +24,26 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; var prev_album_id: ?i32 = null; - var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; + + var row: ?TableRow = null; + var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); if (album.id == prev_album_id) { - var artist_info = try prev_artist_infos.?.append(.object); - try artist_info.put("name", album.artist_name); - try artist_info.put("id", album.artist_id); + try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); continue :blk; + } else { + try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); + + row = TableRow{ + .album = .{ .name = album.name, .id = album.id }, + .artistlist = try artistlist.toOwnedSlice(), + .scrobbles = album.scrobbles, + }; + + try albums_view.append(row); } - var album_view = try albums_view.append(.object); - - try album_view.put("name", album.name); - try album_view.put("id", album.id); - try album_view.put("scrobbles", album.scrobbles); - - var artist_infos = try album_view.put("artist_info", .array); - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", album.artist_name); - try artist_info.put("id", album.artist_id); - - prev_artist_infos = artist_infos; prev_album_id = album.id; } return request.render(.ok); @@ -72,10 +72,12 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { while (try songs_js_result.postgresql.result.next()) |song_row| { const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); - var song_view = try songs_view.append(.object); - try song_view.put("name", song.name); - try song_view.put("id", song.id); - try song_view.put("scrobbles", song.scrobbles); + const row = TableRow{ + .song = .{ .name = song.name, .id = song.id }, + .scrobbles = song.scrobbles, + }; + + try songs_view.append(row); } return request.render(.ok); } diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 34cea3e..461235a 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -1,6 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; +const TableRow = @import("../../types.zig").TableRows; const dateFmt = @import("../../date_fmt.zig").dateFmt; const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; @@ -25,10 +26,12 @@ pub fn index(request: *jetzig.Request) !jetzig.View { while (try artists_jq_result.postgresql.result.next()) |artist_row| { const artist = try artist_row.to(Artist, .{ .dupe = true, .allocator = request.allocator }); - var artist_view = try artists_view.append(.object); - try artist_view.put("name", artist.name); - try artist_view.put("url", artist.id); - try artist_view.put("scrobbles", artist.scrobbles); + const row = TableRow{ + .artist = .{ .name = artist.name, .id = artist.id }, + .scrobbles = artist.scrobbles, + }; + + try artists_view.append(row); } return request.render(.ok); @@ -84,7 +87,12 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { while (try albums_jq_result.postgresql.result.next()) |album_row| { const album = try album_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - try albums_view.append(album); + const album_table_row = TableRow{ + .album = .{ .name = album.name, .id = album.id }, + .scrobbles = album.scrobbles, + }; + + try albums_view.append(album_table_row); } //albums_jq_result.drain(); @@ -107,7 +115,12 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { while (try appears_jq_result.postgresql.result.next()) |appears_row| { const appears = try appears_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - try appears_view.append(appears); + const appears_table_row = TableRow{ + .album = .{ .name = appears.name, .id = appears.id }, + .scrobbles = appears.scrobbles, + }; + + try appears_view.append(appears_table_row); } //appears_jq_result.drain(); diff --git a/src/app/views/artists/index.zmpl b/src/app/views/artists/index.zmpl index 6854e07..0648c91 100644 --- a/src/app/views/artists/index.zmpl +++ b/src/app/views/artists/index.zmpl @@ -1,34 +1,15 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.artist, .scrobbles}; +} + - @partial partials/header

Artists

-
SongAlbumArtistArtist(s){{h}}ScrobblesDate
- {{data.name}} + {{ent.name}} - {{data.name}} - - @for (data.get("artist_info").?) |ai| { - {{ai.name}} + @for (ent.get("artist_info").?) |artist| { + {{artist.name}} } {{data.scrobbles}}{{ent.scrobbles}}{{data.date}}{{ent.date}}
- - - - - - - -@for (.artists) |artist| { - - - - -} - -
NameScrobbles
{{artist.name}}{{artist.scrobbles}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .artists, columns: columns) \ No newline at end of file diff --git a/src/app/views/partials/_newtable.zmpl b/src/app/views/partials/_newtable.zmpl index 8960e21..6746a2a 100644 --- a/src/app/views/partials/_newtable.zmpl +++ b/src/app/views/partials/_newtable.zmpl @@ -32,33 +32,37 @@ @zig { const array = table_data.items(.array); - for (array) |ent| { + for (array) |row| { for (columns) |header| { switch (header) { - .song, .album, .artist => { - const path = switch (header) { - .song => "songs", - .album => "albums", - .artist => "artists", - else => unreachable - }; + .song => { - {{ent.name}} + {{row.song.name}} + + }, + .album => { + + {{row.album.name}} + + }, + .artist => { + + {{row.artist.name}} }, .artistlist => { - @for (ent.get("artist_info").?) |artist| { + @for (row.get("artistlist").?) |artist| { {{artist.name}} } }, .scrobbles => { - {{ent.scrobbles}} + {{row.scrobbles}} }, .date =>{ - {{ent.date}} + {{row.date}} } } } diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 1386a83..9403fa9 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -1,6 +1,8 @@ const std = @import("std"); const jetzig = @import("jetzig"); const zeit = @import("zeit"); +const TableRow = @import("../../types.zig").TableRows; +const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -23,33 +25,30 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Scrobble = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; var prev_s_id: ?i32 = null; - var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; + + var row: ?TableRow = null; + var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); blk: while (try scrobbles_jq_result.postgresql.result.next()) |scrobble_row| { const scrobble = try scrobble_row.to(Scrobble, .{ .dupe = true, .allocator = request.allocator }); if (scrobble.s_id == prev_s_id) { - var artist_info = try prev_artist_infos.?.append(.object); - try artist_info.put("name", scrobble.artist_name); - try artist_info.put("url", scrobble.artist_id); + try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); continue :blk; + } else { + var date = std.ArrayList(u8).init(request.allocator); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(scrobble.date, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + + try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); + + row = TableRow{ + .song = .{ .name = scrobble.song_name, .id = scrobble.song_id }, + .album = .{ .name = scrobble.album_name, .id = scrobble.album_id }, + .artistlist = try artistlist.toOwnedSlice(), + .date = date.items, + }; + + try scrobbles_view.append(row); } - // Not appending the scrobble directly because we don't want the unix timestamp or scrobble id - var scrobble_view = try scrobbles_view.append(.object); - - var artist_infos = try scrobble_view.put("artist_info", .array); - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", scrobble.artist_name); - try artist_info.put("url", scrobble.artist_id); - - try scrobble_view.put("song_name", scrobble.song_name); - try scrobble_view.put("song_id", scrobble.song_id); - try scrobble_view.put("album_name", scrobble.album_name); - try scrobble_view.put("album_id", scrobble.album_id); - var date = std.ArrayList(u8).init(request.allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(scrobble.date, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); - try scrobble_view.put("date", date.items); - - prev_artist_infos = artist_infos; prev_s_id = scrobble.s_id; } diff --git a/src/app/views/scrobbles/index.zmpl b/src/app/views/scrobbles/index.zmpl index b298cb6..c3d6759 100644 --- a/src/app/views/scrobbles/index.zmpl +++ b/src/app/views/scrobbles/index.zmpl @@ -1,42 +1,15 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; +} + - @partial partials/header

Scrobbles

- - - - - - - - - - -@for (.scrobbles) |scrobble| { - - - - - - -} - -
SongArtist(s)AlbumDate
{{scrobble.song_name}} - @for (scrobble.get("artist_info").?) |ai| { - {{ai.name}} - } - {{scrobble.album_name}}{{scrobble.date}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) \ No newline at end of file diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index cd220f8..5302906 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -1,5 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); +const TableRow = @import("../../types.zig").TableRows; +const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -22,28 +24,26 @@ pub fn index(request: *jetzig.Request) !jetzig.View { const Song = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; var prev_song_id: ?i32 = null; - var prev_artist_infos: ?*jetzig.zmpl.Data.Value = null; + + var row: ?TableRow = null; + var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); blk: while (try songs_js_result.postgresql.result.next()) |song_row| { const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); if (song.id == prev_song_id) { - var artist_info = try prev_artist_infos.?.append(.object); - try artist_info.put("name", song.artist_name); - try artist_info.put("url", song.artist_id); + try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); continue :blk; + } else { + try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); + + row = TableRow{ + .song = .{ .name = song.name, .id = song.id }, + .artistlist = try artistlist.toOwnedSlice(), + .scrobbles = song.scrobbles, + }; + + try songs_view.append(row); } - var song_view = try songs_view.append(.object); - - var artist_infos = try song_view.put("artist_info", .array); - var artist_info = try artist_infos.append(.object); - try artist_info.put("name", song.artist_name); - try artist_info.put("url", song.artist_id); - - try song_view.put("name", song.name); - try song_view.put("url", song.id); - try song_view.put("scrobbles", song.scrobbles); - - prev_artist_infos = artist_infos; prev_song_id = song.id; } return request.render(.ok); diff --git a/src/app/views/songs/index.zmpl b/src/app/views/songs/index.zmpl index 181da87..6964630 100644 --- a/src/app/views/songs/index.zmpl +++ b/src/app/views/songs/index.zmpl @@ -1,40 +1,15 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .scrobbles}; +} + - @partial partials/header

Songs

- - - - - - - - - -@for (.songs) |song| { - - - - - -} - -
NameArtist(s)Scrobbles
{{song.name}} - @for (song.get("artist_info").?) |ai| { - {{ai.name}} - } - {{song.scrobbles}}
- - +@partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) \ No newline at end of file diff --git a/src/types.zig b/src/types.zig index 455f282..2a82886 100644 --- a/src/types.zig +++ b/src/types.zig @@ -72,3 +72,18 @@ pub const Rules = struct { // scrobbles, // date, //}; +// + +pub const TableRows = struct { + song: ?HyperlinkData = null, + album: ?HyperlinkData = null, + artist: ?HyperlinkData = null, + artistlist: ?[]HyperlinkData = null, + scrobbles: ?i64 = null, + date: ?[]const u8 = null, +}; + +pub const HyperlinkData = struct { + name: []const u8, + id: i32, +}; From c42b8d24dd0c71afe5b52c7fe0c554331da08dd3 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 15:37:21 -0400 Subject: [PATCH 37/71] Fix typo --- src/app/views/albums.zig | 2 +- src/app/views/artists.zig | 2 +- src/app/views/scrobbles.zig | 2 +- src/app/views/songs.zig | 2 +- src/types.zig | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 95f5b86..9c6063e 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -1,7 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 461235a..9fd9dcd 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -1,7 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const dateFmt = @import("../../date_fmt.zig").dateFmt; const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 9403fa9..321af90 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -1,7 +1,7 @@ const std = @import("std"); const jetzig = @import("jetzig"); const zeit = @import("zeit"); -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 5302906..f91b5ed 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -1,6 +1,6 @@ const std = @import("std"); const jetzig = @import("jetzig"); -const TableRow = @import("../../types.zig").TableRows; +const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; pub fn index(request: *jetzig.Request) !jetzig.View { diff --git a/src/types.zig b/src/types.zig index 2a82886..e71018c 100644 --- a/src/types.zig +++ b/src/types.zig @@ -74,7 +74,7 @@ pub const Rules = struct { //}; // -pub const TableRows = struct { +pub const TableRow = struct { song: ?HyperlinkData = null, album: ?HyperlinkData = null, artist: ?HyperlinkData = null, From 4991bac9a452a6f07cefbc844d1bc070adf54800 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 15:39:21 -0400 Subject: [PATCH 38/71] Add LastFM scrobble type In preparation for importing via LastFM api --- src/app/views/upload.zig | 2 +- src/types.zig | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 5f47cc4..25a9128 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -37,7 +37,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { switch (source) { 0 => { - const content: Data.LastFM = try std.json.parseFromSliceLeaky(Data.LastFM, request.allocator, file.content, .{}); + const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; appends: for (content.scrobbles) |scrobble| { diff --git a/src/types.zig b/src/types.zig index e71018c..e8ac141 100644 --- a/src/types.zig +++ b/src/types.zig @@ -14,7 +14,7 @@ pub const Scrobble = struct { }; // From lastfmstats.com -pub const LastFM = struct { username: []const u8, scrobbles: []ImportedScrobble }; +pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble }; // I derived whether or not these values were optional from searching // the respective fields for null in Vim, so there may be some fields @@ -43,6 +43,37 @@ pub const SpotifyScrobble = struct { //incognito_mode: ?bool, }; +pub const LastFMWeb = struct { + track: []struct { + artist: LastFMWebHyperlinkData, + album: LastFMWebHyperlinkData, + name: []const u8, + mbid: []const u8, + image: struct { + size: []const u8, + @"#text": []const u8, + }, + date: struct { + uts: []const u8, + @"#text": []const u8, + }, + }, + @"@attr": LastFMWebAttr, +}; + +pub const LastFMWebAttr = struct { + perPage: u32, + totalPages: u32, + page: u32, + user: []const u8, + total: u32, +}; + +pub const LastFMWebHyperlinkData = struct { + mbid: []const u8, + @"#text": []const u8, +}; + pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, From 52fefc9ba5b28649c0bfeb926cef4e8d0b77fa27 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:22:09 -0400 Subject: [PATCH 39/71] Create dateCompare function Will eventually try to move away from zeit. Don't need all of it's functionality as long as SQL can format dates --- src/date_fmt.zig | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 603a209..00fe93b 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -1,8 +1,35 @@ const std = @import("std"); const zeit = @import("zeit"); +const Data = @import("types.zig"); pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } + +pub fn dateCompare(self: *[]const u8, earliest: []const u8, latest: []const u8) bool { + const range = if (self.len == 19) 4 else 2; + const time = std.fmt.parseInt(u32, self[0..range], 10) catch 0; + const etime = std.fmt.parseInt(u32, earliest[0..range], 10) catch 0; + const ltime = std.fmt.parseInt(u32, latest[0..range], 10) catch 0; + + if (time != etime and time != ltime) { + return (time > etime and time < ltime); + } else { + return dateCompare(self[range + 1 ..], earliest[range + 1 ..], latest[range + 1 ..]); + } +} + +pub fn scrobbleToRow(allocator: std.mem.Allocator, scrobble: Data.Scrobble) !Data.TableRow { + var artistlist = std.ArrayList(Data.HyperlinkData).init(allocator); + for (scrobble.artists_track) |a| { + try artistlist.append(Data.HyperlinkData{ .name = a, .id = 0 }); + } + return Data.TableRow{ + .song = .{ .name = scrobble.track, .id = 0 }, + .artistlist = artistlist.items, + .album = .{ .name = scrobble.album, .id = 0 }, + .date = try dateFmt(allocator, scrobble.date), + }; +} From f69ffb2b373f6d21647b7550147bed5dc66ba0db Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:22:34 -0400 Subject: [PATCH 40/71] Move upload.zig to the new table partial --- src/app/views/upload/post.zmpl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/app/views/upload/post.zmpl b/src/app/views/upload/post.zmpl index 176f094..91c5347 100644 --- a/src/app/views/upload/post.zmpl +++ b/src/app/views/upload/post.zmpl @@ -1,15 +1,16 @@ +@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; +} + - @partial partials/header

File Uploaded Successfully

-

Scrobbles Added

- -@partial partials/table(table_data: .scrobbles, table_headers: .upload_table, table_context: .context) - +@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) \ No newline at end of file From 89e98c7a47f907c507df88cd74d429090b1de26a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:23:12 -0400 Subject: [PATCH 41/71] Allow uploads from LastFM API Very slow at the moment. Look into ways to speed this up --- src/app/views/upload.zig | 313 ++++++++++++++++++++------------ src/app/views/upload/index.zmpl | 7 +- 2 files changed, 205 insertions(+), 115 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 25a9128..220e2c9 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -4,6 +4,8 @@ const jetquery = @import("jetzig").jetquery; const zeit = @import("zeit"); const rules = @import("../../apply_rule.zig"); const Data = @import("../../types.zig"); +const Utils = @import("../../date_fmt.zig"); +const Client = std.http.Client; pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { _ = data; @@ -13,99 +15,152 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - if (try request.file("upload")) |file| { - const params = try request.params(); - const source = try std.fmt.parseInt(u8, params.get("t").?.string.value, 10); // This param is required in HTML - const before_limiter: bool = if (params.get("bbool")) |_| true else false; - const after_limiter: bool = if (params.get("abool")) |_| true else false; + const params = try request.params(); + const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), + else => err, + }); - var scrobbles_view = try root.put("scrobbles", .array); - var job = try request.job("process_scrobbles"); - var scrobbles_data = try job.params.put("scrobbles", .array); + defer rule_file.close(); + const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); + const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + var job = try request.job("process_scrobbles"); + const source = params.getT(.integer, "t").?; // This param is required in HTML + const before_limiter: bool = if (params.get("bbool")) |_| true else false; + const after_limiter: bool = if (params.get("abool")) |_| true else false; - var skipped_tracks: u64 = 0; - var limited_tracks: u64 = 0; + var scrobbles_view = try root.put("scrobbles", .array); + var scrobbles_data = try job.params.put("scrobbles", .array); - const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), - else => err, - }); + var skipped_tracks: u64 = 0; + var limited_tracks: u64 = 0; - defer rule_file.close(); - const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + switch (source) { + 0, 1 => { + if (try request.file("upload")) |file| { + std.log.debug("{s}", .{file.filename}); + switch (source) { + 0 => { + const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); + const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; + const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; + appends: for (content.scrobbles) |scrobble| { + // We can short-circuit on the limiter bools + if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; - switch (source) { - 0 => { - const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); - const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; - const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; - appends: for (content.scrobbles) |scrobble| { - // We can short-circuit on the limiter bools - if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, scrobble, rl) + else + Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date, + }; - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, scrobble, rl) - else - Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date, - }; + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - var scrobble_view = try scrobbles_view.append(.object); - var artists = try scrobble_view.put("artists", .array); + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); - try scrobble_view.put("track", formatted_scrobble.track); - try scrobble_view.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_track) |artist| { - try artists.append(artist); - } - try scrobble_view.put("date", formatted_scrobble.date); + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } - var scrobble_data = try scrobbles_data.append(.object); - var artists_album = try scrobble_data.put("artists_album", .array); - var artists_track = try scrobble_data.put("artists_track", .array); + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + }, + 1 => { + const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); + const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time(); + const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); + appends: for (content) |scrobble| { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_album) |artist| { - try artists_album.append(artist); - } + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { + limited_tracks += 1; + continue :appends; + } - for (formatted_scrobble.artists_track) |artist| { - try artists_track.append(artist); - } - try scrobble_data.put("date", formatted_scrobble.date); + const pre_formatted_scrobble: Data.ImportedScrobble = .{ + .track = scrobble.master_metadata_track_name.?, + .album = scrobble.master_metadata_album_album_name.?, + .artist = scrobble.master_metadata_album_artist_name.?, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(), + }; + + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; + + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + }, + else => unreachable, } - }, - 1 => { - const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); - const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else (try zeit.instant(.{})).time(); - const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); - appends: for (content) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } + try job.schedule(); + std.log.debug("Skipped {} tracks", .{skipped_tracks}); + std.log.debug("Filtered {} tracks", .{limited_tracks}); + } + }, + 2 => { + if (params.getT(.string, "username")) |username| { + _ = username; + const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&format=json"; + const user_agent: []const u8 = "Zuletzt/0.0.1"; + var client = Client{ .allocator = request.allocator }; + var ar = std.ArrayList(u8).init(request.allocator); + _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + const first_response = try ar.toOwnedSlice(); + const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { - limited_tracks += 1; - continue :appends; - } - - const pre_formatted_scrobble: Data.ImportedScrobble = .{ - .track = scrobble.master_metadata_track_name.?, - .album = scrobble.master_metadata_album_album_name.?, - .artist = scrobble.master_metadata_album_artist_name.?, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000, + for (json.recenttracks.track) |scrobble| { + const pre_formatted_scrobble = Data.ImportedScrobble{ + .track = scrobble.name, + .album = if (scrobble.album) |album| album.@"#text".? else "", + .artist = scrobble.artist.@"#text".?, + .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), }; const formatted_scrobble = if (rule_list) |rl| @@ -119,44 +174,76 @@ pub fn post(request: *jetzig.Request) !jetzig.View { .date = pre_formatted_scrobble.date, }; - var scrobble_view = try scrobbles_view.append(.object); - var artists = try scrobble_view.put("artists", .array); - - try scrobble_view.put("track", formatted_scrobble.track); - try scrobble_view.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_track) |artist| { - try artists.append(artist); - } - try scrobble_view.put("date", formatted_scrobble.date); + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); var scrobble_data = try scrobbles_data.append(.object); - var artists_album = try scrobble_data.put("artists_album", .array); - var artists_track = try scrobble_data.put("artists_track", .array); - - try scrobble_data.put("track", formatted_scrobble.track); try scrobble_data.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_album) |artist| { - try artists_album.append(artist); - } - - for (formatted_scrobble.artists_track) |artist| { - try artists_track.append(artist); - } + try scrobble_data.put("track", formatted_scrobble.track); try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } } - }, - else => unreachable, - } - try job.schedule(); - std.log.debug("Skipped {} tracks", .{skipped_tracks}); - std.log.debug("Filtered {} tracks", .{limited_tracks}); + const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; + for (2..max_pages) |page| { + const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&page={}&format=json", .{page}); + std.log.debug("{s}", .{rest_query}); + _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + const response = try ar.toOwnedSlice(); + const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); + + for (json2.recenttracks.track) |scrobble| { + const pre_formatted_scrobble = Data.ImportedScrobble{ + .track = scrobble.name, + .album = if (scrobble.album) |album| album.@"#text".? else "", + .artist = scrobble.artist.@"#text".?, + .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), + }; + + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; + + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + } + try job.schedule(); + } + }, + else => unreachable, } - - var upload_table = try root.put("upload_table", .array); - try upload_table.append("Track"); - try upload_table.append("Artist"); - try upload_table.append("Album"); - try upload_table.append("Date"); - return request.render(.created); } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 9043a5b..31151e1 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -9,11 +9,14 @@
- - + + + +
Last.fm Spotify + Last.fm (WebAuth) Limit to Scrobbles before: Limit to Scrobbles after:
From 5697f95355d1e8f240eef9a8c9dbfbba3b10a001 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:23:53 -0400 Subject: [PATCH 42/71] Fix LastFM uploading errors Not sure which of these actually made it work, will probably work backwards at some point to reverse engineer it --- src/types.zig | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/types.zig b/src/types.zig index e8ac141..898943a 100644 --- a/src/types.zig +++ b/src/types.zig @@ -44,34 +44,40 @@ pub const SpotifyScrobble = struct { }; pub const LastFMWeb = struct { - track: []struct { - artist: LastFMWebHyperlinkData, - album: LastFMWebHyperlinkData, - name: []const u8, - mbid: []const u8, - image: struct { - size: []const u8, - @"#text": []const u8, - }, - date: struct { - uts: []const u8, - @"#text": []const u8, + recenttracks: struct { + track: []struct { + artist: LastFMWebHyperlinkData, + album: ?LastFMWebHyperlinkData = null, + name: []const u8, + mbid: ?[]const u8 = null, + image: ?[]struct { + size: []const u8, + @"#text": []const u8, + } = null, + date: struct { + uts: []const u8, + @"#text": []const u8, + }, + @"@attr": ?struct { + nowplaying: ?[]const u8 = null, + } = null, + url: ?[]const u8 = null, }, + @"@attr": LastFMWebAttr, }, - @"@attr": LastFMWebAttr, }; pub const LastFMWebAttr = struct { - perPage: u32, - totalPages: u32, - page: u32, - user: []const u8, - total: u32, + perPage: ?[]const u8 = null, + totalPages: []const u8, + page: ?[]const u8 = null, + user: ?[]const u8 = null, + total: ?[]const u8 = null, }; pub const LastFMWebHyperlinkData = struct { - mbid: []const u8, - @"#text": []const u8, + mbid: ?[]const u8 = null, + @"#text": ?[]const u8 = null, }; pub const Rule = struct { From 614607ae7167d986aa5be71e78ccf439ef288f6e Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 16 May 2025 01:05:32 -0400 Subject: [PATCH 43/71] Fix LastFM uploadig error I figured it out; if you have a song currently being played, then it doesn't have a date --- src/app/views/upload.zig | 22 ++++++++++++---------- src/types.zig | 26 +++++++++++++------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 220e2c9..2582ff4 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -147,7 +147,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { 2 => { if (params.getT(.string, "username")) |username| { _ = username; - const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&format=json"; + const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json"; const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var ar = std.ArrayList(u8).init(request.allocator); @@ -155,12 +155,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const first_response = try ar.toOwnedSlice(); const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - for (json.recenttracks.track) |scrobble| { + appends: for (json.recenttracks.track) |scrobble| { + if (scrobble.date == null) continue :appends; const pre_formatted_scrobble = Data.ImportedScrobble{ .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text".? else "", - .artist = scrobble.artist.@"#text".?, - .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artist = scrobble.artist.@"#text", + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), }; const formatted_scrobble = if (rule_list) |rl| @@ -195,18 +196,19 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; for (2..max_pages) |page| { - const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&page={}&format=json", .{page}); + const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page}); std.log.debug("{s}", .{rest_query}); _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); const response = try ar.toOwnedSlice(); const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); - for (json2.recenttracks.track) |scrobble| { + appends: for (json2.recenttracks.track) |scrobble| { + if (scrobble.date == null) continue :appends; const pre_formatted_scrobble = Data.ImportedScrobble{ .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text".? else "", - .artist = scrobble.artist.@"#text".?, - .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artist = scrobble.artist.@"#text", + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), }; const formatted_scrobble = if (rule_list) |rl| diff --git a/src/types.zig b/src/types.zig index 898943a..202bce6 100644 --- a/src/types.zig +++ b/src/types.zig @@ -50,34 +50,34 @@ pub const LastFMWeb = struct { album: ?LastFMWebHyperlinkData = null, name: []const u8, mbid: ?[]const u8 = null, - image: ?[]struct { + image: []struct { size: []const u8, @"#text": []const u8, - } = null, - date: struct { + }, + date: ?struct { uts: []const u8, @"#text": []const u8, - }, - @"@attr": ?struct { - nowplaying: ?[]const u8 = null, } = null, - url: ?[]const u8 = null, + @"@attr": ?struct { + nowplaying: []const u8, + } = null, + url: []const u8, }, @"@attr": LastFMWebAttr, }, }; pub const LastFMWebAttr = struct { - perPage: ?[]const u8 = null, + perPage: []const u8, totalPages: []const u8, - page: ?[]const u8 = null, - user: ?[]const u8 = null, - total: ?[]const u8 = null, + page: []const u8, + user: []const u8, + total: []const u8, }; pub const LastFMWebHyperlinkData = struct { - mbid: ?[]const u8 = null, - @"#text": ?[]const u8 = null, + mbid: []const u8, + @"#text": []const u8, }; pub const Rule = struct { From 4c759433d25f2ec6e961eafd505c8d163e51a29d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 16 May 2025 05:06:59 +0000 Subject: [PATCH 44/71] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f18e7e0..8247b21 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Licensed under MIT. - [ ] Import from Discogs[^2] - [ ] Import listening history - [x] From Lastfmstats.com (.json file)[^3] - - [ ] From Last.fm (authentication) + - [x] From Last.fm (authentication) - [x] From Spotify (.json file) - [ ] From other streaming services[^4] - [ ] "Unofficial scrobbles"[^9] From 6494bbdf6085a12464f3dc1f455fce1d5dcf0e02 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 20 May 2025 09:33:15 -0400 Subject: [PATCH 45/71] Remove Rules type --- src/app/jobs/process_rule.zig | 11 +++++------ src/types.zig | 4 ---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/app/jobs/process_rule.zig b/src/app/jobs/process_rule.zig index 7b60d54..6beb880 100644 --- a/src/app/jobs/process_rule.zig +++ b/src/app/jobs/process_rule.zig @@ -16,7 +16,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig return; }, }; - const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; + const out_rules = &[_]Data.Rule{rule}; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file.writeAll(out); file.close(); @@ -35,18 +35,17 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only }); if (file_content.len == 0) { - const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} }; + const out_rules = &[_]Data.Rule{rule}; const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); try file_write.writeAll(out); file_write.close(); return; } - const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{}); - try rules.appendSlice(content.rules); + const content: []Data.Rule = try std.json.parseFromSliceLeaky([]Data.Rule, allocator, file_content, .{}); + try rules.appendSlice(content); try rules.append(rule); - const out_rules = Data.Rules{ .rules = rules.items }; - const out = try std.json.stringifyAlloc(allocator, out_rules, .{}); + const out = try std.json.stringifyAlloc(allocator, rules.items, .{}); try file_write.writeAll(out); file_write.close(); diff --git a/src/types.zig b/src/types.zig index 202bce6..073b2d3 100644 --- a/src/types.zig +++ b/src/types.zig @@ -95,10 +95,6 @@ pub const Rule = struct { }, }; -pub const Rules = struct { - rules: []const Rule, -}; - // Can't import types in .zmpl files, so defining this here // doesn't really do much (except maybe in the .zig file for views?) //pub const HeaderTypes = []enum { From a2a739bc9c40b5f1a1bafe66408d43d76c2cf1fa Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 20 May 2025 15:07:51 -0400 Subject: [PATCH 46/71] Refactor upload.zig I have been unhappy with the branches, but didn't quite know what to do about it. THis feels much nicer. Also fixes datetime stuff with jetquery. The HTML element parsing isn't quite where I want it to be, but it works for the time being. --- src/app/jobs/process_scrobbles.zig | 1 - src/app/views/upload.zig | 330 +++++++++++------------------ src/app/views/upload/index.zmpl | 6 +- src/apply_rule.zig | 26 ++- src/date_fmt.zig | 2 +- src/types.zig | 77 ++++--- 6 files changed, 184 insertions(+), 258 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index c6b25b1..dcf7f1f 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -1,7 +1,6 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const lastfm = @import("../../types.zig").LastFM; const Data = @import("../../types.zig"); const rules = @import("../../apply_rule.zig"); diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 2582ff4..c86e6f0 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -23,229 +23,141 @@ pub fn post(request: *jetzig.Request) !jetzig.View { defer rule_file.close(); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; var job = try request.job("process_scrobbles"); const source = params.getT(.integer, "t").?; // This param is required in HTML - const before_limiter: bool = if (params.get("bbool")) |_| true else false; - const after_limiter: bool = if (params.get("abool")) |_| true else false; + const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { + const date = params.getT(.string, "latest-date").?; + break :blk try zeit.Time.fromISO8601(date); + } else blk: { + break :blk (try zeit.instant(.{ .source = .now })).time(); + }; + const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { + const date = params.getT(.string, "earliest-date").?; + break :blk try zeit.Time.fromISO8601(date); + } else blk: { + break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); + }; - var scrobbles_view = try root.put("scrobbles", .array); - var scrobbles_data = try job.params.put("scrobbles", .array); + const earliest_timestamp = earliest_date.instant().unixTimestamp(); + const latest_timestamp = latest_date.instant().unixTimestamp(); + + var view_params = try root.put("scrobbles", .array); + var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - switch (source) { - 0, 1 => { - if (try request.file("upload")) |file| { - std.log.debug("{s}", .{file.filename}); - switch (source) { - 0 => { - const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); - const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; - const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; - appends: for (content.scrobbles) |scrobble| { - // We can short-circuit on the limiter bools - if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; + const imported_scrobbles: Data.ImportedScrobbles = switch (source) { + 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles }, + 1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) }, + 2 => blk: { + const user_agent: []const u8 = "Zuletzt/0.0.1"; + var client = Client{ .allocator = request.allocator }; + var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); + var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator); - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, scrobble, rl) - else - Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date, - }; + const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + var page: usize = 1; + var max_pages: ?usize = null; - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - }, - 1 => { - const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); - const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time(); - const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); - appends: for (content) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { - limited_tracks += 1; - continue :appends; - } - - const pre_formatted_scrobble: Data.ImportedScrobble = .{ - .track = scrobble.master_metadata_track_name.?, - .album = scrobble.master_metadata_album_album_name.?, - .artist = scrobble.master_metadata_album_artist_name.?, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - }, - else => unreachable, - } - try job.schedule(); - std.log.debug("Skipped {} tracks", .{skipped_tracks}); - std.log.debug("Filtered {} tracks", .{limited_tracks}); + while (true) : (page += 1) { + if (max_pages != null and page > max_pages.?) break; + const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); + const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + std.log.debug("{}", .{r}); + const response_string = try lastfm_response_buffer.toOwnedSlice(); + const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); + //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); + if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); + try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track); } - }, - 2 => { - if (params.getT(.string, "username")) |username| { - _ = username; - const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json"; - const user_agent: []const u8 = "Zuletzt/0.0.1"; - var client = Client{ .allocator = request.allocator }; - var ar = std.ArrayList(u8).init(request.allocator); - _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - const first_response = try ar.toOwnedSlice(); - const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - appends: for (json.recenttracks.track) |scrobble| { - if (scrobble.date == null) continue :appends; - const pre_formatted_scrobble = Data.ImportedScrobble{ - .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artist = scrobble.artist.@"#text", - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; - for (2..max_pages) |page| { - const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page}); - std.log.debug("{s}", .{rest_query}); - _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - const response = try ar.toOwnedSlice(); - const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); - - appends: for (json2.recenttracks.track) |scrobble| { - if (scrobble.date == null) continue :appends; - const pre_formatted_scrobble = Data.ImportedScrobble{ - .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artist = scrobble.artist.@"#text", - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - } - try job.schedule(); - } + break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items }; }, else => unreachable, + }; + + // Not sure if I should be proud or feel sick + switch (imported_scrobbles) { + inline else => |scrobbles| { + appends: for (scrobbles) |scrobble| { + const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) { + Data.IgnorantScrobble => { + if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { + limited_tracks += 1; + continue :appends; + } + break :blk Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date * 1_000, + }; + }, + Data.SpotifyScrobble => { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } + + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { + limited_tracks += 1; + continue :appends; + } + + break :blk Data.Scrobble{ + .album = scrobble.master_metadata_album_album_name.?, + .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .track = scrobble.master_metadata_track_name.?, + .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, + }; + }, + Data.LastFMWebScrobble => { + break :blk Data.Scrobble{ + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, + .track = scrobble.name, + .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, + }; + }, + else => unreachable, + }; + + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + + try view_params.append(row); + var job_param = try job_params.append(.object); + try job_param.put("album", complete_scrobble.album); + try job_param.put("track", complete_scrobble.track); + try job_param.put("date", complete_scrobble.date); + + var track_artists_array = try job_param.put("artists_track", .array); + for (complete_scrobble.artists_track) |a| { + try track_artists_array.append(a); + } + + var album_artists_array = try job_param.put("artists_album", .array); + for (complete_scrobble.artists_album) |a| { + try album_artists_array.append(a); + } + } + }, } + + std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); + try job.schedule(); + return request.render(.created); } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 31151e1..420e08c 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -17,8 +17,10 @@ Last.fm Spotify Last.fm (WebAuth) - Limit to Scrobbles before: - Limit to Scrobbles after: + + Advanced Options + Limit to Scrobbles before: + Limit to Scrobbles after: diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 6163e93..fc4c0bc 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const Rules = @import("./types.zig").Rules; +const Rule = @import("./types.zig").Rule; const Data = @import("./types.zig"); // Wrapper for containsAtLeast to make the switch below to work @@ -11,18 +11,10 @@ fn eqlWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.eql(u8, haystack, needle); } -pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) !Data.Scrobble { - const artists = try allocator.alloc([]const u8, 1); - artists[0] = scrobble.artist; - var output_scrobble = Data.Scrobble{ - .track = scrobble.track, - .artists_track = artists, - .album = scrobble.album, - .artists_album = artists, - .date = scrobble.date, - }; +pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble, rules: []Rule) !Data.Scrobble { + var output_scrobble = scrobble; - for (rules.rules) |rule| { + for (rules) |rule| { var match_found: bool = switch (rule.cond_req) { .any => false, .all => true, @@ -34,10 +26,16 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc }; switch (rule.cond_req) { .any => switch (cond.match_on) { - inline else => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .album, .track => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .artists_album, .artists_track => |on| { + for (@field(scrobble, @tagName(on))) |artist| match_found = match_found or match_fn(artist, cond.match_txt); + }, }, .all => switch (cond.match_on) { - inline else => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .album, .track => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .artists_album, .artists_track => |on| { + for (@field(scrobble, @tagName(on))) |artist| match_found = match_found and match_fn(artist, cond.match_txt); + }, }, } } diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 00fe93b..7e44da7 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -4,7 +4,7 @@ const Data = @import("types.zig"); pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } diff --git a/src/types.zig b/src/types.zig index 073b2d3..3b2e4ff 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,7 +1,20 @@ -pub const ImportedScrobble = struct { +pub const ImportedScrobbles = union(ScrobbleSources) { + LastFMStats: []IgnorantScrobble, + LastFMWeb: []LastFMWebScrobble, + Spotify: []SpotifyScrobble, +}; + +const ScrobbleSources = enum { + LastFMStats, + LastFMWeb, + Spotify, +}; + +pub const IgnorantScrobble = struct { track: []const u8, artist: []const u8, - album: []const u8 = "", + album: []const u8 = "Not Provided", + //albumId: []const u8, date: i64, }; @@ -14,7 +27,7 @@ pub const Scrobble = struct { }; // From lastfmstats.com -pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble }; +pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble }; // I derived whether or not these values were optional from searching // the respective fields for null in Vim, so there may be some fields @@ -45,46 +58,48 @@ pub const SpotifyScrobble = struct { pub const LastFMWeb = struct { recenttracks: struct { - track: []struct { - artist: LastFMWebHyperlinkData, - album: ?LastFMWebHyperlinkData = null, - name: []const u8, - mbid: ?[]const u8 = null, - image: []struct { - size: []const u8, - @"#text": []const u8, - }, - date: ?struct { - uts: []const u8, - @"#text": []const u8, - } = null, - @"@attr": ?struct { - nowplaying: []const u8, - } = null, - url: []const u8, - }, - @"@attr": LastFMWebAttr, + track: []LastFMWebScrobble, + @"@attr": LastFMWebQueryInfo, }, }; -pub const LastFMWebAttr = struct { - perPage: []const u8, - totalPages: []const u8, - page: []const u8, - user: []const u8, - total: []const u8, -}; - pub const LastFMWebHyperlinkData = struct { mbid: []const u8, @"#text": []const u8, }; +pub const LastFMWebScrobble = struct { + artist: LastFMWebHyperlinkData, + album: ?LastFMWebHyperlinkData = null, + name: []const u8, + mbid: ?[]const u8 = null, + image: []struct { + size: []const u8, + @"#text": []const u8, + }, + date: ?struct { + uts: []const u8, + @"#text": []const u8, + } = null, + @"@attr": ?struct { + nowplaying: []const u8, + } = null, + url: []const u8, +}; + +pub const LastFMWebQueryInfo = struct { + perPage: []const u8, + totalPages: []const u8, + page: []const u8, + user: []const u8, + total: []const u8, +}; + pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, conditionals: []struct { - match_on: enum { artist, album, track }, + match_on: enum { artists_album, artists_track, album, track }, match_cond: enum { is, contains }, match_txt: []const u8, }, From 1734e6a4bbf83062c56dc5a4c50687cbf665c859 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Tue, 20 May 2025 16:29:53 -0400 Subject: [PATCH 47/71] Fix date formatting in scrobble view --- src/app/views/scrobbles.zig | 6 +++--- src/app/views/upload.zig | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 321af90..3fe946a 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -3,6 +3,7 @@ const jetzig = @import("jetzig"); const zeit = @import("zeit"); const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; +const Utils = @import("../../date_fmt.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); @@ -35,8 +36,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); continue :blk; } else { - var date = std.ArrayList(u8).init(request.allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(scrobble.date, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + const date = try Utils.dateFmt(request.allocator, scrobble.date); try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); @@ -44,7 +44,7 @@ pub fn index(request: *jetzig.Request) !jetzig.View { .song = .{ .name = scrobble.song_name, .id = scrobble.song_id }, .album = .{ .name = scrobble.album_name, .id = scrobble.album_id }, .artistlist = try artistlist.toOwnedSlice(), - .date = date.items, + .date = date, }; try scrobbles_view.append(row); diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index c86e6f0..af5d41d 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -66,7 +66,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { if (max_pages != null and page > max_pages.?) break; const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - std.log.debug("{}", .{r}); + std.log.debug("{}: {}", .{ page, r }); const response_string = try lastfm_response_buffer.toOwnedSlice(); const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); From 09f542e26e84b6a1e2dd9e5abed7a20d5547a7be Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 24 May 2025 13:58:31 -0400 Subject: [PATCH 48/71] Add timescale partial Bad name, idk what else to call it --- src/app/views/albums/get.zmpl | 4 ++++ src/app/views/artists/get.zmpl | 5 +++-- src/app/views/partials/_firstlast_listens.zmpl | 10 +++++++--- src/app/views/partials/_timescale.zmpl | 18 ++++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 src/app/views/partials/_timescale.zmpl diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 3a1641b..1024ec5 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -10,6 +10,10 @@ @partial partials/header

{{.album}}

+@partial partials/firstlast_listens(scrobbles: .album.scrobbles, rank: .album.rank, firstlast: .firstlast) +

Yearly Performance

+@partial partials/timescale(range: .yearly) +

Songs

@partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) \ No newline at end of file diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 48c1c25..69bd661 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -10,8 +10,9 @@ @partial partials/header

{{.artist.name}}

-@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, last_song: .last, first_song: .first) - +@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, firstlast: .firstlast) +

Yearly Performance

+@partial partials/timescale(range: .yearly)

Albums

@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl index 322a0e8..ed74361 100644 --- a/src/app/views/partials/_firstlast_listens.zmpl +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -1,9 +1,13 @@ -@args scrobbles: i64, rank: []const u8, last_song: *ZmplValue, first_song: *ZmplValue +@args scrobbles: i64, rank: []const u8, firstlast: *ZmplValue + +@zig { + const songs = firstlast.items(.array); +}
{{scrobbles}} scrobbles ({{rank}} place)
-First listen: {{first_song.name}} ({{first_song.date}}) +First listen: {{songs[0].name}} ({{songs[0].date}})
-Most recent listen: {{last_song.name}} ({{last_song.date}}) +Most recent listen: {{songs[1].name}} ({{songs[1].date}})
\ No newline at end of file diff --git a/src/app/views/partials/_timescale.zmpl b/src/app/views/partials/_timescale.zmpl new file mode 100644 index 0000000..fdec7a1 --- /dev/null +++ b/src/app/views/partials/_timescale.zmpl @@ -0,0 +1,18 @@ +@args range: *ZmplValue + + + + + + + + + +@for (range) |itm| { + + + + +} + +
YearScrobbles
{{itm.year}}:{{itm.scrobbles}}
\ No newline at end of file From 7f3778e82f74627bfe992fd03193acec06adf9ce Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 24 May 2025 13:59:28 -0400 Subject: [PATCH 49/71] Move SQL logic to separate function Idk if this makes any sense, and I don't really like the code atm, but the view .zig files lookk nicer? --- src/app/views/albums.zig | 72 ++------ src/app/views/artists.zig | 164 ++---------------- src/app/views/songs.zig | 44 +---- src/queries.zig | 343 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 370 insertions(+), 253 deletions(-) create mode 100644 src/queries.zig diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 9c6063e..af72456 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -3,81 +3,31 @@ const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; const TableRow = @import("../../types.zig").TableRow; const HyperlinkData = @import("../../types.zig").HyperlinkData; +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var albums_view = try root.put("albums", .array); - const query = - \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN albums ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN artistalbums ON artistalbums.album_id = albums.id - \\INNER JOIN artists ON artists.id = artistalbums.artist_id - \\GROUP BY albums.id, artists.id - \\ORDER BY scrobbles DESC - ; - var albums_jq_result = try request.repo.executeSql(query, .{}); - defer albums_jq_result.deinit(); + const albums = try queries.entityQueryResult(request, queries.generateQuery(.album, .entities), .{}, .array); + try root.put("albums", albums); - const Album = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; - - var prev_album_id: ?i32 = null; - - var row: ?TableRow = null; - var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); - - blk: while (try albums_jq_result.postgresql.result.next()) |album_row| { - const album = try album_row.to(Album, .{ .dupe = true, .allocator = request.allocator }); - if (album.id == prev_album_id) { - try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); - continue :blk; - } else { - try artistlist.append(.{ .name = album.artist_name, .id = album.artist_id }); - - row = TableRow{ - .album = .{ .name = album.name, .id = album.id }, - .artistlist = try artistlist.toOwnedSlice(), - .scrobbles = album.scrobbles, - }; - - try albums_view.append(row); - } - prev_album_id = album.id; - } return request.render(.ok); } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var songs_view = try root.put("songs", .array); - const album_name = try jetzig.database.Query(.Album).find(id).select(.{ .id, .name }).execute(request.repo); - _ = try root.put("album", album_name.?.name); - const query = - \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN songs ON albumsongs.song_id = songs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\WHERE albumsongs.album_id = $1 - \\GROUP BY songs.id - \\ORDER BY scrobbles DESC - ; + const album = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_info), .{id}, .object); + try root.put("album", album.get("entity_info")); - var songs_js_result = try request.repo.executeSql(query, .{id}); - defer songs_js_result.deinit(); + const songs = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_items), .{id}, .array); + try root.put("songs", songs); - const Song = struct { name: []const u8, id: i32, scrobbles: i64 }; + const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.album, .firstlast), .{id}, .array); + try root.put("firstlast", firstlast); - while (try songs_js_result.postgresql.result.next()) |song_row| { - const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); - const row = TableRow{ - .song = .{ .name = song.name, .id = song.id }, - .scrobbles = song.scrobbles, - }; + const timescale = try queries.entityQueryResult(request, queries.generateQuery(.album, .timescale), .{id}, .array); + try root.put("yearly", timescale); - try songs_view.append(row); - } return request.render(.ok); } diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 9fd9dcd..37ca155 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -4,35 +4,13 @@ const jetquery = @import("jetzig").jetquery; const TableRow = @import("../../types.zig").TableRow; const dateFmt = @import("../../date_fmt.zig").dateFmt; const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt; +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var artists_view = try root.put("artists", .array); + const artists = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entities), .{}, .array); - const query = - \\SELECT artists.name, artists.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongsartists - \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id - \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\GROUP BY artists.id - \\ORDER BY scrobbles DESC; - ; - - var artists_jq_result = try request.repo.executeSql(query, .{}); - defer artists_jq_result.deinit(); - - const Artist = struct { name: []const u8, id: i32, scrobbles: i64 }; - - while (try artists_jq_result.postgresql.result.next()) |artist_row| { - const artist = try artist_row.to(Artist, .{ .dupe = true, .allocator = request.allocator }); - const row = TableRow{ - .artist = .{ .name = artist.name, .id = artist.id }, - .scrobbles = artist.scrobbles, - }; - - try artists_view.append(row); - } + try root.put("artists", artists); return request.render(.ok); } @@ -40,136 +18,20 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const ArtistResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: i64 }; - const AlbumsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; - const ScrobbleResult = struct { name: []const u8, id: i32, date: i64 }; + const artist = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_info), .{id}, .object); + try root.put("artist", artist.get("entity_info")); - const artist_info_query = - \\SELECT * FROM - \\(SELECT *, ROW_NUMBER() OVER (ORDER BY scrobbles DESC) AS rank FROM - \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongsartists - \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id - \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\GROUP BY artists.id)) - \\WHERE id = $1 - ; + const albums = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_items), .{id}, .array); + try root.put("albums", albums); - var artist_jq_result = try request.repo.executeSql(artist_info_query, .{id}); - defer artist_jq_result.deinit(); + const appears = try queries.entityQueryResult(request, queries.generateQuery(.artist, .appears), .{id}, .array); + try root.put("appears", appears); - if (try artist_jq_result.postgresql.result.next()) |artist_row| { - const artist = try artist_row.to(ArtistResult, .{ .dupe = true, .allocator = request.allocator }); - var artist_view = try root.put("artist", .object); - try artist_view.put("name", artist.name); - try artist_view.put("id", artist.id); - try artist_view.put("scrobbles", artist.scrobbles); - try artist_view.put("rank", ordinalFmt(request.allocator, artist.rank)); - } - try artist_jq_result.drain(); + const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.artist, .firstlast), .{id}, .array); + try root.put("firstlast", firstlast); - const albums_query = - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles - \\FROM artistalbums - \\INNER JOIN albums ON albums.id = artistalbums.album_id - \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\WHERE artistalbums.artist_id = $1 - \\GROUP BY albums.id - \\ORDER BY scrobbles DESC - ; - - var albums_view = try root.put("albums", .array); - - var albums_jq_result = try request.repo.executeSql(albums_query, .{id}); - defer albums_jq_result.deinit(); - - while (try albums_jq_result.postgresql.result.next()) |album_row| { - const album = try album_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - const album_table_row = TableRow{ - .album = .{ .name = album.name, .id = album.id }, - .scrobbles = album.scrobbles, - }; - - try albums_view.append(album_table_row); - } - //albums_jq_result.drain(); - - const appears_query = - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles - \\FROM artistalbums - \\INNER JOIN albums ON albums.id = artistalbums.album_id - \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 AND artistalbums.artist_id != $1 - \\GROUP BY albums.id - \\ORDER BY scrobbles DESC; - ; - - var appears_view = try root.put("appears", .array); - - var appears_jq_result = try request.repo.executeSql(appears_query, .{id}); - defer appears_jq_result.deinit(); - - while (try appears_jq_result.postgresql.result.next()) |appears_row| { - const appears = try appears_row.to(AlbumsResult, .{ .dupe = true, .allocator = request.allocator }); - const appears_table_row = TableRow{ - .album = .{ .name = appears.name, .id = appears.id }, - .scrobbles = appears.scrobbles, - }; - - try appears_view.append(appears_table_row); - } - - //appears_jq_result.drain(); - - const first_last_songs_query = - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date - \\FROM albumsongs - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 - \\ORDER BY scrobbles.datetime DESC - \\LIMIT 1) - \\ - \\UNION ALL - \\ - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date - \\FROM albumsongs - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 - \\ORDER BY scrobbles.datetime ASC - \\LIMIT 1) - ; - - var first_last_songs_jq_result = try request.repo.executeSql(first_last_songs_query, .{id}); - defer first_last_songs_jq_result.deinit(); - - // These look backwards, but it's correct this way - var last_song_view = try root.put("last", .object); - - if (try first_last_songs_jq_result.postgresql.result.next()) |last_song_row| { - const last_song = try last_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try last_song_view.put("name", last_song.name); - try last_song_view.put("id", last_song.id); - try last_song_view.put("date", (try dateFmt(request.allocator, last_song.date))); - } else unreachable; - - var first_song_view = try root.put("first", .object); - - if (try first_last_songs_jq_result.postgresql.result.next()) |first_song_row| { - const first_song = try first_song_row.to(ScrobbleResult, .{ .dupe = true, .allocator = request.allocator }); - try first_song_view.put("name", first_song.name); - try first_song_view.put("id", first_song.id); - try first_song_view.put("date", (try dateFmt(request.allocator, first_song.date))); - } else unreachable; - - try first_last_songs_jq_result.drain(); + const timescale = try queries.entityQueryResult(request, queries.generateQuery(.artist, .timescale), .{id}, .array); + try root.put("yearly", timescale); return request.render(.ok); } diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index f91b5ed..d5a7944 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -1,51 +1,13 @@ const std = @import("std"); const jetzig = @import("jetzig"); -const TableRow = @import("../../types.zig").TableRow; -const HyperlinkData = @import("../../types.zig").HyperlinkData; +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var songs_view = try root.put("songs", .array); - const query = - \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles - \\FROM albumsongs - \\INNER JOIN songs ON albumsongs.song_id = songs.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\GROUP BY songs.id, artists.id - \\ORDER BY scrobbles DESC, songs.name ASC - ; + const songs = try queries.entityQueryResult(request, queries.generateQuery(.song, .entities), .{}, .array); + try root.put("songs", songs); - var songs_js_result = try request.repo.executeSql(query, .{}); - defer songs_js_result.deinit(); - - const Song = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; - - var prev_song_id: ?i32 = null; - - var row: ?TableRow = null; - var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); - - blk: while (try songs_js_result.postgresql.result.next()) |song_row| { - const song = try song_row.to(Song, .{ .dupe = true, .allocator = request.allocator }); - if (song.id == prev_song_id) { - try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); - continue :blk; - } else { - try artistlist.append(.{ .name = song.artist_name, .id = song.artist_id }); - - row = TableRow{ - .song = .{ .name = song.name, .id = song.id }, - .artistlist = try artistlist.toOwnedSlice(), - .scrobbles = song.scrobbles, - }; - - try songs_view.append(row); - } - prev_song_id = song.id; - } return request.render(.ok); } diff --git a/src/queries.zig b/src/queries.zig new file mode 100644 index 0000000..cc5137b --- /dev/null +++ b/src/queries.zig @@ -0,0 +1,343 @@ +// Probably the worst code in the project +const jetzig = @import("jetzig"); +const TableRow = @import("types.zig").TableRow; +const HyperlinkData = @import("types.zig").HyperlinkData; +const std = @import("std"); + +pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: anytype, root_type: enum { object, array }) !*jetzig.Data.Value { + var result = try request.repo.executeSql(query.query, args); + + defer result.deinit(); + + var Data = jetzig.Data.init(request.allocator); + var out: *jetzig.Data.Value = switch (root_type) { + .array => try Data.array(), + .object => try Data.object(), + }; + + var artist_list = if (query.ResultType == EntitiesAlbumResult or query.ResultType == EntitiesArtistResult or query.ResultType == EntitiesScrobbleResult or query.ResultType == EntitiesSongResult) + std.ArrayList(HyperlinkData).init(request.allocator); + + blk: while (try result.postgresql.result.next()) |entity_row| { + switch (query.ResultType) { + FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult => |T| { + const entity = try entity_row.to(T, .{ .dupe = true, .allocator = request.allocator }); + const item: ?TableRow = switch (query.ResultType) { + EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { + .artist => TableRow{ .artist = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + .album => album_entities: { + const last_artist = artist_list.getLastOrNull(); + try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); + if (last_artist) |la| { + if (la.id == entity.artist_id) continue :blk; + } + break :album_entities TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; + }, + .song => song_entities: { + const last_artist = artist_list.getLastOrNull(); + try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); + if (last_artist) |la| { + if (la.id == entity.artist_id) continue :blk; + } + break :song_entities TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; + }, + else => unreachable, + }, + EntityItemsResult => switch (query.entity) { + .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + .album => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + else => unreachable, + }, + AppearsResult => switch (query.entity) { + .song, .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, + .album, .scrobble => unreachable, + }, + else => null, + }; + + if (item) |itm| { + switch (root_type) { + .array => try out.append(itm), + .object => switch (query.query_type) { + inline else => |qt| try out.put(@tagName(qt), itm), + }, + } + } else { + switch (root_type) { + .array => try out.append(entity), + .object => switch (query.query_type) { + inline else => |qt| try out.put(@tagName(qt), entity), + }, + } + } + }, + else => unreachable, + } + } + if (root_type == .object) {} + return out; +} + +const GeneratedQuery = struct { + entity: EntityType, + query_type: QueryTypeEnum, + ResultType: type, + query: []const u8, +}; + +const EntityType = enum { scrobble, song, album, artist }; +const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info }; +const FirstlastResult = struct { name: []const u8, id: i32, date: []const u8 }; +const TimescaleResult = struct { year: []const u8, scrobbles: i64 }; +const EntitiesArtistResult = struct { name: []const u8, id: i32, scrobbles: i64 }; +const EntitiesAlbumResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; +const EntitiesSongResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; +const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; +const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; +const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; +const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; + +pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { + return GeneratedQuery{ + .entity = entity, + .query_type = query_type, + .ResultType = switch (query_type) { + .firstlast => FirstlastResult, + .timescale => TimescaleResult, + .entities => switch (entity) { + .scrobble => EntitiesScrobbleResult, + .song => EntitiesSongResult, + .album => EntitiesAlbumResult, + .artist => EntitiesArtistResult, + }, + .entity_items => EntityItemsResult, + .appears => AppearsResult, + .entity_info => EntityInfoResult, + }, + .query = switch (query_type) { + .firstlast => + //.ResultType = FirstlastResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime ASC + \\LIMIT 1) + \\ + \\UNION ALL + \\ + \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime DESC + \\LIMIT 1) + , + + .album => + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1 + \\ORDER BY scrobbles.datetime ASC + \\LIMIT 1) + \\ + \\UNION ALL + \\ + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1 + \\ORDER BY scrobbles.datetime DESC + \\LIMIT 1) + , + + .artist => + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime ASC + \\LIMIT 1) + \\ + \\UNION ALL + \\ + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\ORDER BY scrobbles.datetime DESC + \\LIMIT 1) + , + }, + + .timescale => + //.ResultType = TimescaleResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE artists.id = $1 + \\GROUP BY year + \\ORDER BY year ASC; + , + + .album => + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1 + \\GROUP BY year + \\ORDER BY year ASC; + , + + .artist => + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\FROM scrobbles + \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE artists.id = $1 + \\GROUP BY year + \\ORDER BY year ASC; + , + }, + + .entities => + //.ResultType = EntitiesResult, + switch (entity) { + .scrobble => + \\none + , + .song => + \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\GROUP BY songs.id, artists.id + \\ORDER BY scrobbles DESC, songs.name ASC + , + .album => + \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN albums ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong + \\INNER JOIN artistalbums ON artistalbums.album_id = albums.id + \\INNER JOIN artists ON artists.id = artistalbums.artist_id + \\GROUP BY albums.id, artists.id + \\ORDER BY scrobbles DESC + , + .artist => + \\SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongsartists + \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id + \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY artists.id + \\ORDER BY scrobbles DESC; + , + }, + + .appears => + //.ResultType = AppearsResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\ NOTHING YET + , + .album => + \\nope + , + .artist => + \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 AND artistalbums.artist_id != $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC; + , + }, + + .entity_items => + //.ResultType = EntityItemsResult, + switch (entity) { + .scrobble => unreachable, + .song => + \\get all scrobbles + , + .album => + \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albumsongs.album_id = $1 + \\GROUP BY songs.id + \\ORDER BY scrobbles DESC + , + .artist => + \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE artistalbums.artist_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + , + }, + + .entity_info => switch (entity) { + .scrobble => unreachable, + .song => + \\lol idk + , + .album => + \\SELECT * FROM + \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM + \\(SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN albums ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY albums.id)) + \\WHERE id = $1 + , + .artist => + \\SELECT * FROM + \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM + \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongsartists + \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id + \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY artists.id)) + \\WHERE id = $1 + , + }, + }, + }; +} From aab61631a397012f581fc2b9fd078eb9e09650cf Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sun, 25 May 2025 16:16:18 -0400 Subject: [PATCH 50/71] Directly append complete_scrobble in upload.zig Thanks bob :) --- build.zig.zon | 4 ++-- src/app/views/upload.zig | 15 +-------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index d7fb785..5db05e3 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -17,8 +17,8 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/7be1d137fcab5c422e05f12092f6e04a02900d6f.tar.gz", - .hash = "jetzig-0.0.0-IpAgLURbDwB1NlywUH7lnQ3zptNvSQWVosaA1k7l1cNz", + .url = "https://github.com/jetzig-framework/jetzig/archive/695664bc10e6e73389ee2fe7fbcaedc27fa8adc9.tar.gz", + .hash = "jetzig-0.0.0-IpAgLSFeDwBEhfEaDBo0JqhRbvQA3ScbWjssBqhJHFmb", }, .zeit = .{ .url = "https://github.com/rockorager/zeit/archive/refs/tags/v0.6.0.tar.gz", diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index af5d41d..f605982 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -138,20 +138,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - var job_param = try job_params.append(.object); - try job_param.put("album", complete_scrobble.album); - try job_param.put("track", complete_scrobble.track); - try job_param.put("date", complete_scrobble.date); - - var track_artists_array = try job_param.put("artists_track", .array); - for (complete_scrobble.artists_track) |a| { - try track_artists_array.append(a); - } - - var album_artists_array = try job_param.put("artists_album", .array); - for (complete_scrobble.artists_album) |a| { - try album_artists_array.append(a); - } + try job_params.append(complete_scrobble); } }, } From f59eec79a82584bd7bcc88277b3a0b6f86ba0eb8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 26 May 2025 11:15:19 -0400 Subject: [PATCH 51/71] Removed inline else from upload.zig If I can figure out a way to get an array of a union instead of a union of arrays, we're in business to make this even better, but this is fine right now. The inline else was just a dumb way to keep the for on the outside --- src/app/views/upload.zig | 108 +++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index f605982..bdb5cef 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -81,58 +81,68 @@ pub fn post(request: *jetzig.Request) !jetzig.View { // Not sure if I should be proud or feel sick switch (imported_scrobbles) { - inline else => |scrobbles| { + .LastFMStats => |scrobbles| { appends: for (scrobbles) |scrobble| { - const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) { - Data.IgnorantScrobble => { - if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { - limited_tracks += 1; - continue :appends; - } - break :blk Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date * 1_000, - }; - }, - Data.SpotifyScrobble => { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { - limited_tracks += 1; - continue :appends; - } - - break :blk Data.Scrobble{ - .album = scrobble.master_metadata_album_album_name.?, - .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .track = scrobble.master_metadata_track_name.?, - .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, - }; - }, - Data.LastFMWebScrobble => { - break :blk Data.Scrobble{ - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, - .track = scrobble.name, - .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, - }; - }, - else => unreachable, + if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { + limited_tracks += 1; + continue :appends; + } + const filtered_scrobble = Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date * 1_000, }; + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + + try view_params.append(row); + try job_params.append(complete_scrobble); + } + }, + .LastFMWeb => |scrobbles| { + for (scrobbles) |scrobble| { + const filtered_scrobble = Data.Scrobble{ + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, + .track = scrobble.name, + .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, + }; + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + + try view_params.append(row); + try job_params.append(complete_scrobble); + } + }, + .Spotify => |scrobbles| { + appends: for (scrobbles) |scrobble| { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } + + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { + limited_tracks += 1; + continue :appends; + } + + const filtered_scrobble = Data.Scrobble{ + .album = scrobble.master_metadata_album_album_name.?, + .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .track = scrobble.master_metadata_track_name.?, + .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, + }; const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); From 3ff973e193f331676c0470a6dfc4ff38330aff64 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 26 May 2025 11:15:51 -0400 Subject: [PATCH 52/71] Use queries.zig in scrobbles view --- src/app/views/scrobbles.zig | 50 +++---------------------------------- src/queries.zig | 37 ++++++++++++++------------- 2 files changed, 23 insertions(+), 64 deletions(-) diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index 3fe946a..e61df4d 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -1,56 +1,12 @@ const std = @import("std"); const jetzig = @import("jetzig"); -const zeit = @import("zeit"); -const TableRow = @import("../../types.zig").TableRow; -const HyperlinkData = @import("../../types.zig").HyperlinkData; -const Utils = @import("../../date_fmt.zig"); +const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - var scrobbles_view = try root.put("scrobbles", .array); - const query = - \\SELECT songs.name, songs.id, albums.name, albums.id, artists.name, artists.id, scrobbles.id, scrobbles.datetime - \\FROM albumsongs - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN albums ON albums.id = albumsongs.album_id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\ORDER BY scrobbles.datetime ASC - ; - - var scrobbles_jq_result = try request.repo.executeSql(query, .{}); - defer scrobbles_jq_result.deinit(); - - const Scrobble = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; - - var prev_s_id: ?i32 = null; - - var row: ?TableRow = null; - var artistlist = std.ArrayList(HyperlinkData).init(request.allocator); - - blk: while (try scrobbles_jq_result.postgresql.result.next()) |scrobble_row| { - const scrobble = try scrobble_row.to(Scrobble, .{ .dupe = true, .allocator = request.allocator }); - if (scrobble.s_id == prev_s_id) { - try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); - continue :blk; - } else { - const date = try Utils.dateFmt(request.allocator, scrobble.date); - - try artistlist.append(.{ .name = scrobble.artist_name, .id = scrobble.artist_id }); - - row = TableRow{ - .song = .{ .name = scrobble.song_name, .id = scrobble.song_id }, - .album = .{ .name = scrobble.album_name, .id = scrobble.album_id }, - .artistlist = try artistlist.toOwnedSlice(), - .date = date, - }; - - try scrobbles_view.append(row); - } - prev_s_id = scrobble.s_id; - } + const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.scrobble, .entities), .{}, .array); + try root.put("scrobbles", scrobbles); return request.render(.ok); } diff --git a/src/queries.zig b/src/queries.zig index cc5137b..faba14d 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -25,23 +25,19 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: const item: ?TableRow = switch (query.ResultType) { EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { .artist => TableRow{ .artist = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .album => album_entities: { + .scrobble, .song, .album => album_entities: { const last_artist = artist_list.getLastOrNull(); try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); if (last_artist) |la| { if (la.id == entity.artist_id) continue :blk; } - break :album_entities TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; + break :album_entities switch (query.entity) { + .scrobble => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .artistlist = try artist_list.toOwnedSlice(), .date = entity.date }, + .song => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, + .album => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, + else => unreachable, + }; }, - .song => song_entities: { - const last_artist = artist_list.getLastOrNull(); - try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); - if (last_artist) |la| { - if (la.id == entity.artist_id) continue :blk; - } - break :song_entities TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }; - }, - else => unreachable, }, EntityItemsResult => switch (query.entity) { .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, @@ -92,7 +88,7 @@ const TimescaleResult = struct { year: []const u8, scrobbles: i64 }; const EntitiesArtistResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const EntitiesAlbumResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; const EntitiesSongResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; -const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, s_id: i32, date: i64 }; +const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, date: []const u8 }; const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; @@ -142,7 +138,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .album => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -153,7 +149,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -164,7 +160,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .artist => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -175,7 +171,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MM:SS') AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -227,7 +223,14 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue //.ResultType = EntitiesResult, switch (entity) { .scrobble => - \\none + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\ORDER BY scrobbles.datetime ASC , .song => \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles From d638fa66c550e79ccd04101e093b06b5d56970a2 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 29 May 2025 15:33:10 -0400 Subject: [PATCH 53/71] Create GET function for a song view --- src/app/views/songs.zig | 17 ++++++- src/app/views/songs/get.zmpl | 22 +++++++-- src/queries.zig | 88 +++++++++++++++++++++++++++++------- 3 files changed, 106 insertions(+), 21 deletions(-) diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index d5a7944..3143f01 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -12,6 +12,21 @@ pub fn index(request: *jetzig.Request) !jetzig.View { } pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { - _ = id; + var root = try request.data(.object); + + const song = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_info), .{id}, .object); + try root.put("song", song.get("entity_info")); + + const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_items), .{id}, .array); + try root.put("scrobbles", scrobbles); + + const appears = try queries.entityQueryResult(request, queries.generateQuery(.song, .appears), .{id}, .array); + try root.put("appears", appears); + + const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.song, .firstlast), .{id}, .array); + try root.put("firstlast", firstlast); + + const timescale = try queries.entityQueryResult(request, queries.generateQuery(.song, .timescale), .{id}, .array); + try root.put("yearly", timescale); return request.render(.ok); } diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 76457d0..0a051a4 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -1,3 +1,19 @@ -
- Content goes here -
+@zig { + const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; + const columns: ColumnChoices = &.{.song, .artistlist, .album, .date}; +} + + + + + + +@partial partials/header +

{{.song}}

+@partial partials/firstlast_listens(scrobbles: .song.scrobbles, rank: .song.rank, firstlast: .firstlast) +

Yearly Performance

+@partial partials/timescale(range: .yearly) +

Scrobbles

+@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) + + \ No newline at end of file diff --git a/src/queries.zig b/src/queries.zig index faba14d..061b5b3 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -20,7 +20,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: blk: while (try result.postgresql.result.next()) |entity_row| { switch (query.ResultType) { - FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult => |T| { + FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult, EntityItemsSongResult => |T| { const entity = try entity_row.to(T, .{ .dupe = true, .allocator = request.allocator }); const item: ?TableRow = switch (query.ResultType) { EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { @@ -39,7 +39,8 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: }; }, }, - EntityItemsResult => switch (query.entity) { + EntityItemsResult, EntityItemsSongResult => switch (query.entity) { + .song => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .date = entity.date }, .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, .album => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, else => unreachable, @@ -92,6 +93,17 @@ const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, alb const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; +const EntityItemsSongResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, date: []const u8 }; +const UnifiedResult = struct { + album_name: ?[]const u8 = null, + album_id: ?i32 = null, + song_name: ?[]const u8 = null, + song_id: ?i32 = null, + artist_names: ?[]const []const u8 = null, + artist_ids: ?[]i32 = null, + scrobbles: ?i64 = null, + date: ?[]const u8 = null, +}; pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { return GeneratedQuery{ @@ -106,7 +118,11 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue .album => EntitiesAlbumResult, .artist => EntitiesArtistResult, }, - .entity_items => EntityItemsResult, + .entity_items => switch (entity) { + .scrobble => unreachable, + .song => EntityItemsSongResult, + else => EntityItemsResult, + }, .appears => AppearsResult, .entity_info => EntityInfoResult, }, @@ -116,23 +132,21 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime,'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 + \\WHERE albumsongs.song_id = $1 \\ORDER BY scrobbles.datetime ASC \\LIMIT 1) \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, scrobbles.datetime AS date + \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongsartists.artist_id = $1 + \\WHERE albumsongs.song_id = $1 \\ORDER BY scrobbles.datetime DESC \\LIMIT 1) , @@ -190,9 +204,8 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\WHERE artists.id = $1 + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\WHERE songs.id = $1 \\GROUP BY year \\ORDER BY year ASC; , @@ -268,13 +281,21 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\ NOTHING YET + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongs.song_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC; , .album => \\nope , .artist => - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums \\INNER JOIN albums ON albums.id = artistalbums.album_id \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id @@ -291,7 +312,13 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\get all scrobbles + \\SELECT songs.name, songs.id, albums.name, albums.id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE songs.id = $1 + \\ORDER BY date ASC , .album => \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles @@ -317,12 +344,19 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue .entity_info => switch (entity) { .scrobble => unreachable, .song => - \\lol idk + \\SELECT * FROM + \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM99999th') AS rank FROM + \\(SELECT songs.name AS name, songs.id AS id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\GROUP BY songs.id)) + \\WHERE id = $1 , .album => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT albums.name, albums.id, COUNT(scrobbles) AS scrobbles + \\(SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id @@ -341,6 +375,26 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\WHERE id = $1 , }, + + .datestreak => switch (entity) { + .song => + \\SELECT songs.name, albums.name, streak.maxseq, streak.ds, streak.de FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT albumsong, grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\COUNT(DISTINCT datetime::date) AS numdays + \\FROM (SELECT scrobbles.*, + \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY albumsong ORDER BY datetime::date)) AS grp + \\FROM scrobbles + \\) scrobbles + \\GROUP BY albumsong, grp + \\) scrobbles + \\GROUP BY albumsong, ds, de) AS streak + \\INNER JOIN albumsongs ON albumsongs.id = streak.albumsong + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\ORDER BY streak.maxseq DESC; + , + else => unreachable, + }, }, }; } From 62590fee374fab21c36976724d014a5a1b15f787 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 29 May 2025 19:39:51 -0400 Subject: [PATCH 54/71] Made queries.zig look significantly nicer There's a little bit of weird stuff happening, but holy cannoli, that's so much easier to maintain and parse --- src/app/views/albums.zig | 12 +- src/app/views/artists.zig | 14 +- .../views/partials/_firstlast_listens.zmpl | 4 +- src/app/views/partials/_timescale.zmpl | 2 +- src/app/views/scrobbles.zig | 2 +- src/app/views/songs.zig | 14 +- src/queries.zig | 179 ++++++------------ 7 files changed, 87 insertions(+), 140 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index af72456..713e6d6 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -8,7 +8,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const albums = try queries.entityQueryResult(request, queries.generateQuery(.album, .entities), .{}, .array); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.album, .entities), .{}); try root.put("albums", albums); return request.render(.ok); @@ -17,16 +17,16 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const album = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_info), .{id}, .object); - try root.put("album", album.get("entity_info")); + const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id}); + try root.put("album", album); - const songs = try queries.entityQueryResult(request, queries.generateQuery(.album, .entity_items), .{id}, .array); + const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_items), .{id}); try root.put("songs", songs); - const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.album, .firstlast), .{id}, .array); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.generateQuery(.album, .timescale), .{id}, .array); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.album, .timescale), .{id}); try root.put("yearly", timescale); return request.render(.ok); diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 37ca155..1f7ff85 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -8,7 +8,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const artists = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entities), .{}, .array); + const artists = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entities), .{}); try root.put("artists", artists); @@ -18,19 +18,19 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const artist = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_info), .{id}, .object); - try root.put("artist", artist.get("entity_info")); + const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id}); + try root.put("artist", artist); - const albums = try queries.entityQueryResult(request, queries.generateQuery(.artist, .entity_items), .{id}, .array); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_items), .{id}); try root.put("albums", albums); - const appears = try queries.entityQueryResult(request, queries.generateQuery(.artist, .appears), .{id}, .array); + const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id}); try root.put("appears", appears); - const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.artist, .firstlast), .{id}, .array); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.artist, .firstlast), .{id}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.generateQuery(.artist, .timescale), .{id}, .array); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.artist, .timescale), .{id}); try root.put("yearly", timescale); return request.render(.ok); diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl index ed74361..a21114b 100644 --- a/src/app/views/partials/_firstlast_listens.zmpl +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -7,7 +7,7 @@
{{scrobbles}} scrobbles ({{rank}} place)
-First listen: {{songs[0].name}} ({{songs[0].date}}) +First listen: {{songs[0].song.name}} ({{songs[0].date}})
-Most recent listen: {{songs[1].name}} ({{songs[1].date}}) +Most recent listen: {{songs[1].song.name}} ({{songs[1].date}})
\ No newline at end of file diff --git a/src/app/views/partials/_timescale.zmpl b/src/app/views/partials/_timescale.zmpl index fdec7a1..9937ec6 100644 --- a/src/app/views/partials/_timescale.zmpl +++ b/src/app/views/partials/_timescale.zmpl @@ -10,7 +10,7 @@ @for (range) |itm| { - {{itm.year}}: + {{itm.date}}: {{itm.scrobbles}} } diff --git a/src/app/views/scrobbles.zig b/src/app/views/scrobbles.zig index e61df4d..754a7c8 100644 --- a/src/app/views/scrobbles.zig +++ b/src/app/views/scrobbles.zig @@ -5,7 +5,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.scrobble, .entities), .{}, .array); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.scrobble, .entities), .{}); try root.put("scrobbles", scrobbles); return request.render(.ok); diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index 3143f01..b24dbf5 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -5,7 +5,7 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const songs = try queries.entityQueryResult(request, queries.generateQuery(.song, .entities), .{}, .array); + const songs = try queries.entityQueryResult(request, queries.loadQuery(.song, .entities), .{}); try root.put("songs", songs); return request.render(.ok); @@ -14,19 +14,19 @@ pub fn index(request: *jetzig.Request) !jetzig.View { pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const song = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_info), .{id}, .object); - try root.put("song", song.get("entity_info")); + const song = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_info), .{id}); + try root.put("song", song); - const scrobbles = try queries.entityQueryResult(request, queries.generateQuery(.song, .entity_items), .{id}, .array); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_items), .{id}); try root.put("scrobbles", scrobbles); - const appears = try queries.entityQueryResult(request, queries.generateQuery(.song, .appears), .{id}, .array); + const appears = try queries.entityQueryResult(request, queries.loadQuery(.song, .appears), .{id}); try root.put("appears", appears); - const firstlast = try queries.entityQueryResult(request, queries.generateQuery(.song, .firstlast), .{id}, .array); + const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.song, .firstlast), .{id}); try root.put("firstlast", firstlast); - const timescale = try queries.entityQueryResult(request, queries.generateQuery(.song, .timescale), .{id}, .array); + const timescale = try queries.entityQueryResult(request, queries.loadQuery(.song, .timescale), .{id}); try root.put("yearly", timescale); return request.render(.ok); } diff --git a/src/queries.zig b/src/queries.zig index 061b5b3..0411022 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -4,135 +4,82 @@ const TableRow = @import("types.zig").TableRow; const HyperlinkData = @import("types.zig").HyperlinkData; const std = @import("std"); -pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: anytype, root_type: enum { object, array }) !*jetzig.Data.Value { - var result = try request.repo.executeSql(query.query, args); +pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: anytype) !*jetzig.Data.Value { + //var result = try request.repo.executeSql(query.query, args); + // + var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, args, .{ .allocator = request.allocator, .column_names = true }); defer result.deinit(); var Data = jetzig.Data.init(request.allocator); - var out: *jetzig.Data.Value = switch (root_type) { - .array => try Data.array(), - .object => try Data.object(), - }; - var artist_list = if (query.ResultType == EntitiesAlbumResult or query.ResultType == EntitiesArtistResult or query.ResultType == EntitiesScrobbleResult or query.ResultType == EntitiesSongResult) - std.ArrayList(HyperlinkData).init(request.allocator); + var artist_list = std.ArrayList(HyperlinkData).init(request.allocator); - blk: while (try result.postgresql.result.next()) |entity_row| { - switch (query.ResultType) { - FirstlastResult, TimescaleResult, EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult, EntityItemsResult, AppearsResult, EntityInfoResult, EntityItemsSongResult => |T| { - const entity = try entity_row.to(T, .{ .dupe = true, .allocator = request.allocator }); - const item: ?TableRow = switch (query.ResultType) { - EntitiesScrobbleResult, EntitiesSongResult, EntitiesAlbumResult, EntitiesArtistResult => switch (query.entity) { - .artist => TableRow{ .artist = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .scrobble, .song, .album => album_entities: { - const last_artist = artist_list.getLastOrNull(); - try artist_list.append(.{ .name = entity.artist_name, .id = entity.artist_id }); - if (last_artist) |la| { - if (la.id == entity.artist_id) continue :blk; - } - break :album_entities switch (query.entity) { - .scrobble => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .artistlist = try artist_list.toOwnedSlice(), .date = entity.date }, - .song => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, - .album => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .artistlist = try artist_list.toOwnedSlice(), .scrobbles = entity.scrobbles }, - else => unreachable, - }; - }, - }, - EntityItemsResult, EntityItemsSongResult => switch (query.entity) { - .song => TableRow{ .song = .{ .name = entity.song_name, .id = entity.song_id }, .album = .{ .name = entity.album_name, .id = entity.album_id }, .date = entity.date }, - .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .album => TableRow{ .song = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - else => unreachable, - }, - AppearsResult => switch (query.entity) { - .song, .artist => TableRow{ .album = .{ .name = entity.name, .id = entity.id }, .scrobbles = entity.scrobbles }, - .album, .scrobble => unreachable, - }, - else => null, - }; - - if (item) |itm| { - switch (root_type) { - .array => try out.append(itm), - .object => switch (query.query_type) { - inline else => |qt| try out.put(@tagName(qt), itm), - }, - } - } else { - switch (root_type) { - .array => try out.append(entity), - .object => switch (query.query_type) { - inline else => |qt| try out.put(@tagName(qt), entity), - }, - } - } - }, - else => unreachable, - } + if (query.query_type == .entity_info) { + var out: *jetzig.Data.Value = try Data.object(); + const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); + const entity_struct = .{ .name = entity.name, .id = entity.id, .scrobbles = entity.scrobbles, .rank = entity.rank }; + try out.put("entity_info", entity_struct); + try result.drain(); + return out.get("entity_info").?; } - if (root_type == .object) {} + + var out: *jetzig.Data.Value = try Data.array(); + var mapper = result.mapper(UnifiedResult, .{ .dupe = true, .allocator = request.allocator }); + + blk: while (try mapper.next()) |entity| { + if (entity.artist_id) |_| { + const last_artist = artist_list.getLastOrNull(); + try artist_list.append(.{ .name = entity.artist_name.?, .id = entity.artist_id.? }); + if (last_artist) |la| { + if (la.id == entity.artist_id) continue :blk; + } + } + + try out.append(TableRow{ + .artist = if (entity.artist_id) |_| .{ .id = entity.artist_id.?, .name = entity.artist_name.? } else null, + .album = if (entity.album_id) |_| .{ .id = entity.album_id.?, .name = entity.album_name.? } else null, + .song = if (entity.song_id) |_| .{ .id = entity.song_id.?, .name = entity.song_name.? } else null, + .artistlist = if (artist_list.getLastOrNull()) |_| try artist_list.toOwnedSlice() else null, + .scrobbles = if (entity.scrobbles) |scrobbles| scrobbles else null, + .date = if (entity.date) |date| date else null, + }); + } + return out; } +const EntityType = enum { scrobble, song, album, artist }; +const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak }; + const GeneratedQuery = struct { entity: EntityType, query_type: QueryTypeEnum, - ResultType: type, query: []const u8, }; -const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info }; -const FirstlastResult = struct { name: []const u8, id: i32, date: []const u8 }; -const TimescaleResult = struct { year: []const u8, scrobbles: i64 }; -const EntitiesArtistResult = struct { name: []const u8, id: i32, scrobbles: i64 }; -const EntitiesAlbumResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; -const EntitiesSongResult = struct { name: []const u8, id: i32, artist_name: []const u8, artist_id: i32, scrobbles: i64 }; -const EntitiesScrobbleResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, artist_name: []const u8, artist_id: i32, date: []const u8 }; -const EntityItemsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; -const AppearsResult = struct { name: []const u8, id: i32, scrobbles: i64 }; -const EntityInfoResult = struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }; -const EntityItemsSongResult = struct { song_name: []const u8, song_id: i32, album_name: []const u8, album_id: i32, date: []const u8 }; const UnifiedResult = struct { album_name: ?[]const u8 = null, album_id: ?i32 = null, song_name: ?[]const u8 = null, song_id: ?i32 = null, - artist_names: ?[]const []const u8 = null, - artist_ids: ?[]i32 = null, + artist_name: ?[]const u8 = null, + artist_id: ?i32 = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, }; -pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { +pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { return GeneratedQuery{ .entity = entity, .query_type = query_type, - .ResultType = switch (query_type) { - .firstlast => FirstlastResult, - .timescale => TimescaleResult, - .entities => switch (entity) { - .scrobble => EntitiesScrobbleResult, - .song => EntitiesSongResult, - .album => EntitiesAlbumResult, - .artist => EntitiesArtistResult, - }, - .entity_items => switch (entity) { - .scrobble => unreachable, - .song => EntityItemsSongResult, - else => EntityItemsResult, - }, - .appears => AppearsResult, - .entity_info => EntityInfoResult, - }, .query = switch (query_type) { .firstlast => //.ResultType = FirstlastResult, switch (entity) { .scrobble => unreachable, .song => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime,'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime,'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -142,7 +89,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -152,7 +99,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .album => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -163,7 +110,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -174,7 +121,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue , .artist => - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -185,7 +132,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ \\UNION ALL \\ - \\(SELECT songs.name AS name, songs.id AS id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\(SELECT songs.name AS song_name, songs.id AS song_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN songs ON songs.id = albumsongs.song_id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -201,34 +148,34 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN songs ON songs.id = albumsongs.song_id \\WHERE songs.id = $1 - \\GROUP BY year - \\ORDER BY year ASC; + \\GROUP BY date + \\ORDER BY date ASC; , .album => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN albums ON albums.id = albumsongs.album_id \\WHERE albums.id = $1 - \\GROUP BY year - \\ORDER BY year ASC; + \\GROUP BY date + \\ORDER BY date ASC; , .artist => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS year, COUNT(*) as scrobbles + \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id \\WHERE artists.id = $1 - \\GROUP BY year - \\ORDER BY year ASC; + \\GROUP BY date + \\ORDER BY date ASC; , }, @@ -246,7 +193,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles.datetime ASC , .song => - \\SELECT songs.name, songs.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\SELECT songs.name AS song_name, songs.id AS song_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id @@ -256,7 +203,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles DESC, songs.name ASC , .album => - \\SELECT albums.name, albums.id, artists.name, artists.id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON albumsongs.id = scrobbles.albumsong @@ -266,7 +213,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles DESC , .artist => - \\SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\SELECT artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongsartists \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id @@ -312,7 +259,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue switch (entity) { .scrobble => unreachable, .song => - \\SELECT songs.name, songs.id, albums.name, albums.id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date \\FROM albumsongs \\INNER JOIN albums ON albums.id = albumsongs.album_id \\INNER JOIN songs ON songs.id = albumsongs.song_id @@ -321,7 +268,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY date ASC , .album => - \\SELECT songs.name, songs.id, COUNT(scrobbles) AS scrobbles + \\SELECT songs.name AS song_name, songs.id AS song_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id @@ -330,7 +277,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue \\ORDER BY scrobbles DESC , .artist => - \\SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums \\INNER JOIN albums ON albums.id = artistalbums.album_id \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id @@ -378,7 +325,7 @@ pub fn generateQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQue .datestreak => switch (entity) { .song => - \\SELECT songs.name, albums.name, streak.maxseq, streak.ds, streak.de FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de + \\SELECT songs.name AS song_name, albums.name AS album_name, streak.maxseq AS intval, FORMAT('%s - %s, streak.ds, streak.de) AS date FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de \\FROM (SELECT albumsong, grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, \\COUNT(DISTINCT datetime::date) AS numdays \\FROM (SELECT scrobbles.*, From d81681e6983df014a405d4a3c2fb57362f19011d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 13:37:34 -0400 Subject: [PATCH 55/71] Move scrobble rank from firstlast partial to view. Eventually moving this to its own partial (probably) --- src/app/views/albums/get.zmpl | 6 ++++-- src/app/views/artists/get.zmpl | 9 +++++++-- src/app/views/partials/_firstlast_listens.zmpl | 4 +--- src/app/views/partials/_timescale.zmpl | 18 ++++++++++-------- src/app/views/songs/get.zmpl | 5 +++-- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 1024ec5..22f99d8 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -9,8 +9,10 @@ @partial partials/header -

{{.album}}

-@partial partials/firstlast_listens(scrobbles: .album.scrobbles, rank: .album.rank, firstlast: .firstlast) +

{{.album.name}}

+
{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
+
{{.album.song_num}} songs
+@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly)

Songs

diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 69bd661..14976b5 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -10,9 +10,14 @@ @partial partials/header

{{.artist.name}}

-@partial partials/firstlast_listens(scrobbles: .artist.scrobbles, rank: .artist.rank, firstlast: .firstlast) -

Yearly Performance

+
+
{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
+
{{.artist.song_num}} songs
+
{{.artist.album_num}} albums
+
@partial partials/timescale(range: .yearly) +
+@partial partials/firstlast_listens(firstlast: .firstlast)

Albums

@partial partials/newtable(T: ColumnChoices, table_data: .albums, columns: columns) diff --git a/src/app/views/partials/_firstlast_listens.zmpl b/src/app/views/partials/_firstlast_listens.zmpl index a21114b..3cf3ac9 100644 --- a/src/app/views/partials/_firstlast_listens.zmpl +++ b/src/app/views/partials/_firstlast_listens.zmpl @@ -1,12 +1,10 @@ -@args scrobbles: i64, rank: []const u8, firstlast: *ZmplValue +@args firstlast: *ZmplValue @zig { const songs = firstlast.items(.array); }
-{{scrobbles}} scrobbles ({{rank}} place) -
First listen: {{songs[0].song.name}} ({{songs[0].date}})
Most recent listen: {{songs[1].song.name}} ({{songs[1].date}}) diff --git a/src/app/views/partials/_timescale.zmpl b/src/app/views/partials/_timescale.zmpl index 9937ec6..24ef925 100644 --- a/src/app/views/partials/_timescale.zmpl +++ b/src/app/views/partials/_timescale.zmpl @@ -1,5 +1,6 @@ @args range: *ZmplValue +
@@ -8,11 +9,12 @@ -@for (range) |itm| { - - - - -} - -
{{itm.date}}:{{itm.scrobbles}}
\ No newline at end of file + @for (range) |itm| { + + {{itm.date}}: + {{itm.scrobbles}} + + } + + +
\ No newline at end of file diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 0a051a4..5b68145 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -9,8 +9,9 @@ @partial partials/header -

{{.song}}

-@partial partials/firstlast_listens(scrobbles: .song.scrobbles, rank: .song.rank, firstlast: .firstlast) +

{{.song.name}}

+
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
+@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly)

Scrobbles

From c57bf186273c34be8b19093db67a1a06afde9622 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 13:39:03 -0400 Subject: [PATCH 56/71] Update queries Adds datestreak query, provides the number of songs/albums when relevant, and provides timescale with all years, regardless of the number of plays (defaults to 0) --- src/queries.zig | 75 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/src/queries.zig b/src/queries.zig index 0411022..a22b68e 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -17,13 +17,18 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: if (query.query_type == .entity_info) { var out: *jetzig.Data.Value = try Data.object(); - const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); - const entity_struct = .{ .name = entity.name, .id = entity.id, .scrobbles = entity.scrobbles, .rank = entity.rank }; - try out.put("entity_info", entity_struct); + const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, album_num: ?i64 = null, song_num: ?i64 = null, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator, .map = .name }); + try out.put("entity_info", entity); try result.drain(); return out.get("entity_info").?; } + //if (query.query_type == .datestreak) { + // var out: *jetzig.Data.Value = try Data.object(); + // const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); + // + //} + var out: *jetzig.Data.Value = try Data.array(); var mapper = result.mapper(UnifiedResult, .{ .dupe = true, .allocator = request.allocator }); @@ -168,14 +173,17 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , .artist => - \\SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles + \\SELECT y.year AS date, COALESCE(DT.scrobbles, 0) AS scrobbles + \\FROM (SELECT GENERATE_SERIES(2016,date_part('year',now())::int)::text) AS y(year) + \\LEFT JOIN (SELECT TO_CHAR(date_trunc('year', datetime), 'YYYY') AS date, COUNT(*) as scrobbles \\FROM scrobbles \\INNER JOIN albumsongs ON albumsongs.id = scrobbles.albumsong \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id \\WHERE artists.id = $1 \\GROUP BY date - \\ORDER BY date ASC; + \\ORDER BY date ASC) AS DT + \\ON DT.date = y.year; , }, @@ -303,21 +311,24 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .album => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT albums.name AS name, albums.id AS id, COUNT(scrobbles) AS scrobbles + \\(SELECT albums.name AS name, albums.id AS id, COUNT(DISTINCT songs.id) AS song_num, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN songs ON songs.id = albumsongs.song_id \\GROUP BY albums.id)) - \\WHERE id = $1 + \\WHERE id = $1; , .artist => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles + \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles, COUNT(DISTINCT albums.id) AS album_num, COUNT(DISTINCT songs.id) AS song_num \\FROM albumsongsartists \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id \\GROUP BY artists.id)) \\WHERE id = $1 , @@ -325,22 +336,44 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .datestreak => switch (entity) { .song => - \\SELECT songs.name AS song_name, albums.name AS album_name, streak.maxseq AS intval, FORMAT('%s - %s, streak.ds, streak.de) AS date FROM (SELECT albumsong, MAX(numdays) AS maxseq, ds, de - \\FROM (SELECT albumsong, grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\SELECT maxseq AS streak, FORMAT('%s - %s', ds, de) AS date FROM (SELECT MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, \\COUNT(DISTINCT datetime::date) AS numdays - \\FROM (SELECT scrobbles.*, - \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY albumsong ORDER BY datetime::date)) AS grp - \\FROM scrobbles + \\FROM (SELECT scrobbles.datetime, + \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY songs.id ORDER BY datetime::date)) AS grp + \\FROM scrobbles INNER JOIN albumsongs ON albumsongs.id = albumsong INNER JOIN songs ON songs.id = albumsongs.song_id + \\WHERE songs.id = $1) scrobbles + \\GROUP BY grp \\) scrobbles - \\GROUP BY albumsong, grp - \\) scrobbles - \\GROUP BY albumsong, ds, de) AS streak - \\INNER JOIN albumsongs ON albumsongs.id = streak.albumsong - \\INNER JOIN albums ON albums.id = albumsongs.album_id - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\ORDER BY streak.maxseq DESC; + \\GROUP BY ds, de) + \\ORDER BY maxseq DESC; , - else => unreachable, + .artist => + \\SELECT maxseq AS streak, FORMAT('%s - %s' ,ds,de) AS date FROM (SELECT MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\COUNT(DISTINCT datetime::date) AS numdays + \\FROM (SELECT scrobbles.datetime, + \\((scrobbles.datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY artists.id ORDER BY scrobbles.datetime::date)) AS grp + \\FROM scrobbles INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = scrobbles.albumsong INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\WHERE artists.id = $1) scrobbles + \\GROUP BY grp + \\) scrobbles + \\GROUP BY ds, de) + \\ORDER BY maxseq DESC; + , + .album => + \\SELECT maxseq AS streak, FORMAT('%s - %s', ds, de) AS date FROM (SELECT MAX(numdays) AS maxseq, ds, de + \\FROM (SELECT grp, MIN(datetime::date) AS ds, MAX(datetime::date) AS de, + \\COUNT(DISTINCT datetime::date) AS numdays + \\FROM (SELECT scrobbles.datetime, + \\((datetime::date - '1970-01-01'::date) - DENSE_RANK() OVER (PARTITION BY albums.id ORDER BY datetime::date)) AS grp FROM scrobbles INNER JOIN albumsongs ON albumsongs.id = albumsong INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE albums.id = $1) scrobbles + \\GROUP BY grp + \\) scrobbles + \\GROUP BY ds, de) + \\ORDER BY maxseq DESC; + , + .scrobble => unreachable, }, }, }; From a314fd447d4b7e21df030e57c349eb7bfdf0ca2d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 14:31:15 -0400 Subject: [PATCH 57/71] Fix LastFM API scrobble parsing when song is currently playing --- src/app/views/upload.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index bdb5cef..ebe9395 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -103,7 +103,8 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } }, .LastFMWeb => |scrobbles| { - for (scrobbles) |scrobble| { + appends: for (scrobbles) |scrobble| { + if (scrobble.date == null) continue :appends; const filtered_scrobble = Data.Scrobble{ .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, From 3777b818e3aae4fbe9489084f9770bf81ff2fe41 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 14:45:45 -0400 Subject: [PATCH 58/71] Create view for groups One of the largest components that makes zuletzt unique - implementing groups the way MusicBrainz has (release groups in particular). I thought for a while that I would just connect songs via a shared ID, but for remixes and such, I don't think they should be so tightly coupled. This also gives the user freedom for how they want to do the grouping (a remix can be included in the group if they choose to, or it may not). This will allow someone to see a combined scrobble number for an album with, for example, a regular release, a deluxe release, and an anniversary release, in addition to the individual releases. This will complicate SQL queries rather significantly I imagine, and I'm not sure what the interface for creating/deleting groups will be (although it will likely be easier when I have full use of TS), but it's a necessity for the project. --- src/app/views/groups.zig | 104 +++++++++++++++++++++++++++++++ src/app/views/groups/delete.zmpl | 3 + src/app/views/groups/edit.zmpl | 3 + src/app/views/groups/get.zmpl | 3 + src/app/views/groups/index.zmpl | 3 + src/app/views/groups/new.zmpl | 3 + src/app/views/groups/patch.zmpl | 3 + src/app/views/groups/post.zmpl | 3 + src/app/views/groups/put.zmpl | 3 + 9 files changed, 128 insertions(+) create mode 100644 src/app/views/groups.zig create mode 100644 src/app/views/groups/delete.zmpl create mode 100644 src/app/views/groups/edit.zmpl create mode 100644 src/app/views/groups/get.zmpl create mode 100644 src/app/views/groups/index.zmpl create mode 100644 src/app/views/groups/new.zmpl create mode 100644 src/app/views/groups/patch.zmpl create mode 100644 src/app/views/groups/post.zmpl create mode 100644 src/app/views/groups/put.zmpl diff --git a/src/app/views/groups.zig b/src/app/views/groups.zig new file mode 100644 index 0000000..85505c8 --- /dev/null +++ b/src/app/views/groups.zig @@ -0,0 +1,104 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +pub fn index(request: *jetzig.Request) !jetzig.View { + return request.render(.ok); +} + +pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn new(request: *jetzig.Request) !jetzig.View { + return request.render(.ok); +} + +pub fn edit(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn post(request: *jetzig.Request) !jetzig.View { + return request.render(.created); +} + +pub fn put(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn patch(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + +pub fn delete(id: []const u8, request: *jetzig.Request) !jetzig.View { + _ = id; + return request.render(.ok); +} + + +test "index" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups", .{}); + try response.expectStatus(.ok); +} + +test "get" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} + +test "new" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups/new", .{}); + try response.expectStatus(.ok); +} + +test "edit" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.GET, "/groups/example-id/edit", .{}); + try response.expectStatus(.ok); +} + +test "post" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.POST, "/groups", .{}); + try response.expectStatus(.created); +} + +test "put" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.PUT, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} + +test "patch" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.PATCH, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} + +test "delete" { + var app = try jetzig.testing.app(std.testing.allocator, @import("routes")); + defer app.deinit(); + + const response = try app.request(.DELETE, "/groups/example-id", .{}); + try response.expectStatus(.ok); +} diff --git a/src/app/views/groups/delete.zmpl b/src/app/views/groups/delete.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/delete.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/edit.zmpl b/src/app/views/groups/edit.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/edit.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/get.zmpl b/src/app/views/groups/get.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/get.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/index.zmpl b/src/app/views/groups/index.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/index.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/new.zmpl b/src/app/views/groups/new.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/new.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/patch.zmpl b/src/app/views/groups/patch.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/patch.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/post.zmpl b/src/app/views/groups/post.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/post.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
diff --git a/src/app/views/groups/put.zmpl b/src/app/views/groups/put.zmpl new file mode 100644 index 0000000..76457d0 --- /dev/null +++ b/src/app/views/groups/put.zmpl @@ -0,0 +1,3 @@ +
+ Content goes here +
From 906ba6d2e5f41dc9ad16ea65a6868fb598a1ce19 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 14:47:52 -0400 Subject: [PATCH 59/71] Update header partial and remoev table partial Long overdue --- src/app/views/partials/_header.zmpl | 4 ++++ src/app/views/partials/_table.zmpl | 22 ---------------------- 2 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 src/app/views/partials/_table.zmpl diff --git a/src/app/views/partials/_header.zmpl b/src/app/views/partials/_header.zmpl index bf80e53..3f9089d 100644 --- a/src/app/views/partials/_header.zmpl +++ b/src/app/views/partials/_header.zmpl @@ -1,7 +1,11 @@ Zuletzt +Artists +Albums +Songs Scrobbles Concerts Collection Ratings Lists +Groups
\ No newline at end of file diff --git a/src/app/views/partials/_table.zmpl b/src/app/views/partials/_table.zmpl deleted file mode 100644 index 83c1cd4..0000000 --- a/src/app/views/partials/_table.zmpl +++ /dev/null @@ -1,22 +0,0 @@ -@args table_data: *ZmplValue, table_headers: *ZmplValue - - - -@for (table_headers) |text| { - -} - - - @for (table_data) |value| { - - - - - - - } -
{{text}}
{{value.track}} - @for (value.get("artists").?) |artist| { - {{artist}} - } - {{value.album}}{{value.date}}
\ No newline at end of file From 566edf18189391559576af803f73c956951a3e91 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 31 May 2025 15:48:30 -0400 Subject: [PATCH 60/71] Include artist(s) name in album GET view This also makes the entity_info struct very similar to the UnifiedResult struct, so we'll probably see a merge at some point. Would be nice if we used the fields from the entity_info result more commonly. --- src/app/views/albums/get.zmpl | 3 ++- src/app/views/artists/get.zmpl | 2 +- src/app/views/songs/get.zmpl | 2 +- src/queries.zig | 34 +++++++++++++++++++++++++--------- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index 22f99d8..7e089e7 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -9,7 +9,8 @@ @partial partials/header -

{{.album.name}}

+

{{.album.album_name}}

+

{{.album.artist_name}}

{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
{{.album.song_num}} songs
@partial partials/firstlast_listens(firstlast: .firstlast) diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 14976b5..9f1ff07 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -9,7 +9,7 @@ @partial partials/header -

{{.artist.name}}

+

{{.artist.artist_name}}

{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
{{.artist.song_num}} songs
diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 5b68145..e912930 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -9,7 +9,7 @@ @partial partials/header -

{{.song.name}}

+

{{.song.song_name}}

{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

diff --git a/src/queries.zig b/src/queries.zig index a22b68e..194ef7f 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -17,7 +17,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: if (query.query_type == .entity_info) { var out: *jetzig.Data.Value = try Data.object(); - const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, album_num: ?i64 = null, song_num: ?i64 = null, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator, .map = .name }); + const entity = try (try result.next()).?.to(EntityInfoResult, .{ .dupe = true, .allocator = request.allocator, .map = .name }); try out.put("entity_info", entity); try result.drain(); return out.get("entity_info").?; @@ -74,6 +74,20 @@ const UnifiedResult = struct { date: ?[]const u8 = null, }; +const EntityInfoResult = struct { + album_name: ?[]const u8 = null, + album_id: ?i32 = null, + song_name: ?[]const u8 = null, + song_id: ?i32 = null, + artist_name: ?[]const u8 = null, + artist_id: ?i32 = null, + scrobbles: ?i64 = null, + date: ?[]const u8 = null, + rank: []const u8, + song_num: ?i64 = null, + album_num: ?i64 = null, +}; + pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { return GeneratedQuery{ .entity = entity, @@ -301,28 +315,30 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .song => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM99999th') AS rank FROM - \\(SELECT songs.name AS name, songs.id AS id, COUNT(scrobbles) AS scrobbles + \\(SELECT songs.name AS song_name, songs.id AS song_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id \\GROUP BY songs.id)) - \\WHERE id = $1 + \\WHERE song_id = $1 , .album => - \\SELECT * FROM + \\SELECT album_name, t.album_id, artists.name AS artist_name, artists.id AS artist_id, song_num, scrobbles, rank FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT albums.name AS name, albums.id AS id, COUNT(DISTINCT songs.id) AS song_num, COUNT(scrobbles) AS scrobbles + \\(SELECT albums.name AS album_name, albums.id AS album_id, COUNT(DISTINCT songs.id) AS song_num, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN albums ON albumsongs.album_id = albums.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\GROUP BY albums.id)) - \\WHERE id = $1; + \\GROUP BY albums.id)) AS t + \\INNER JOIN artistalbums ON artistalbums.album_id = t.album_id + \\INNER JOIN artists ON artists.id = artistalbums.artist_id + \\WHERE t.album_id = $1 , .artist => \\SELECT * FROM \\(SELECT *, TO_CHAR(ROW_NUMBER() OVER (ORDER BY scrobbles DESC),'FM9999th') AS rank FROM - \\(SELECT artists.name AS name, artists.id AS id, COUNT(scrobbles) AS scrobbles, COUNT(DISTINCT albums.id) AS album_num, COUNT(DISTINCT songs.id) AS song_num + \\(SELECT artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles, COUNT(DISTINCT albums.id) AS album_num, COUNT(DISTINCT songs.id) AS song_num \\FROM albumsongsartists \\INNER JOIN artists ON albumsongsartists.artist_id = artists.id \\INNER JOIN albumsongs ON albumsongsartists.albumsong_id = albumsongs.id @@ -330,7 +346,7 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\INNER JOIN albums ON albums.id = albumsongs.album_id \\INNER JOIN songs ON songs.id = albumsongs.song_id \\GROUP BY artists.id)) - \\WHERE id = $1 + \\WHERE artist_id = $1 , }, From adcaff34ea5726fb9e1e3e87394ecaea136349f5 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 2 Jun 2025 00:13:27 -0400 Subject: [PATCH 61/71] Fix dumb appears query for albums --- src/queries.zig | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/queries.zig b/src/queries.zig index 194ef7f..8db7809 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -248,7 +248,7 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { .appears => //.ResultType = AppearsResult, switch (entity) { - .scrobble => unreachable, + .scrobble, .album => unreachable, .song => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums @@ -260,9 +260,6 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\GROUP BY albums.id \\ORDER BY scrobbles DESC; , - .album => - \\nope - , .artist => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums From 3ef17fcd468d710186d0448424d15ea42786b395 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 6 Jun 2025 14:28:15 -0400 Subject: [PATCH 62/71] Split entity_items and appears query into more granular queries We can be a bit more specific about the information we get this way --- src/app/views/albums.zig | 2 +- src/app/views/artists.zig | 2 +- src/app/views/songs.zig | 6 +-- src/queries.zig | 91 +++++++++++++++++++++++++++------------ 4 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 713e6d6..03046b3 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -20,7 +20,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const album = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_info), .{id}); try root.put("album", album); - const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .entity_items), .{id}); + const songs = try queries.entityQueryResult(request, queries.loadQuery(.album, .get_songs), .{id}); try root.put("songs", songs); const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.album, .firstlast), .{id}); diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 1f7ff85..3ec8eba 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -21,7 +21,7 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const artist = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_info), .{id}); try root.put("artist", artist); - const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .entity_items), .{id}); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.artist, .get_albums), .{id}); try root.put("albums", albums); const appears = try queries.entityQueryResult(request, queries.loadQuery(.artist, .appears), .{id}); diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index b24dbf5..164f928 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -17,11 +17,11 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const song = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_info), .{id}); try root.put("song", song); - const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .entity_items), .{id}); + const scrobbles = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_scrobbles), .{id}); try root.put("scrobbles", scrobbles); - const appears = try queries.entityQueryResult(request, queries.loadQuery(.song, .appears), .{id}); - try root.put("appears", appears); + const albums = try queries.entityQueryResult(request, queries.loadQuery(.song, .get_albums), .{id}); + try root.put("albums", albums); const firstlast = try queries.entityQueryResult(request, queries.loadQuery(.song, .firstlast), .{id}); try root.put("firstlast", firstlast); diff --git a/src/queries.zig b/src/queries.zig index 8db7809..5d92869 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -55,7 +55,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: } const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak }; +const QueryTypeEnum = enum { firstlast, timescale, entities, get_songs, get_albums, get_scrobbles, appears, entity_info, datestreak }; const GeneratedQuery = struct { entity: EntityType, @@ -246,20 +246,9 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { }, .appears => - //.ResultType = AppearsResult, + // Not sure how I feel about this one switch (entity) { - .scrobble, .album => unreachable, - .song => - \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles - \\FROM artistalbums - \\INNER JOIN albums ON albums.id = artistalbums.album_id - \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id - \\WHERE albumsongs.song_id = $1 - \\GROUP BY albums.id - \\ORDER BY scrobbles DESC; - , + .scrobble, .song, .album => unreachable, .artist => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums @@ -273,19 +262,8 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , }, - .entity_items => - //.ResultType = EntityItemsResult, - switch (entity) { - .scrobble => unreachable, - .song => - \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date - \\FROM albumsongs - \\INNER JOIN albums ON albums.id = albumsongs.album_id - \\INNER JOIN songs ON songs.id = albumsongs.song_id - \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id - \\WHERE songs.id = $1 - \\ORDER BY date ASC - , + .get_songs => switch (entity) { + .scrobble, .song => unreachable, // Might be able to use this with SongGroups? .album => \\SELECT songs.name AS song_name, songs.id AS song_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs @@ -296,6 +274,34 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\ORDER BY scrobbles DESC , .artist => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albumsongsartists.artist_id = $1 + \\GROUP BY songs.id, albums.id + \\ORDER BY scrobbles DESC + , + }, + + .get_albums => + //.ResultType = EntityItemsResult, + switch (entity) { + .scrobble, .album => unreachable, // Might be able to use this with ReleaseGroups? + .song => + \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles + \\FROM artistalbums + \\INNER JOIN albums ON albums.id = artistalbums.album_id + \\INNER JOIN albumsongs ON albumsongs.album_id = albums.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\WHERE albumsongs.song_id = $1 + \\GROUP BY albums.id + \\ORDER BY scrobbles DESC + , + .artist => \\SELECT albums.name AS album_name, albums.id AS album_id, COUNT(scrobbles) AS scrobbles \\FROM artistalbums \\INNER JOIN albums ON albums.id = artistalbums.album_id @@ -307,6 +313,37 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , }, + .get_scrobbles => switch (entity) { + .scrobble => unreachable, + .song => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE songs.id = $1 + \\ORDER BY date ASC + , + .album => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE albums.id = $1 + \\ORDER BY date ASC + , + .artist => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, TO_CHAR(scrobbles.datetime, 'YYYY-MM-DD HH24:MI:SS') AS date + \\FROM albumsongs + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\INNER JOIN songs ON songs.id = albumsongs.song_id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\WHERE artists.id = $1 + \\ORDER BY date ASC + , + }, + .entity_info => switch (entity) { .scrobble => unreachable, .song => From c8f2ef57c8f1f8932f5faa1612be30ed4792d592 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Fri, 6 Jun 2025 15:55:20 -0400 Subject: [PATCH 63/71] Add some tyling to songs view This can (will) be easily replicated for the other views, I just first tested it on songs. I think this looks much nicer, and I'll probably roll with a layout similar to this for the other views, with some minor adjustments for each particular view. --- src/app/views/partials/_newtable.zmpl | 4 +++- src/app/views/songs/get.zmpl | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/app/views/partials/_newtable.zmpl b/src/app/views/partials/_newtable.zmpl index 6746a2a..f9dc618 100644 --- a/src/app/views/partials/_newtable.zmpl +++ b/src/app/views/partials/_newtable.zmpl @@ -1,5 +1,6 @@ @args T: type, table_data: *ZmplValue, columns: T +
@@ -70,4 +71,5 @@ } } -
\ No newline at end of file + +
\ No newline at end of file diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index e912930..1ceddef 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -9,12 +9,24 @@ @partial partials/header +

{{.song.song_name}}

-
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
-@partial partials/firstlast_listens(firstlast: .firstlast) -

Yearly Performance

-@partial partials/timescale(range: .yearly) -

Scrobbles

-@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) +
+ +
+
+
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
+ @partial partials/firstlast_listens(firstlast: .firstlast) +

Yearly Performance

+ @partial partials/timescale(range: .yearly) +

Scrobbles

+ @partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns) +
+
+

Rating

+ + +
+
\ No newline at end of file From 162341fb5f696196375c33f5ae3c795f8058c952 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 9 Jun 2025 21:41:52 -0400 Subject: [PATCH 64/71] Convert ids to i64 The birthday paradox is a real problem with the size of our datasets. i64 is the largest numerical value we can use, and there's a 0.1% chance of collision with ~2,000,000 values, so I feel pretty comfortable with this --- .../2025-04-07_14-31-14_create_songs.zig | 2 +- .../2025-04-07_14-31-45_create_albums.zig | 2 +- .../2025-04-07_14-34-39_create_albumsongs.zig | 6 +++--- .../2025-04-07_14-35-53_create_scrobbles.zig | 4 ++-- .../2025-04-07_14-38-02_create_artists.zig | 2 +- ...04-07_14-39-09_create_albumsongsartists.zig | 6 +++--- ...2025-04-07_14-40-17_create_artistalbums.zig | 6 +++--- src/queries.zig | 18 ++++++------------ src/types.zig | 2 +- 9 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig index 9a52b6b..e8ae1d6 100644 --- a/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig +++ b/src/app/database/migrations/2025-04-07_14-31-14_create_songs.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "songs", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("length", .float, .{ .optional = true }), t.column("hidden", .boolean, .{}), diff --git a/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig index d706cfe..86b6184 100644 --- a/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig +++ b/src/app/database/migrations/2025-04-07_14-31-45_create_albums.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albums", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("length", .float, .{ .optional = true }), t.timestamps(.{}), diff --git a/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig index 4381a5b..96c3063 100644 --- a/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig +++ b/src/app/database/migrations/2025-04-07_14-34-39_create_albumsongs.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albumsongs", &.{ - t.primaryKey("id", .{}), - t.column("song_id", .integer, .{ .reference = .{ "songs", "id" } }), - t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("song_id", .bigint, .{ .reference = .{ "songs", "id" } }), + t.column("album_id", .bigint, .{ .reference = .{ "albums", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index 088a43c..c429e19 100644 --- a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -6,8 +6,8 @@ pub fn up(repo: anytype) !void { try repo.createTable( "scrobbles", &.{ - t.primaryKey("id", .{}), - t.column("albumsong", .integer, .{ .reference = .{ "albumsongs", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("albumsong", .bigint, .{ .reference = .{ "albumsongs", "id" } }), t.column("datetime", .datetime, .{}), t.timestamps(.{}), }, diff --git a/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig index ed0d3d4..97c5bfe 100644 --- a/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig +++ b/src/app/database/migrations/2025-04-07_14-38-02_create_artists.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "artists", &.{ - t.primaryKey("id", .{}), + t.primaryKey("id", .{ .type = .bigint }), t.column("name", .string, .{}), t.column("disambiguation", .string, .{ .optional = true }), t.timestamps(.{}), diff --git a/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig index 55c7d84..3355196 100644 --- a/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig +++ b/src/app/database/migrations/2025-04-07_14-39-09_create_albumsongsartists.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "albumsongsartists", &.{ - t.primaryKey("id", .{}), - t.column("albumsong_id", .integer, .{ .reference = .{ "albumsongs", "id" } }), - t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("albumsong_id", .bigint, .{ .reference = .{ "albumsongs", "id" } }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig index 76bac1e..3c3ea7f 100644 --- a/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig +++ b/src/app/database/migrations/2025-04-07_14-40-17_create_artistalbums.zig @@ -6,9 +6,9 @@ pub fn up(repo: anytype) !void { try repo.createTable( "artistalbums", &.{ - t.primaryKey("id", .{}), - t.column("album_id", .integer, .{ .reference = .{ "albums", "id" } }), - t.column("artist_id", .integer, .{ .reference = .{ "artists", "id" } }), + t.primaryKey("id", .{ .type = .bigint }), + t.column("album_id", .bigint, .{ .reference = .{ "albums", "id" } }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), t.timestamps(.{}), }, .{}, diff --git a/src/queries.zig b/src/queries.zig index 5d92869..ba2a734 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -23,12 +23,6 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: return out.get("entity_info").?; } - //if (query.query_type == .datestreak) { - // var out: *jetzig.Data.Value = try Data.object(); - // const entity = try (try result.next()).?.to(struct { name: []const u8, id: i32, scrobbles: i64, rank: []const u8 }, .{ .dupe = true, .allocator = request.allocator }); - // - //} - var out: *jetzig.Data.Value = try Data.array(); var mapper = result.mapper(UnifiedResult, .{ .dupe = true, .allocator = request.allocator }); @@ -65,22 +59,22 @@ const GeneratedQuery = struct { const UnifiedResult = struct { album_name: ?[]const u8 = null, - album_id: ?i32 = null, + album_id: ?i64 = null, song_name: ?[]const u8 = null, - song_id: ?i32 = null, + song_id: ?i64 = null, artist_name: ?[]const u8 = null, - artist_id: ?i32 = null, + artist_id: ?i64 = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, }; const EntityInfoResult = struct { album_name: ?[]const u8 = null, - album_id: ?i32 = null, + album_id: ?i64 = null, song_name: ?[]const u8 = null, - song_id: ?i32 = null, + song_id: ?i64 = null, artist_name: ?[]const u8 = null, - artist_id: ?i32 = null, + artist_id: ?i64 = null, scrobbles: ?i64 = null, date: ?[]const u8 = null, rank: []const u8, diff --git a/src/types.zig b/src/types.zig index 3b2e4ff..5f3e0e3 100644 --- a/src/types.zig +++ b/src/types.zig @@ -133,5 +133,5 @@ pub const TableRow = struct { pub const HyperlinkData = struct { name: []const u8, - id: i32, + id: i64, }; From 85552f39c1145acf18d9b74b1252a1b32ac12168 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 9 Jun 2025 21:45:41 -0400 Subject: [PATCH 65/71] Add Artistsongs table Whether or not a song is covered, there was an original artist who originally performed the song. The only issue is that an Artistsongs table will almost be the exact same as the Albumsongsartists table, since most songs aren't covered. So, it may be better not to populate that table by default, and then if two albumsongs with different artists share the same song, then fill the Artistsongs table. --- src/app/database/Schema.zig | 56 ++++++++++++------- ...2025-06-06_19-46-32_create_artistsongs.zig | 20 +++++++ 2 files changed, 56 insertions(+), 20 deletions(-) create mode 100644 src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig diff --git a/src/app/database/Schema.zig b/src/app/database/Schema.zig index eeec984..0a30a19 100644 --- a/src/app/database/Schema.zig +++ b/src/app/database/Schema.zig @@ -4,7 +4,7 @@ pub const Album = jetquery.Model( @This(), "albums", struct { - id: i32, + id: i64, name: []const u8, length: ?f32, created_at: jetquery.DateTime, @@ -22,9 +22,9 @@ pub const Albumsong = jetquery.Model( @This(), "albumsongs", struct { - id: i32, - song_id: i32, - album_id: i32, + id: i64, + song_id: i64, + album_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -32,9 +32,7 @@ pub const Albumsong = jetquery.Model( .relations = .{ .song = jetquery.belongsTo(.Song, .{}), .album = jetquery.belongsTo(.Album, .{}), - .scrobbles = jetquery.hasMany(.Scrobble, .{ - .foreign_key = "albumsong", - }), + .scrobbles = jetquery.hasMany(.Scrobble, .{ .foreign_key = "albumsong" }), .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), }, }, @@ -44,9 +42,9 @@ pub const Albumsongsartist = jetquery.Model( @This(), "albumsongsartists", struct { - id: i32, - albumsong_id: i32, - artist_id: i32, + id: i64, + albumsong_id: i64, + artist_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -62,9 +60,9 @@ pub const Artistalbum = jetquery.Model( @This(), "artistalbums", struct { - id: i32, - album_id: i32, - artist_id: i32, + id: i64, + album_id: i64, + artist_id: i64, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, @@ -80,7 +78,7 @@ pub const Artist = jetquery.Model( @This(), "artists", struct { - id: i32, + id: i64, name: []const u8, disambiguation: ?[]const u8, created_at: jetquery.DateTime, @@ -90,6 +88,7 @@ pub const Artist = jetquery.Model( .relations = .{ .albumsongsartists = jetquery.hasMany(.Albumsongsartist, .{}), .artistalbums = jetquery.hasMany(.Artistalbum, .{}), + .artistsongs = jetquery.hasMany(.Artistsong, .{}), }, }, ); @@ -98,17 +97,15 @@ pub const Scrobble = jetquery.Model( @This(), "scrobbles", struct { - id: i32, - albumsong: i32, + id: i64, + albumsong: i64, datetime: jetquery.DateTime, created_at: jetquery.DateTime, updated_at: jetquery.DateTime, }, .{ .relations = .{ - .albumsong = jetquery.belongsTo(.Albumsong, .{ - .foreign_key = "albumsong", - }), + .albumsong = jetquery.belongsTo(.Albumsong, .{ .foreign_key = "albumsong" }), }, }, ); @@ -117,7 +114,7 @@ pub const Song = jetquery.Model( @This(), "songs", struct { - id: i32, + id: i64, name: []const u8, length: ?f32, hidden: bool, @@ -127,6 +124,25 @@ pub const Song = jetquery.Model( .{ .relations = .{ .albumsongs = jetquery.hasMany(.Albumsong, .{}), + .artistsongs = jetquery.hasMany(.Artistsong, .{}), + }, + }, +); + +pub const Artistsong = jetquery.Model( + @This(), + "artistsongs", + struct { + id: i64, + artist_id: i64, + song_id: i64, + created_at: jetquery.DateTime, + updated_at: jetquery.DateTime, + }, + .{ + .relations = .{ + .artist = jetquery.belongsTo(.Artist, .{}), + .song = jetquery.belongsTo(.Song, .{}), }, }, ); diff --git a/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig b/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig new file mode 100644 index 0000000..7d0a6c1 --- /dev/null +++ b/src/app/database/migrations/2025-06-06_19-46-32_create_artistsongs.zig @@ -0,0 +1,20 @@ +const std = @import("std"); +const jetquery = @import("jetquery"); +const t = jetquery.schema.table; + +pub fn up(repo: anytype) !void { + try repo.createTable( + "artistsongs", + &.{ + t.primaryKey("id", .{ .type = .bigint }), + t.column("artist_id", .bigint, .{ .reference = .{ "artists", "id" } }), + t.column("song_id", .bigint, .{ .reference = .{ "songs", "id" } }), + t.timestamps(.{}), + }, + .{}, + ); +} + +pub fn down(repo: anytype) !void { + try repo.dropTable("artistsongs", .{}); +} From 1e4a271b8d7e78c37140e6e129d5292dda2cd117 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:22:06 -0400 Subject: [PATCH 66/71] Update README --- README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8247b21..06f45c2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ Inspired by [Last.fm](https://last.fm), [Maloja](https://github.com/krateng/maloja), and [Lastfmstats.com](https://www.lastfmstats.com). - **Z**uletzt is written with [**Z**ig](https://github.com/ziglang/zig) and [Jetzig](https://github.com/jetzig-framework/jetzig) as a means of learning the language, reintroducing myself to programming, and combining the functionality of the aforementioned inspirations. @@ -12,6 +11,35 @@ Zuletzt means "last" in German. Licensed under MIT. +## Usage +Zuletzt allows uploads of Scrobbles at the `/upload` page, where you +can import Scrobbles from a Spotify data export, Last.FM data export (a `.json` +file from lastfmstats.com), or by providing a Last.FM username and connecting +to Last.FM directly. + +Zuletzt will not make any assumptions about the data, and only change metadata when asked to by a rule. Two albums will be considered the same if: +- They share the same title (case/diacritic sensitive) +- The album artist(s) are the same + +Zuletzt allows you to list multiple artists under an album using rules, but +does not try to automatically split artists along common delimiters. For +example, there's no way to know that "Mermaid Avenue" by "Billy Bragg, Wilco" +is performed by two artists, while "Ants From Up There" by "Black Country, New +Road" is performed by one artist. Thus, a rule needs to be made to tell Zuletzt +"Mermaid Avenue" is performed by "Billy Bragg" and "Wilco". + +Two songs will only be considered the same if: +- They share the same title (case/diacritic sensitive) +- They appear on the same album + +If two or more songs with the same spelling appear on an album, they are +necessarily grouped under the same name, as there is no way to differentiate +them (see "Once In Royal David's City" on Sufjan Stevens's "Songs For +Christmas", for example). Every artist that performs on those songs with +receive attribution for the combined song. + +If two artists have the same name, they are necessarily listed as the same artist, but can be separated with a rule, or after the fact, with a disambiguation string. + ## To-Do List: - [ ] Entity statistics - [x] See all artists under "/artists" @@ -60,6 +88,7 @@ Licensed under MIT. - [ ] Rank songs - [ ] Custom statistics[^7] - [ ] "Playlists"[^8] +- [ ] First launch setup [^1]: I do not intend to exactly replicate all the statistics Lastfmstats.com provides, but I would at least like to give the user the option to see those kinds of statistics, or generate them themselves (see 7). From a8a4ed27c46baeb213b687f89fbb8fb71cdc04f6 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:22:38 -0400 Subject: [PATCH 67/71] Make process_scrobbles vars more readable, and change hashing --- src/app/jobs/process_scrobbles.zig | 107 +++++++++++++++-------------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index dcf7f1f..ea743b9 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -20,12 +20,13 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig // Probably want to include artist name here, but not sure how to yet - const track_artist_count = item.getT(.array, "artists_track").?.count(); - const album_artist_count = item.getT(.array, "artists_album").?.count(); - var track_artist_name_buffer = try allocator.alloc([]const u8, track_artist_count); - var album_artist_name_buffer = try allocator.alloc([]const u8, album_artist_count); - var track_artist_id_buffer = try allocator.alloc(i32, track_artist_count); - var album_artist_id_buffer = try allocator.alloc(i32, album_artist_count); + const track_artists = item.getT(.array, "artists_track").?.items(); + const album_artists = item.getT(.array, "artists_album").?.items(); + + var track_artist_name_buffer = try allocator.alloc([]const u8, track_artists.len); + var album_artist_name_buffer = try allocator.alloc([]const u8, album_artists.len); + var track_artist_id_buffer = try allocator.alloc(i64, track_artists.len); + var album_artist_id_buffer = try allocator.alloc(i64, album_artists.len); const scrobble: Data.Scrobble = .{ .track = item.getT(.string, "track").?, @@ -35,95 +36,99 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig .date = @as(i64, @truncate(item.getT(.integer, "date").?)), }; - var id_prehash = std.ArrayList(u8).init(allocator); + var album_hash_string = std.ArrayList(u8).init(allocator); + var track_hash_string = std.ArrayList(u8).init(allocator); - for (item.getT(.array, "artists_track").?.items(), 0..track_artist_count) |artist, i| { + // I theoretically don't need this for loop + for (track_artists, 0..track_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); track_artist_name_buffer[i] = artist_name; - track_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + track_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); } - for (item.getT(.array, "artists_album").?.items(), 0..album_artist_count) |artist, i| { + for (album_artists, 0..album_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; - album_artist_id_buffer[i] = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); - try id_prehash.appendSlice(artist_name); + album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + try album_hash_string.appendSlice(artist_name); } - try id_prehash.appendSlice(scrobble.album); - const album_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); - try id_prehash.appendSlice(scrobble.track); - const song_id = @as(i32, @bitCast(std.hash.Fnv1a_32.hash(id_prehash.items))); + try album_hash_string.appendSlice(scrobble.album); + try track_hash_string.appendSlice(scrobble.album); + const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(album_hash_string.items))); + try track_hash_string.appendSlice(scrobble.track); + const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(track_hash_string.items))); - var albumsong = try jetzig.database.Query(.Albumsong) - .findBy(.{ .album_id = album_id, .song_id = song_id }) + var albumsong_id = try jetzig.database.Query(.Albumsong) + .find(album_hash ^ track_hash) .select(.{.id}).execute(env.repo); - var ins_album = try jetzig.database.Query(.Album) - .find(album_id) + var album_id = try jetzig.database.Query(.Album) + .find(album_hash) .select(.{.id}).execute(env.repo); - for (track_artist_name_buffer, track_artist_id_buffer) |artist_name, artist_id| { - var ins_artist = try jetzig.database.Query(.Artist) - .find(artist_id) + for (track_artist_name_buffer, track_artist_id_buffer) |scrobble_track_artist, track_artist_hash| { + var artist_id = try jetzig.database.Query(.Artist) + .find(track_artist_hash) .select(.{.id}).execute(env.repo); - if (ins_artist == null) - ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + if (artist_id == null) + artist_id = try jetzig.database.Query(.Artist) + .insert(.{ .id = track_artist_hash, .name = scrobble_track_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); - if (albumsong == null) { - var ins_song = try jetzig.database.Query(.Song) - .find(song_id) + if (albumsong_id == null) { + var track_id = try jetzig.database.Query(.Song) + .find(track_hash) .select(.{.id}).execute(env.repo); - if (ins_song == null) - ins_song = try jetzig.database.Query(.Song) - .insert(.{ .id = song_id, .name = scrobble.track, .length = null, .hidden = false }) + if (track_id == null) + track_id = try jetzig.database.Query(.Song) + .insert(.{ .id = track_hash, .name = scrobble.track, .length = null, .hidden = false }) .returning(.{.id}).execute(env.repo); - if (ins_album == null) - ins_album = try jetzig.database.Query(.Album) - .insert(.{ .id = album_id, .name = scrobble.album, .length = null }) + if (album_id == null) + album_id = try jetzig.database.Query(.Album) + .insert(.{ .id = album_hash, .name = scrobble.album, .length = null }) .returning(.{.id}).execute(env.repo); - albumsong = try jetzig.database.Query(.Albumsong) - .insert(.{ .song_id = ins_song.?.id, .album_id = ins_album.?.id }) + albumsong_id = try jetzig.database.Query(.Albumsong) + .insert(.{ .song_id = track_id.?.id, .album_id = album_id.?.id }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } else { const ins_albumsongartist = try jetzig.database.Query(.Albumsongsartist) - .findBy(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }) + .findBy(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }) .select(.{.id}).execute(env.repo); if (ins_albumsongartist == null) try jetzig.database.Query(.Albumsongsartist) - .insert(.{ .albumsong_id = albumsong.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .albumsong_id = albumsong_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } } - for (album_artist_name_buffer, album_artist_id_buffer) |artist_name, artist_id| { - const ins_artistalbum = try jetzig.database.Query(.Artistalbum) - .findBy(.{ .album_id = ins_album.?.id, .artist_id = artist_id }) + + for (album_artist_name_buffer, album_artist_id_buffer) |scrobble_album_artist, album_artist_hash| { + const artistalbum_id = try jetzig.database.Query(.Artistalbum) + .findBy(.{ .album_id = album_id.?.id, .artist_id = album_artist_hash }) .select(.{.id}).execute(env.repo); - if (ins_artistalbum == null) { - var ins_artist = try jetzig.database.Query(.Artist) - .find(artist_id) + if (artistalbum_id == null) { + var artist_id = try jetzig.database.Query(.Artist) + .find(album_artist_hash) .select(.{.id}).execute(env.repo); - if (ins_artist == null) - ins_artist = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = artist_name, .disambiguation = null }) + if (artist_id == null) + artist_id = try jetzig.database.Query(.Artist) + .insert(.{ .id = artist_id, .name = scrobble_album_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Artistalbum) - .insert(.{ .album_id = ins_album.?.id, .artist_id = ins_artist.?.id }).execute(env.repo); + .insert(.{ .album_id = album_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); } } try jetzig.database.Query(.Scrobble) - .insert(.{ .albumsong = albumsong.?.id, .datetime = scrobble.date }).execute(env.repo); + .insert(.{ .albumsong = albumsong_id.?.id, .datetime = scrobble.date }).execute(env.repo); } } } From 6a1c82242074e296d7576913c467be236e9be336 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:25:40 -0400 Subject: [PATCH 68/71] Add entities_by_name query Will probably be used for disambiguation pages (among other things, but disambiguation pages are coming up soon) --- src/queries.zig | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/queries.zig b/src/queries.zig index 8db7809..adb1447 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -55,7 +55,7 @@ pub fn entityQueryResult(request: *jetzig.Request, query: GeneratedQuery, args: } const EntityType = enum { scrobble, song, album, artist }; -const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak }; +const QueryTypeEnum = enum { firstlast, timescale, entities, entity_items, appears, entity_info, datestreak, entities_by_name }; const GeneratedQuery = struct { entity: EntityType, @@ -215,13 +215,14 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { \\ORDER BY scrobbles.datetime ASC , .song => - \\SELECT songs.name AS song_name, songs.id AS song_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles \\FROM albumsongs \\INNER JOIN songs ON albumsongs.song_id = songs.id \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id - \\GROUP BY songs.id, artists.id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\GROUP BY songs.id, albums.id, artists.id \\ORDER BY scrobbles DESC, songs.name ASC , .album => @@ -388,6 +389,21 @@ pub fn loadQuery(entity: EntityType, query_type: QueryTypeEnum) GeneratedQuery { , .scrobble => unreachable, }, + .entities_by_name => switch (entity) { + .song => + \\SELECT songs.name AS song_name, songs.id AS song_id, albums.name AS album_name, albums.id AS album_id, artists.name AS artist_name, artists.id AS artist_id, COUNT(scrobbles) AS scrobbles + \\FROM albumsongs + \\INNER JOIN songs ON albumsongs.song_id = songs.id + \\INNER JOIN scrobbles ON scrobbles.albumsong = albumsongs.id + \\INNER JOIN albumsongsartists ON albumsongsartists.albumsong_id = albumsongs.id + \\INNER JOIN artists ON artists.id = albumsongsartists.artist_id + \\INNER JOIN albums ON albums.id = albumsongs.album_id + \\WHERE LOWER(songs.name) LIKE LOWER($1) + \\GROUP BY songs.id, albums.id, artists.id + \\ORDER BY songs.name ASC, scrobbles DESC; + , + else => unreachable, + }, }, }; } From 2f420bc5ce6d86822e62a819a616a1a10d3fcf19 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 09:26:12 -0400 Subject: [PATCH 69/71] Testing with groups and htmx --- src/app/views/groups/index.zmpl | 14 +++++++++++--- src/app/views/songs.zig | 10 +++++++++- src/app/views/songs/index.zmpl | 4 +++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/app/views/groups/index.zmpl b/src/app/views/groups/index.zmpl index 76457d0..1522924 100644 --- a/src/app/views/groups/index.zmpl +++ b/src/app/views/groups/index.zmpl @@ -1,3 +1,11 @@ -
- Content goes here -
+ + + + +@partial partials/header + +

Merge Songs

+
+ +
+
\ No newline at end of file diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index b24dbf5..2ed2e06 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -5,7 +5,15 @@ const queries = @import("../../queries.zig"); pub fn index(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const songs = try queries.entityQueryResult(request, queries.loadQuery(.song, .entities), .{}); + const htmx_query = (try request.queryParams()).getT(.string, "s"); + + try root.put("htmx", htmx_query != null); + + const songs = if (htmx_query) |name| + try queries.entityQueryResult(request, queries.loadQuery(.song, .entities_by_name), .{name}) + else + try queries.entityQueryResult(request, queries.loadQuery(.song, .entities), .{}); + try root.put("songs", songs); return request.render(.ok); diff --git a/src/app/views/songs/index.zmpl b/src/app/views/songs/index.zmpl index 6964630..862b753 100644 --- a/src/app/views/songs/index.zmpl +++ b/src/app/views/songs/index.zmpl @@ -1,6 +1,6 @@ @zig { const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date}; - const columns: ColumnChoices = &.{.song, .artistlist, .scrobbles}; + const columns: ColumnChoices = &.{.song, .album, .artistlist, .scrobbles}; } @@ -8,8 +8,10 @@ + @if (! $.htmx) @partial partials/header

Songs

+ @end @partial partials/newtable(T: ColumnChoices, table_data: .songs, columns: columns) \ No newline at end of file From 36873053bca53dc5d39f3d0578701805c966f4a9 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Wed, 11 Jun 2025 20:08:13 -0400 Subject: [PATCH 70/71] Make new scrobble processing function This uses the zmpl data as a hash map to check if we've already checked the db for some song/album/artist/etc. and now only checks once per entity/assoc. table entry to speed things up. Previously, for each scrobble, we checked if its metadata appeared in the respective table, regardless of whether or not we've scrobbled that albumsong before. So, a song like Starless had to be checked (at the time of writing) ~180 times, but is now only checked once. Similarly, Wilco was checked ~3000+ times, as Hurry Up, We're Dreaming was cheked ~700 times. The only problem now is the way it was implemented. Obviously, copying and pasting those huge chunks of code isn't very nice looking. ATM, I don't really care, and I'm more happy about the overall speed increase, as well as the readability increase of the job. However, I don't want to leave it like that. The way I see it, I have two options: either create a funcion which does this, or I can do something even better, which is create a jsonParse function, which, if my thought process works, would remove the need for an intermediary source type, meaning we no longer need to switch on that type, which means we can just have one for loop that does everything, which would mean we just need to have that code in one place. Also not entirely happy with the code concerning all the conversions to i64s and []const u8s, but I think I have to. --- .../2025-04-07_14-35-53_create_scrobbles.zig | 2 +- src/app/jobs/process_scrobbles.zig | 8 +- src/app/jobs/process_scrobbles2.zig | 131 +++++++++++ src/app/views/upload.zig | 207 +++++++++++++++++- 4 files changed, 332 insertions(+), 16 deletions(-) create mode 100644 src/app/jobs/process_scrobbles2.zig diff --git a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index c429e19..c3bcd12 100644 --- a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "scrobbles", &.{ - t.primaryKey("id", .{ .type = .bigint }), + t.primaryKey("id", .{}), t.column("albumsong", .bigint, .{ .reference = .{ "albumsongs", "id" } }), t.column("datetime", .datetime, .{}), t.timestamps(.{}), diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index ea743b9..770bde7 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -49,15 +49,15 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig for (album_artists, 0..album_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; - album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); try album_hash_string.appendSlice(artist_name); } try album_hash_string.appendSlice(scrobble.album); try track_hash_string.appendSlice(scrobble.album); - const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(album_hash_string.items))); + const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(album_hash_string.items))); try track_hash_string.appendSlice(scrobble.track); - const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(track_hash_string.items))); + const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(track_hash_string.items))); var albumsong_id = try jetzig.database.Query(.Albumsong) .find(album_hash ^ track_hash) @@ -120,7 +120,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig .select(.{.id}).execute(env.repo); if (artist_id == null) artist_id = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = scrobble_album_artist, .disambiguation = null }) + .insert(.{ .id = album_artist_hash, .name = scrobble_album_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Artistalbum) .insert(.{ .album_id = album_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); diff --git a/src/app/jobs/process_scrobbles2.zig b/src/app/jobs/process_scrobbles2.zig new file mode 100644 index 0000000..8b3ae66 --- /dev/null +++ b/src/app/jobs/process_scrobbles2.zig @@ -0,0 +1,131 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +// The `run` function for a job is invoked every time the job is processed by a queue worker +// (or by the Jetzig server if the job is processed in-line). +// +// Arguments: +// * allocator: Arena allocator for use during the job execution process. +// * params: Params assigned to a job (from a request, values added to response data). +// * env: Provides the following fields: +// - logger: Logger attached to the same stream as the Jetzig server. +// - environment: Enum of `{ production, development }`. +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + + for (params.getT(.object, "tracks").?.items()) |track| { + const id = try std.fmt.parseInt(i64, track.key, 10); + + const track_query = try jetzig.database.Query(.Song) + .find(id).execute(env.repo); + + if (track_query == null) { + const name = try track.value.coerce([]const u8); + try jetzig.database.Query(.Song) + .insert(.{ .id = id, .name = name, .length = null, .hidden = false }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albums").?.items()) |album| { + const id = try std.fmt.parseInt(i64, album.key, 10); + + const album_query = try jetzig.database.Query(.Album) + .find(id).execute(env.repo); + + if (album_query == null) { + const name = try album.value.coerce([]const u8); + try jetzig.database.Query(.Album) + .insert(.{ .id = id, .name = name, .length = null }) + .execute(env.repo); + } + } + + for (params.getT(.object, "artists").?.items()) |artist| { + const id = try std.fmt.parseInt(i64, artist.key, 10); + + const artist_query = try jetzig.database.Query(.Artist) + .find(id).execute(env.repo); + + if (artist_query == null) { + const name = try artist.value.coerce([]const u8); + try jetzig.database.Query(.Artist) + .insert(.{ .id = id, .name = name }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albumsongs").?.items()) |as| { + const id = try std.fmt.parseInt(i64, as.key, 10); + + const as_query = try jetzig.database.Query(.Albumsong) + .find(id).execute(env.repo); + + if (as_query == null) { + const track_id = @as(i64, @intCast(as.value.getT(.integer, "song").?)); + const album_id = @as(i64, @intCast(as.value.getT(.integer, "album").?)); + try jetzig.database.Query(.Albumsong) + .insert(.{ .id = id, .song_id = track_id, .album_id = album_id }) + .execute(env.repo); + } + + const scrobbles = as.value.getT(.array, "scrobbles").?; + for (scrobbles.items()) |date| { + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = id, .datetime = date }) + .execute(env.repo); + } + } + + for (params.getT(.object, "artistalbums").?.items()) |aa| { + const id = try std.fmt.parseInt(i64, aa.key, 10); + + const aa_query = try jetzig.database.Query(.Artistalbum) + .find(id).execute(env.repo); + + if (aa_query == null) { + const artist_id = @as(i64, @intCast(aa.value.getT(.integer, "artist").?)); + const album_id = @as(i64, @intCast(aa.value.getT(.integer, "album").?)); + try jetzig.database.Query(.Artistalbum) + .insert(.{ .id = id, .artist_id = artist_id, .album_id = album_id }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albumsongsartists").?.items()) |asa| { + const id = try std.fmt.parseInt(i64, asa.key, 10); + + const asa_query = try jetzig.database.Query(.Albumsongsartist) + .find(id).execute(env.repo); + + if (asa_query == null) { + const albumsong_id = @as(i64, @intCast(asa.value.getT(.integer, "albumsong").?)); + const artist_id = @as(i64, @intCast(asa.value.getT(.integer, "artist").?)); + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ .id = id, .albumsong_id = albumsong_id, .artist_id = artist_id }) + .execute(env.repo); + } + } + + //for ((params.getT(.object, "albumsongsartists")).?.items(.object)) |asa| { + // const id = try std.fmt.parseInt(i64, asa.key, 10); + // const albumsong_id = asa.value.getT(.integer, "albumsong"); + // const track_artist_id = asa.value.getT(.integer, "artist"); + + // const albumsongartist = try jetzig.database.Query(.Albumsongsartist) + // .find(id) + // .select(.{.id}).execute(env.repo); + + // if (albumsongartist == null) { + // var artist_id = try jetzig.database.Query(.Artist) + // .find(track_artist_id) + // .select(.{.id}).execute(env.repo); + // + // if (artist_id == null) { + // const artist = params.chain(.{"artists",}) + // artist_id = try jetzig.database.Query(.Artist) + // .insert(.{ .id = track_artist_hash, .name = scrobble_track_artist, .disambiguation = null }) + // .execute(env.repo); + // } + // } + //} +} diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index ebe9395..ac2c665 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -24,26 +24,25 @@ pub fn post(request: *jetzig.Request) !jetzig.View { defer rule_file.close(); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; - var job = try request.job("process_scrobbles"); + //var job = try request.job("process_scrobbles"); + var job = try request.job("process_scrobbles2"); const source = params.getT(.integer, "t").?; // This param is required in HTML + const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { const date = params.getT(.string, "latest-date").?; break :blk try zeit.Time.fromISO8601(date); - } else blk: { - break :blk (try zeit.instant(.{ .source = .now })).time(); - }; + } else (try zeit.instant(.{ .source = .now })).time(); + const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { const date = params.getT(.string, "earliest-date").?; break :blk try zeit.Time.fromISO8601(date); - } else blk: { - break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); - }; + } else (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); const earliest_timestamp = earliest_date.instant().unixTimestamp(); const latest_timestamp = latest_date.instant().unixTimestamp(); var view_params = try root.put("scrobbles", .array); - var job_params = try job.params.put("scrobbles", .array); + //var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; @@ -79,6 +78,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View { else => unreachable, }; + var artists = try job.params.put("artists", .object); + var albums = try job.params.put("albums", .object); + var tracks = try job.params.put("tracks", .object); + var artistalbums = try job.params.put("artistalbums", .object); + var albumsongs = try job.params.put("albumsongs", .object); + var albumsongsartists = try job.params.put("albumsongsartists", .object); + // Not sure if I should be proud or feel sick switch (imported_scrobbles) { .LastFMStats => |scrobbles| { @@ -99,7 +105,65 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, .LastFMWeb => |scrobbles| { @@ -117,7 +181,65 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, .Spotify => |scrobbles| { @@ -149,7 +271,66 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, } @@ -159,3 +340,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { return request.render(.created); } + +fn pair(a: u64, b: u64) u64 { + return @divFloor((a +% b) *% (a +% b +% 1) +% b, 2); +} From acef5d8e49358705b3d51ce2366dac44c59f0fd8 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 14 Jun 2025 16:29:16 -0400 Subject: [PATCH 71/71] Parse ScrobbleArray type rather than individual types using jsonParse POC for parsing and data validation inside jsonParse rather than in the view. I believe this is necessarily slower (and actually doesn't work for Spotify scrobbles) but could be made to work/be faster if we implemented our own json parsing for each type, but I think that's too much work for too little gain (atm) --- src/app/views/upload.zig | 324 ++++++++------------------------------- src/types.zig | 76 +++++++++ 2 files changed, 144 insertions(+), 256 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index ac2c665..f712ba4 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -24,7 +24,6 @@ pub fn post(request: *jetzig.Request) !jetzig.View { defer rule_file.close(); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; - //var job = try request.job("process_scrobbles"); var job = try request.job("process_scrobbles2"); const source = params.getT(.integer, "t").?; // This param is required in HTML @@ -42,38 +41,36 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const latest_timestamp = latest_date.instant().unixTimestamp(); var view_params = try root.put("scrobbles", .array); - //var job_params = try job.params.put("scrobbles", .array); - var skipped_tracks: u64 = 0; - var limited_tracks: u64 = 0; - - const imported_scrobbles: Data.ImportedScrobbles = switch (source) { - 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles }, - 1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) }, + const imported_scrobbles = switch (source) { + 0, 1 => (try std.json.parseFromSliceLeaky(Data.ScrobbleArray, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles, 2 => blk: { const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); - var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator); + var scrobble_buffer = std.ArrayList(Data.Scrobble).init(request.allocator); const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; + const mp_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, 0 }); + const mp_r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = mp_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + std.log.debug("Max page query: {}", .{mp_r}); + const parsed_mp_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, (try lastfm_response_buffer.toOwnedSlice()), .{ .ignore_unknown_fields = true }); + var page: usize = 1; - var max_pages: ?usize = null; + const max_pages: usize = try std.fmt.parseInt(usize, parsed_mp_response.recenttracks.@"@attr".totalPages, 10); while (true) : (page += 1) { - if (max_pages != null and page > max_pages.?) break; + if (page > max_pages) break; const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); std.log.debug("{}: {}", .{ page, r }); const response_string = try lastfm_response_buffer.toOwnedSlice(); - const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); - //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); - if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); - try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track); + const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.ScrobbleArray, request.allocator, response_string, .{ .ignore_unknown_fields = true }); + try scrobble_buffer.appendSlice(parsed_lastfm_response.scrobbles); } - break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items }; + break :blk scrobble_buffer.items; }, else => unreachable, }; @@ -85,257 +82,72 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var albumsongs = try job.params.put("albumsongs", .object); var albumsongsartists = try job.params.put("albumsongsartists", .object); - // Not sure if I should be proud or feel sick - switch (imported_scrobbles) { - .LastFMStats => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { - limited_tracks += 1; - continue :appends; - } - const filtered_scrobble = Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date * 1_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + for (imported_scrobbles) |scrobble| { + if (scrobble.date > latest_timestamp * 1_000_000 or scrobble.date < earliest_timestamp * 1_000_000) continue; + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, scrobble, rl) else scrobble; - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - try view_params.append(row); - //try job_params.append(complete_scrobble); - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + try view_params.append(row); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); } - }, - .LastFMWeb => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.date == null) continue :appends; - const filtered_scrobble = Data.Scrobble{ - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, - .track = scrobble.name, - .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + } - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - try view_params.append(row); - //try job_params.append(complete_scrobble); - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } - - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); } - }, - .Spotify => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { - limited_tracks += 1; - continue :appends; - } - - const filtered_scrobble = Data.Scrobble{ - .album = scrobble.master_metadata_album_album_name.?, - .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .track = scrobble.master_metadata_track_name.?, - .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; - - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - - try view_params.append(row); - //try job_params.append(complete_scrobble); - - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); - - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } - - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } - } - }, + } } - - std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); try job.schedule(); return request.render(.created); diff --git a/src/types.zig b/src/types.zig index 5f3e0e3..222d898 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,9 +1,21 @@ +const std = @import("std"); +const zeit = @import("zeit"); + pub const ImportedScrobbles = union(ScrobbleSources) { LastFMStats: []IgnorantScrobble, LastFMWeb: []LastFMWebScrobble, Spotify: []SpotifyScrobble, }; +const Litmus = struct { + username: ?[]const u8 = null, + ts: ?[]const u8 = null, + recenttracks: ?struct { + track: []LastFMWebScrobble, + @"@attr": LastFMWebQueryInfo, + } = null, +}; + const ScrobbleSources = enum { LastFMStats, LastFMWeb, @@ -26,6 +38,70 @@ pub const Scrobble = struct { date: i64, }; +pub const ScrobbleArray = struct { + scrobbles: []Scrobble, + + // This is an abuse of the jsonParse function. I don't like the idea of doing it, but I really like the results + // (or at least I will, assuming it works) + pub fn jsonParse(allocator: std.mem.Allocator, source: *std.json.Scanner, options: std.json.ParseOptions) !ScrobbleArray { + while (try source.peekNextTokenType() != .end_of_document) try source.skipValue(); + const litmus_test = try std.json.parseFromSliceLeaky(Litmus, allocator, source.input, .{ .ignore_unknown_fields = true }); + + if (litmus_test.username != null) { // LastFMStats + const lastfm_file = try std.json.parseFromSliceLeaky(LastFMStats, allocator, source.input, options); + var scrobble_buffer = try allocator.alloc(Scrobble, lastfm_file.scrobbles.len); + for (lastfm_file.scrobbles, 0..lastfm_file.scrobbles.len) |scrobble, i| { + const artist = try allocator.alloc([]const u8, 1); + artist[0] = scrobble.artist; + scrobble_buffer[i] = Scrobble{ + .album = scrobble.album, + .artists_album = artist, + .track = scrobble.track, + .artists_track = artist, + .date = scrobble.date * 1_000, + }; + } + return ScrobbleArray{ .scrobbles = scrobble_buffer }; + } + + if (litmus_test.ts != null) { // Spotify + const spotify = try std.json.parseFromSliceLeaky([]SpotifyScrobble, allocator, source.input, options); + var scrobble_buffer = try allocator.alloc(Scrobble, spotify.len); + for (spotify, 0..spotify.len) |scrobble, i| { + const artist = try allocator.alloc([]const u8, 1); + artist[0] = scrobble.master_metadata_album_artist_name.?; + scrobble_buffer[i] = Scrobble{ + .album = scrobble.master_metadata_album_album_name.?, + .artists_album = artist, + .track = scrobble.master_metadata_track_name.?, + .artists_track = artist, + .date = (zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } }) catch return error.OutOfMemory).unixTimestamp() * 1_000_000, + }; + } + return ScrobbleArray{ .scrobbles = scrobble_buffer }; + } + + if (litmus_test.recenttracks != null) { // LastFM API + const lastfm_web = try std.json.parseFromSliceLeaky(LastFMWeb, allocator, source.input, options); + var scrobble_buffer = try allocator.alloc(Scrobble, lastfm_web.recenttracks.track.len); + for (lastfm_web.recenttracks.track, 0..lastfm_web.recenttracks.track.len) |scrobble, i| { + const artist = try allocator.alloc([]const u8, 1); + artist[0] = scrobble.artist.@"#text"; + scrobble_buffer[i] = Scrobble{ + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artists_album = artist, + .track = scrobble.name, + .artists_track = artist, + .date = (std.fmt.parseInt(i64, scrobble.date.?.uts, 10) catch return error.OutOfMemory) * 1_000_000, + }; + } + return ScrobbleArray{ .scrobbles = scrobble_buffer }; + } + + return error.UnexpectedToken; + } +}; + // From lastfmstats.com pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble };