From 851aec3a975be5fd7924f93a47d645dd876fa277 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 12:14:27 -0400 Subject: [PATCH 1/6] I learned about std.math.maxInt --- src/queries.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queries.zig b/src/queries.zig index 371c15e..51d4500 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -18,7 +18,7 @@ pub fn entityQueryResult(request: *jetzig.Request, comptime query: GeneratedQuer var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, .{}, .{ .allocator = request.allocator, .column_names = true }); var out: *jetzig.Data.Value = try Data.object(); var mapper = result.mapper(PeakResult, .{ .dupe = true, .allocator = request.allocator }); - var rank = comptime (std.math.pow(u64, 2, 63) - 1); + var rank = comptime (std.math.maxInt(u64)); var date: []const u8 = undefined; var scrobbles = std.AutoArrayHashMap(i64, u32).init(request.allocator); From 0dec52af0100b42b0451836c453cd24488be5d94 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 16:09:30 -0400 Subject: [PATCH 2/6] Type rank --- src/queries.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queries.zig b/src/queries.zig index 51d4500..c8c313b 100644 --- a/src/queries.zig +++ b/src/queries.zig @@ -18,7 +18,7 @@ pub fn entityQueryResult(request: *jetzig.Request, comptime query: GeneratedQuer var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, .{}, .{ .allocator = request.allocator, .column_names = true }); var out: *jetzig.Data.Value = try Data.object(); var mapper = result.mapper(PeakResult, .{ .dupe = true, .allocator = request.allocator }); - var rank = comptime (std.math.maxInt(u64)); + var rank: u64 = comptime std.math.maxInt(u64); var date: []const u8 = undefined; var scrobbles = std.AutoArrayHashMap(i64, u32).init(request.allocator); From 6fe885132a9c227c26364fc5428b2d726da7208f Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 16:40:56 -0400 Subject: [PATCH 3/6] Do not use peak I only tested it on small datasets, and it wasn't so bad, but with my whole LastFM dataset, it is very bad --- src/app/views/albums.zig | 4 ++-- src/app/views/albums/get.zmpl | 1 - src/app/views/artists.zig | 4 ++-- src/app/views/artists/get.zmpl | 1 - src/app/views/songs.zig | 4 ++-- src/app/views/songs/get.zmpl | 1 - 6 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/app/views/albums.zig b/src/app/views/albums.zig index 5f3e0e4..1a3c799 100644 --- a/src/app/views/albums.zig +++ b/src/app/views/albums.zig @@ -55,8 +55,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); - const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int}); - try root.put("peak", peak); + //const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int}); + //try root.put("peak", peak); return request.render(.ok); } diff --git a/src/app/views/albums/get.zmpl b/src/app/views/albums/get.zmpl index d62af1c..7f40eb5 100644 --- a/src/app/views/albums/get.zmpl +++ b/src/app/views/albums/get.zmpl @@ -31,7 +31,6 @@ @else
{{.album.scrobbles}} scrobbles ({{.album.rank}} place)
@end -
All-time peak: {{.peak.rank}} ({{.peak.date}})
{{.album.song_num}} songs
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

diff --git a/src/app/views/artists.zig b/src/app/views/artists.zig index 540ccd3..3136def 100644 --- a/src/app/views/artists.zig +++ b/src/app/views/artists.zig @@ -54,8 +54,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const timescale = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .timescale), .{id_int}); try root.put("yearly", timescale); - const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int}); - try root.put("peak", peak); + //const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int}); + //try root.put("peak", peak); return request.render(.ok); } diff --git a/src/app/views/artists/get.zmpl b/src/app/views/artists/get.zmpl index 37a5379..89663f8 100644 --- a/src/app/views/artists/get.zmpl +++ b/src/app/views/artists/get.zmpl @@ -22,7 +22,6 @@ @else
{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)
@end -
All-time peak: {{.peak.rank}} ({{.peak.date}})
{{.artist.song_num}} songs
{{.artist.album_num}} albums
diff --git a/src/app/views/songs.zig b/src/app/views/songs.zig index f3d4bf5..72d21ce 100644 --- a/src/app/views/songs.zig +++ b/src/app/views/songs.zig @@ -60,8 +60,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View { const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); try root.put("reviews", ratings); - const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .peak), .{id_int}); - try root.put("peak", peak); + //const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .peak), .{id_int}); + //try root.put("peak", peak); return request.render(.ok); } diff --git a/src/app/views/songs/get.zmpl b/src/app/views/songs/get.zmpl index 05e2313..a9416ce 100644 --- a/src/app/views/songs/get.zmpl +++ b/src/app/views/songs/get.zmpl @@ -30,7 +30,6 @@ @else
{{.song.scrobbles}} scrobbles ({{.song.rank}} place)
@end -
All-time peak: {{.peak.rank}} ({{.peak.date}})
@partial partials/firstlast_listens(firstlast: .firstlast)

Yearly Performance

@partial partials/timescale(range: .yearly) From c95ac51e0561f10d881c02f9fdd7ec5d23d06928 Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 17:03:08 -0400 Subject: [PATCH 4/6] Prevent upload from crashing if 500 is received from LastFM --- src/app/views/upload.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index ac2c665..f22781d 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -66,6 +66,11 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); std.log.debug("{}: {}", .{ page, r }); + if (@intFromEnum(r.status) == 500) { + page -= 1; + std.time.sleep(3 * std.time.ns_per_s); + continue; + } const response_string = try lastfm_response_buffer.toOwnedSlice(); const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); From 8af6341f95898db84c369a26687163d40173391d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 17:07:14 -0400 Subject: [PATCH 5/6] Switch to defined constant when converting between s/ms/ns --- src/date_fmt.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 9746fe6..28e7b10 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -4,7 +4,7 @@ const Data = @import("types.zig"); pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, std.time.ns_per_s) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } From 902fcd4447fde65eafb518c35c0d8d63506e510e Mon Sep 17 00:00:00 2001 From: mitteneer Date: Mon, 7 Jul 2025 17:13:48 -0400 Subject: [PATCH 6/6] Create own parsing function I have dreamt of this for a long time. It is a minor optimization to be honest, but I previously had to choose between ugly code (what I had just prior to this) or looping through the data twice (slow). This parses the data and puts it into my intermediary type directly, along with relevant information about whetehr or not the scrobble is valid, and then I only need to loop over it once with "nice enough" code. There is still more I can do. My ultimate goal is to remove the looping entirely, and verify the data as it's being parsed, queuing up much smaller jobs that handle the individual entities (albums, artistsongs, etc.) as they come, rather than collecting it all and running it at once. I can get over the problem of needing to wit for all the LastFM responses that way as well. Furthermore, the data checking in the function expects a rather rigid structure that I'm not certain I can guarantee, but I'm pretty sure it'll stay that way. In any case, it may behoove me to make it more dynamic at some point. In any case, I am very excited about this change, and I hope I can continue improving upon it. --- src/app/views/upload.zig | 331 ++++++++++----------------------------- src/date_fmt.zig | 230 +++++++++++++++++++++++++++ src/types.zig | 14 ++ 3 files changed, 328 insertions(+), 247 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index f22781d..15deee3 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -47,22 +47,21 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - const imported_scrobbles: Data.ImportedScrobbles = switch (source) { - 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles }, - 1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) }, + const imported_scrobbles: []Data.UnifiedScrobble = switch (source) { + 0, 1 => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), 2 => blk: { const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); - var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator); + var scrobble_buffer = std.ArrayList(Data.UnifiedScrobble).init(request.allocator); const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; var page: usize = 1; - var max_pages: ?usize = null; + //var max_pages: ?usize = null; while (true) : (page += 1) { - if (max_pages != null and page > max_pages.?) break; + if (page > 91) break; const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); std.log.debug("{}: {}", .{ page, r }); @@ -72,13 +71,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View { continue; } const response_string = try lastfm_response_buffer.toOwnedSlice(); - const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); - //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); - if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); - try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track); + const parsed_lastfm_response = try Utils.scrobbleIngest(request.allocator, response_string); + //const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); + //if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); + try scrobble_buffer.appendSlice(parsed_lastfm_response); } - break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items }; + break :blk try scrobble_buffer.toOwnedSlice(); }, else => unreachable, }; @@ -90,254 +89,92 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var albumsongs = try job.params.put("albumsongs", .object); var albumsongsartists = try job.params.put("albumsongsartists", .object); - // Not sure if I should be proud or feel sick - switch (imported_scrobbles) { - .LastFMStats => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { - limited_tracks += 1; - continue :appends; - } - const filtered_scrobble = Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date * 1_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + appends: for (imported_scrobbles) |scrobble| { + if (scrobble.date > latest_timestamp * std.time.ns_per_s or scrobble.date < earliest_timestamp * std.time.ns_per_s) { + limited_tracks += 1; + continue :appends; + } + if (scrobble.playtime != null and scrobble.playtime.? < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.track_artist == null or scrobble.album_artist == null or scrobble.track == null) { + skipped_tracks += 1; + continue :appends; + } - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + const filtered_scrobble = Data.Scrobble{ + .album = scrobble.album.?, + .artists_album = &.{scrobble.album_artist.?}, + .artists_track = &.{scrobble.track_artist.?}, + .date = scrobble.date, + .track = scrobble.track.?, + }; - try view_params.append(row); - //try job_params.append(complete_scrobble); - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + try view_params.append(row); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); } - }, - .LastFMWeb => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.date == null) continue :appends; - const filtered_scrobble = Data.Scrobble{ - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, - .track = scrobble.name, - .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + } - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - try view_params.append(row); - //try job_params.append(complete_scrobble); - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } - - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); } - }, - .Spotify => |scrobbles| { - appends: for (scrobbles) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { - limited_tracks += 1; - continue :appends; - } - - const filtered_scrobble = Data.Scrobble{ - .album = scrobble.master_metadata_album_album_name.?, - .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .track = scrobble.master_metadata_track_name.?, - .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, - }; - const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; - - const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); - - try view_params.append(row); - //try job_params.append(complete_scrobble); - - var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); - - var album_hash_string = std.ArrayList(u8).init(request.allocator); - for (complete_scrobble.artists_album) |artist| { - try album_hash_string.appendSlice(artist); - const artist_hash = std.hash.Fnv1a_64.hash(artist); - try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); - } - - try album_hash_string.appendSlice(complete_scrobble.album); - const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); - if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); - - for (stored_artist_hashes.items) |artist_hash| { - const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); - if (tracks.get(signed_artistalbums_hash_string) == null) { - var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); - try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); - try artistalbum.put("album", @as(i64, @bitCast(album_hash))); - } - } - - var track_hash_string = std.ArrayList(u8).init(request.allocator); - try track_hash_string.appendSlice(complete_scrobble.album); - try track_hash_string.appendSlice(complete_scrobble.track); - const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); - if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); - - const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); - if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { - var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); - } else { - var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); - try albumsong.put("album", @as(i64, @bitCast(album_hash))); - try albumsong.put("song", @as(i64, @bitCast(track_hash))); - var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); - } - - for (complete_scrobble.artists_track) |artist| { - const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); - if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); - const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); - if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { - var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); - try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); - try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); - } - } - } - }, + } } std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 28e7b10..da677f9 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -50,3 +50,233 @@ pub fn urlDecode(allocator: std.mem.Allocator, str: []const u8) ![]const u8 { } return decoded.toOwnedSlice(); } + +const ScrobbleFields = enum { + date, // LastFM(Stats) timestamp + ts, // Spotify timestamp + name, // LastFM track name + track, // LastFMStats track name + master_metadata_track_name, // Spotify track name + artist, // LastFM(Stats) artist name + master_metadata_album_artist_name, // Spotify artist name + album, // LastFM(Stats) album name + master_metadata_album_album_name, // Spotify album name + ms_played, // Spotify playtime + reason_end, // Spotify reason end,1_000 + @"@attr", // LastFM now playing + irrelevant, +}; + +pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.UnifiedScrobble { + var scanner = std.json.Scanner.initCompleteInput(allocator, input); + defer scanner.deinit(); + + var out = std.ArrayList(Data.UnifiedScrobble).init(allocator); + + array: switch (try scanner.peekNextTokenType()) { + .array_begin => { + // Go into array + _ = try scanner.next(); + while (try scanner.peekNextTokenType() != .array_end) { + var r: Data.UnifiedScrobble = undefined; + // Go into object + _ = try scanner.next(); + while (try scanner.peekNextTokenType() != .object_end) { + const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const field_name = std.meta.stringToEnum(ScrobbleFields, switch (key_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }) orelse .irrelevant; + switch (field_name) { + .@"@attr" => { + freeAllocated(allocator, key_token); + r = undefined; + try scanner.skipUntilStackHeight(3); + }, + .ts, .date => |d| { + freeAllocated(allocator, key_token); + const date = switch (d) { + .date => blk: { + if (try scanner.peekNextTokenType() == .object_begin) { + // For now, try to just skip over the object_begin and assume the next field is uts + _ = try scanner.next(); + try scanner.skipValue(); + const lfw_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + try scanner.skipValue(); + try scanner.skipValue(); + _ = try scanner.next(); + const lfw_date = try std.fmt.parseInt(i64, switch (lfw_date_token) { + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }, 10) * std.time.ns_per_s; + freeAllocated(allocator, lfw_date_token); + break :blk lfw_date; + } else { + const lfs_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const lfs_date = try std.fmt.parseInt(i64, switch (lfs_date_token) { + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }, 10) * std.time.ns_per_ms; + freeAllocated(allocator, lfs_date_token); + break :blk lfs_date; + } + }, + .ts => blk: { + // This might need to be an alloc_always, but I'm gonna try if_needed first + const spotify_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const spotify_date = try zeit.instant(.{ .source = .{ .iso8601 = switch (spotify_date_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + } } }); + freeAllocated(allocator, spotify_date_token); + break :blk spotify_date.unixTimestamp() * std.time.ns_per_s; + }, + else => unreachable, + }; + @field(r, "date") = date; + }, + .ms_played => { + freeAllocated(allocator, key_token); + const spotify_ms_played = try scanner.nextAlloc(allocator, .alloc_if_needed); + @field(r, "playtime") = try std.fmt.parseInt(u64, switch (spotify_ms_played) { + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }, 10); + freeAllocated(allocator, spotify_ms_played); + }, + .master_metadata_track_name, .track, .name => { + freeAllocated(allocator, key_token); + const track = try scanner.nextAlloc(allocator, .alloc_always); + @field(r, "track") = switch (track) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }, + .master_metadata_album_artist_name, .artist => { + freeAllocated(allocator, key_token); + const artist = if (try scanner.peekNextTokenType() == .object_begin) blk: { + // Skip object_begin, mbid key, mbid, and #text key + _ = try scanner.next(); + try scanner.skipValue(); + try scanner.skipValue(); + try scanner.skipValue(); + const lfw_artist_token = try scanner.nextAlloc(allocator, .alloc_always); + // Leave object + _ = try scanner.next(); + break :blk switch (lfw_artist_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + } else blk: { + const artist_token = try scanner.nextAlloc(allocator, .alloc_always); + break :blk switch (artist_token) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }; + @field(r, "track_artist") = artist; + @field(r, "album_artist") = artist; + }, + .master_metadata_album_album_name, .album => { + freeAllocated(allocator, key_token); + const album = if (try scanner.peekNextTokenType() == .object_begin) blk: { + // Skip object_begin, mbid key, mbid, and #text key + _ = try scanner.next(); + try scanner.skipValue(); + try scanner.skipValue(); + try scanner.skipValue(); + const lfw_album_token = try scanner.nextAlloc(allocator, .alloc_always); + // Leave object + _ = try scanner.next(); + break :blk switch (lfw_album_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + } else blk: { + const album_token = try scanner.nextAlloc(allocator, .alloc_always); + break :blk switch (album_token) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }; + @field(r, "album") = album; + }, + .reason_end => { + freeAllocated(allocator, key_token); + const reason_end = try scanner.nextAlloc(allocator, .alloc_always); + @field(r, "reason_end") = switch (reason_end) { + inline .string, .allocated_string => |slice| slice, + .null => null, + else => return error.UnexpectedToken, + }; + }, + else => { + freeAllocated(allocator, key_token); + try scanner.skipValue(); + }, + } + } + // Exit object + _ = try scanner.next(); + try out.append(r); + } + }, + // LastFM(stats) + .object_begin => { + _ = try scanner.next(); + find_array: while (true) { + const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const field_name = switch (key_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + if (!std.mem.eql(u8, field_name, "scrobbles") and !std.mem.eql(u8, field_name, "recenttracks")) { + freeAllocated(allocator, key_token); + try scanner.skipValue(); + } else { + freeAllocated(allocator, key_token); + break :find_array; + } + } + switch (try scanner.peekNextTokenType()) { + // LastFM Stats + .array_begin => continue :array .array_begin, + // LastFM + .object_begin => { + // Enter recenttracks + _ = try scanner.next(); + while (true) { + const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed); + const field_name = switch (key_token) { + inline .string, .allocated_string => |slice| slice, + else => return error.UnexpectedToken, + }; + if (!std.mem.eql(u8, field_name, "track")) { + freeAllocated(allocator, key_token); + try scanner.skipValue(); + } else { + freeAllocated(allocator, key_token); + continue :array .array_begin; + } + } + }, + else => unreachable, + } + }, + else => return error.UnexpectedToken, + } + const scrobbles = try out.toOwnedSlice(); + return scrobbles; +} + +fn freeAllocated(allocator: std.mem.Allocator, token: std.json.Token) void { + switch (token) { + .allocated_number, .allocated_string => |slice| { + allocator.free(slice); + }, + else => {}, + } +} diff --git a/src/types.zig b/src/types.zig index 0a66ae7..df067e8 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,3 +1,17 @@ +const std = @import("std"); + +pub const UnifiedScrobble = struct { + track: ?[]const u8, + // These can be null per Spotify + track_artist: ?[]const u8, + album: ?[]const u8, + album_artist: ?[]const u8, + date: i64, + // Relevant Spotify data + playtime: ?u64 = null, + reason_end: ?[]const u8 = null, +}; + pub const ImportedScrobbles = union(ScrobbleSources) { LastFMStats: []IgnorantScrobble, LastFMWeb: []LastFMWebScrobble,