mitteneer/zuletzt
1
0
Fork 0
Code Issues 14 Pull requests Projects Releases Packages Wiki Activity Actions

Parse ScrobbleArray type rather than individual types using jsonParse

Browse source 
POC for parsing and data validation inside jsonParse rather than in the view. I believe this is necessarily slower (and actually doesn't work for Spotify scrobbles) but could be made to work/be faster if we implemented our own json parsing for each type, but I think that's too much work for too little gain (atm)
This commit is contained in:
mitteneer 2025-06-14 16:29:16 -04:00
parent 36873053bc
commit acef5d8e49
2 changed files with 144 additions and 256 deletions
Download patch file Download diff file Expand all files Collapse all files

220
src/app/views/upload.zig
View file

@ -24,7 +24,6 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
defer rule_file.close(); defer rule_file.close();
const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000);
const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null;
//var job = try request.job("process_scrobbles");
var job = try request.job("process_scrobbles2"); var job = try request.job("process_scrobbles2");
const source = params.getT(.integer, "t").?; // This param is required in HTML const source = params.getT(.integer, "t").?; // This param is required in HTML
@ -42,38 +41,36 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
const latest_timestamp = latest_date.instant().unixTimestamp(); const latest_timestamp = latest_date.instant().unixTimestamp();
var view_params = try root.put("scrobbles", .array); var view_params = try root.put("scrobbles", .array);
//var job_params = try job.params.put("scrobbles", .array);
var skipped_tracks: u64 = 0; const imported_scrobbles = switch (source) {
var limited_tracks: u64 = 0; 0, 1 => (try std.json.parseFromSliceLeaky(Data.ScrobbleArray, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles,
const imported_scrobbles: Data.ImportedScrobbles = switch (source) {
0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles },
1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) },
2 => blk: { 2 => blk: {
const user_agent: []const u8 = "Zuletzt/0.0.1"; const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator }; var client = Client{ .allocator = request.allocator };
var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator);
var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator); var scrobble_buffer = std.ArrayList(Data.Scrobble).init(request.allocator);
const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; const username = if (params.getT(.string, "username")) |un| un else "VAOTM";
const mp_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, 0 });
const mp_r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = mp_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
std.log.debug("Max page query: {}", .{mp_r});
const parsed_mp_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, (try lastfm_response_buffer.toOwnedSlice()), .{ .ignore_unknown_fields = true });
var page: usize = 1; var page: usize = 1;
var max_pages: ?usize = null; const max_pages: usize = try std.fmt.parseInt(usize, parsed_mp_response.recenttracks.@"@attr".totalPages, 10);
while (true) : (page += 1) { while (true) : (page += 1) {
if (max_pages != null and page > max_pages.?) break; if (page > max_pages) break;
const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page });
const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
std.log.debug("{}: {}", .{ page, r }); std.log.debug("{}: {}", .{ page, r });
const response_string = try lastfm_response_buffer.toOwnedSlice(); const response_string = try lastfm_response_buffer.toOwnedSlice();
const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.ScrobbleArray, request.allocator, response_string, .{ .ignore_unknown_fields = true });
//const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); try scrobble_buffer.appendSlice(parsed_lastfm_response.scrobbles);
if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10);
try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track);
} }
break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items }; break :blk scrobble_buffer.items;
}, },
else => unreachable, else => unreachable,
}; };
@ -85,27 +82,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
var albumsongs = try job.params.put("albumsongs", .object); var albumsongs = try job.params.put("albumsongs", .object);
var albumsongsartists = try job.params.put("albumsongsartists", .object); var albumsongsartists = try job.params.put("albumsongsartists", .object);
// Not sure if I should be proud or feel sick for (imported_scrobbles) |scrobble| {
switch (imported_scrobbles) { if (scrobble.date > latest_timestamp * 1_000_000 or scrobble.date < earliest_timestamp * 1_000_000) continue;
.LastFMStats => |scrobbles| { const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, scrobble, rl) else scrobble;
appends: for (scrobbles) |scrobble| {
if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) {
limited_tracks += 1;
continue :appends;
}
const filtered_scrobble = Data.Scrobble{
.album = scrobble.album,
.artists_album = &[_][]const u8{scrobble.artist},
.track = scrobble.track,
.artists_track = &[_][]const u8{scrobble.artist},
.date = scrobble.date * 1_000,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
try view_params.append(row); try view_params.append(row);
//try job_params.append(complete_scrobble);
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
var album_hash_string = std.ArrayList(u8).init(request.allocator); var album_hash_string = std.ArrayList(u8).init(request.allocator);
@ -165,177 +148,6 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
} }
} }
} }
},
.LastFMWeb => |scrobbles| {
appends: for (scrobbles) |scrobble| {
if (scrobble.date == null) continue :appends;
const filtered_scrobble = Data.Scrobble{
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artists_album = &[_][]const u8{scrobble.artist.@"#text"},
.track = scrobble.name,
.artists_track = &[_][]const u8{scrobble.artist.@"#text"},
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
try view_params.append(row);
//try job_params.append(complete_scrobble);
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
var album_hash_string = std.ArrayList(u8).init(request.allocator);
for (complete_scrobble.artists_album) |artist| {
try album_hash_string.appendSlice(artist);
const artist_hash = std.hash.Fnv1a_64.hash(artist);
try stored_artist_hashes.append(artist_hash);
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
}
try album_hash_string.appendSlice(complete_scrobble.album);
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
for (stored_artist_hashes.items) |artist_hash| {
const artistalbum_hash = pair(artist_hash, album_hash);
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
if (tracks.get(signed_artistalbums_hash_string) == null) {
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
}
}
var track_hash_string = std.ArrayList(u8).init(request.allocator);
try track_hash_string.appendSlice(complete_scrobble.album);
try track_hash_string.appendSlice(complete_scrobble.track);
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
const albumsong_hash = pair(album_hash, track_hash);
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
var albumsong_scrobbles = albumsong.get("scrobbles");
try albumsong_scrobbles.?.append(complete_scrobble.date);
} else {
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
try albumsong_scrobbles.append(complete_scrobble.date);
}
for (complete_scrobble.artists_track) |artist| {
const artist_hash = std.hash.Fnv1a_64.hash(artist);
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
}
}
}
},
.Spotify => |scrobbles| {
appends: for (scrobbles) |scrobble| {
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
skipped_tracks += 1;
continue :appends;
}
if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
skipped_tracks += 1;
continue :appends;
}
const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) {
limited_tracks += 1;
continue :appends;
}
const filtered_scrobble = Data.Scrobble{
.album = scrobble.master_metadata_album_album_name.?,
.artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
.track = scrobble.master_metadata_track_name.?,
.artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
try view_params.append(row);
//try job_params.append(complete_scrobble);
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
var album_hash_string = std.ArrayList(u8).init(request.allocator);
for (complete_scrobble.artists_album) |artist| {
try album_hash_string.appendSlice(artist);
const artist_hash = std.hash.Fnv1a_64.hash(artist);
try stored_artist_hashes.append(artist_hash);
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
}
try album_hash_string.appendSlice(complete_scrobble.album);
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
for (stored_artist_hashes.items) |artist_hash| {
const artistalbum_hash = pair(artist_hash, album_hash);
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
if (tracks.get(signed_artistalbums_hash_string) == null) {
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
}
}
var track_hash_string = std.ArrayList(u8).init(request.allocator);
try track_hash_string.appendSlice(complete_scrobble.album);
try track_hash_string.appendSlice(complete_scrobble.track);
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
const albumsong_hash = pair(album_hash, track_hash);
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
var albumsong_scrobbles = albumsong.get("scrobbles");
try albumsong_scrobbles.?.append(complete_scrobble.date);
} else {
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
try albumsong_scrobbles.append(complete_scrobble.date);
}
for (complete_scrobble.artists_track) |artist| {
const artist_hash = std.hash.Fnv1a_64.hash(artist);
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
}
}
}
},
}
std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks });
try job.schedule(); try job.schedule();
return request.render(.created); return request.render(.created);

76
src/types.zig
View file

@ -1,9 +1,21 @@
const std = @import("std");
const zeit = @import("zeit");
pub const ImportedScrobbles = union(ScrobbleSources) { pub const ImportedScrobbles = union(ScrobbleSources) {
LastFMStats: []IgnorantScrobble, LastFMStats: []IgnorantScrobble,
LastFMWeb: []LastFMWebScrobble, LastFMWeb: []LastFMWebScrobble,
Spotify: []SpotifyScrobble, Spotify: []SpotifyScrobble,
}; };
const Litmus = struct {
username: ?[]const u8 = null,
ts: ?[]const u8 = null,
recenttracks: ?struct {
track: []LastFMWebScrobble,
@"@attr": LastFMWebQueryInfo,
} = null,
};
const ScrobbleSources = enum { const ScrobbleSources = enum {
LastFMStats, LastFMStats,
LastFMWeb, LastFMWeb,
@ -26,6 +38,70 @@ pub const Scrobble = struct {
date: i64, date: i64,
}; };
pub const ScrobbleArray = struct {
scrobbles: []Scrobble,
// This is an abuse of the jsonParse function. I don't like the idea of doing it, but I really like the results
// (or at least I will, assuming it works)
pub fn jsonParse(allocator: std.mem.Allocator, source: *std.json.Scanner, options: std.json.ParseOptions) !ScrobbleArray {
while (try source.peekNextTokenType() != .end_of_document) try source.skipValue();
const litmus_test = try std.json.parseFromSliceLeaky(Litmus, allocator, source.input, .{ .ignore_unknown_fields = true });
if (litmus_test.username != null) { // LastFMStats
const lastfm_file = try std.json.parseFromSliceLeaky(LastFMStats, allocator, source.input, options);
var scrobble_buffer = try allocator.alloc(Scrobble, lastfm_file.scrobbles.len);
for (lastfm_file.scrobbles, 0..lastfm_file.scrobbles.len) |scrobble, i| {
const artist = try allocator.alloc([]const u8, 1);
artist[0] = scrobble.artist;
scrobble_buffer[i] = Scrobble{
.album = scrobble.album,
.artists_album = artist,
.track = scrobble.track,
.artists_track = artist,
.date = scrobble.date * 1_000,
};
}
return ScrobbleArray{ .scrobbles = scrobble_buffer };
}
if (litmus_test.ts != null) { // Spotify
const spotify = try std.json.parseFromSliceLeaky([]SpotifyScrobble, allocator, source.input, options);
var scrobble_buffer = try allocator.alloc(Scrobble, spotify.len);
for (spotify, 0..spotify.len) |scrobble, i| {
const artist = try allocator.alloc([]const u8, 1);
artist[0] = scrobble.master_metadata_album_artist_name.?;
scrobble_buffer[i] = Scrobble{
.album = scrobble.master_metadata_album_album_name.?,
.artists_album = artist,
.track = scrobble.master_metadata_track_name.?,
.artists_track = artist,
.date = (zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } }) catch return error.OutOfMemory).unixTimestamp() * 1_000_000,
};
}
return ScrobbleArray{ .scrobbles = scrobble_buffer };
}
if (litmus_test.recenttracks != null) { // LastFM API
const lastfm_web = try std.json.parseFromSliceLeaky(LastFMWeb, allocator, source.input, options);
var scrobble_buffer = try allocator.alloc(Scrobble, lastfm_web.recenttracks.track.len);
for (lastfm_web.recenttracks.track, 0..lastfm_web.recenttracks.track.len) |scrobble, i| {
const artist = try allocator.alloc([]const u8, 1);
artist[0] = scrobble.artist.@"#text";
scrobble_buffer[i] = Scrobble{
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artists_album = artist,
.track = scrobble.name,
.artists_track = artist,
.date = (std.fmt.parseInt(i64, scrobble.date.?.uts, 10) catch return error.OutOfMemory) * 1_000_000,
};
}
return ScrobbleArray{ .scrobbles = scrobble_buffer };
}
return error.UnexpectedToken;
}
};
// From lastfmstats.com // From lastfmstats.com
pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble }; pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble };