mitteneer/zuletzt
1
0
Fork 0
Code Issues 14 Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: mitteneer:902fcd4447fde65eafb518c35c0d8d63506e510e
Branches Tags
mitteneer:main
mitteneer:rawsql
mitteneer:hijack-jsonParse
..
compare: mitteneer:15e72ea3262de56b2e15823a354a979b3d649637
Branches Tags
mitteneer:rawsql
mitteneer:hijack-jsonParse
mitteneer:main

No commits in common. "902fcd4447fde65eafb518c35c0d8d63506e510e" and "15e72ea3262de56b2e15823a354a979b3d649637" have entirely different histories.
902fcd4447 ... 15e72ea326

10 changed files with 258 additions and 341 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/app/views/albums.zig
View file

@ -55,8 +55,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View {
const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int});
try root.put("reviews", ratings); try root.put("reviews", ratings);
//const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int}); const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.album, .peak), .{id_int});
//try root.put("peak", peak); try root.put("peak", peak);
return request.render(.ok); return request.render(.ok);
} }

1
src/app/views/albums/get.zmpl
View file

@ -31,6 +31,7 @@
@else @else
<div>{{.album.scrobbles}} scrobbles ({{.album.rank}} place)</div> <div>{{.album.scrobbles}} scrobbles ({{.album.rank}} place)</div>
@end @end
<div>All-time peak: {{.peak.rank}} ({{.peak.date}})</div>
<div>{{.album.song_num}} songs</div> <div>{{.album.song_num}} songs</div>
@partial partials/firstlast_listens(firstlast: .firstlast) @partial partials/firstlast_listens(firstlast: .firstlast)
<h3>Yearly Performance</h3> <h3>Yearly Performance</h3>

4
src/app/views/artists.zig
View file

@ -54,8 +54,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View {
const timescale = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .timescale), .{id_int}); const timescale = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .timescale), .{id_int});
try root.put("yearly", timescale); try root.put("yearly", timescale);
//const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int}); const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.artist, .peak), .{id_int});
//try root.put("peak", peak); try root.put("peak", peak);
return request.render(.ok); return request.render(.ok);
} }

1
src/app/views/artists/get.zmpl
View file

@ -22,6 +22,7 @@
@else @else
<div>{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)</div> <div>{{.artist.scrobbles}} scrobbles ({{.artist.rank}} place)</div>
@end @end
<div>All-time peak: {{.peak.rank}} ({{.peak.date}})</div>
<div>{{.artist.song_num}} songs</div> <div>{{.artist.song_num}} songs</div>
<div>{{.artist.album_num}} albums</div> <div>{{.artist.album_num}} albums</div>
</div> </div>

4
src/app/views/songs.zig
View file

@ -60,8 +60,8 @@ pub fn get(id: []const u8, request: *jetzig.Request) !jetzig.View {
const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int}); const ratings = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .get_ratings), .{id_int});
try root.put("reviews", ratings); try root.put("reviews", ratings);
//const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .peak), .{id_int}); const peak = try queries.entityQueryResult(request, comptime queries.loadQuery(.song, .peak), .{id_int});
//try root.put("peak", peak); try root.put("peak", peak);
return request.render(.ok); return request.render(.ok);
} }

1
src/app/views/songs/get.zmpl
View file

@ -30,6 +30,7 @@
@else @else
<div>{{.song.scrobbles}} scrobbles ({{.song.rank}} place)</div> <div>{{.song.scrobbles}} scrobbles ({{.song.rank}} place)</div>
@end @end
<div>All-time peak: {{.peak.rank}} ({{.peak.date}})</div>
@partial partials/firstlast_listens(firstlast: .firstlast) @partial partials/firstlast_listens(firstlast: .firstlast)
<h3>Yearly Performance</h3> <h3>Yearly Performance</h3>
@partial partials/timescale(range: .yearly) @partial partials/timescale(range: .yearly)

336
src/app/views/upload.zig
View file

@ -47,37 +47,33 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
var skipped_tracks: u64 = 0; var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0; var limited_tracks: u64 = 0;
const imported_scrobbles: []Data.UnifiedScrobble = switch (source) { const imported_scrobbles: Data.ImportedScrobbles = switch (source) {
0, 1 => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles },
1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) },
2 => blk: { 2 => blk: {
const user_agent: []const u8 = "Zuletzt/0.0.1"; const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator }; var client = Client{ .allocator = request.allocator };
var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator);
var scrobble_buffer = std.ArrayList(Data.UnifiedScrobble).init(request.allocator); var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator);
const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; const username = if (params.getT(.string, "username")) |un| un else "VAOTM";
var page: usize = 1; var page: usize = 1;
//var max_pages: ?usize = null; var max_pages: ?usize = null;
while (true) : (page += 1) { while (true) : (page += 1) {
if (page > 91) break; if (max_pages != null and page > max_pages.?) break;
const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page });
const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
std.log.debug("{}: {}", .{ page, r }); std.log.debug("{}: {}", .{ page, r });
if (@intFromEnum(r.status) == 500) {
page -= 1;
std.time.sleep(3 * std.time.ns_per_s);
continue;
}
const response_string = try lastfm_response_buffer.toOwnedSlice(); const response_string = try lastfm_response_buffer.toOwnedSlice();
const parsed_lastfm_response = try Utils.scrobbleIngest(request.allocator, response_string); const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true });
//const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10);
//if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10);
try scrobble_buffer.appendSlice(parsed_lastfm_response); try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track);
} }
break :blk try scrobble_buffer.toOwnedSlice(); break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items };
}, },
else => unreachable, else => unreachable,
}; };
@ -89,92 +85,254 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
var albumsongs = try job.params.put("albumsongs", .object); var albumsongs = try job.params.put("albumsongs", .object);
var albumsongsartists = try job.params.put("albumsongsartists", .object); var albumsongsartists = try job.params.put("albumsongsartists", .object);
appends: for (imported_scrobbles) |scrobble| { // Not sure if I should be proud or feel sick
if (scrobble.date > latest_timestamp * std.time.ns_per_s or scrobble.date < earliest_timestamp * std.time.ns_per_s) { switch (imported_scrobbles) {
limited_tracks += 1; .LastFMStats => |scrobbles| {
continue :appends; appends: for (scrobbles) |scrobble| {
} if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) {
if (scrobble.playtime != null and scrobble.playtime.? < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { limited_tracks += 1;
skipped_tracks += 1; continue :appends;
continue :appends; }
} const filtered_scrobble = Data.Scrobble{
if (scrobble.track_artist == null or scrobble.album_artist == null or scrobble.track == null) { .album = scrobble.album,
skipped_tracks += 1; .artists_album = &[_][]const u8{scrobble.artist},
continue :appends; .track = scrobble.track,
} .artists_track = &[_][]const u8{scrobble.artist},
.date = scrobble.date * 1_000,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
const filtered_scrobble = Data.Scrobble{ const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
.album = scrobble.album.?,
.artists_album = &.{scrobble.album_artist.?},
.artists_track = &.{scrobble.track_artist.?},
.date = scrobble.date,
.track = scrobble.track.?,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; try view_params.append(row);
//try job_params.append(complete_scrobble);
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); var album_hash_string = std.ArrayList(u8).init(request.allocator);
for (complete_scrobble.artists_album) |artist| {
try album_hash_string.appendSlice(artist);
const artist_hash = std.hash.Fnv1a_64.hash(artist);
try stored_artist_hashes.append(artist_hash);
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
}
try view_params.append(row); try album_hash_string.appendSlice(complete_scrobble.album);
//try job_params.append(complete_scrobble); const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
var album_hash_string = std.ArrayList(u8).init(request.allocator); for (stored_artist_hashes.items) |artist_hash| {
for (complete_scrobble.artists_album) |artist| { const artistalbum_hash = pair(artist_hash, album_hash);
try album_hash_string.appendSlice(artist); const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
const artist_hash = std.hash.Fnv1a_64.hash(artist); if (tracks.get(signed_artistalbums_hash_string) == null) {
try stored_artist_hashes.append(artist_hash); var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
} }
}
try album_hash_string.appendSlice(complete_scrobble.album); var track_hash_string = std.ArrayList(u8).init(request.allocator);
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); try track_hash_string.appendSlice(complete_scrobble.album);
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); try track_hash_string.appendSlice(complete_scrobble.track);
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
for (stored_artist_hashes.items) |artist_hash| { const albumsong_hash = pair(album_hash, track_hash);
const artistalbum_hash = pair(artist_hash, album_hash); const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
if (tracks.get(signed_artistalbums_hash_string) == null) { var albumsong_scrobbles = albumsong.get("scrobbles");
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); try albumsong_scrobbles.?.append(complete_scrobble.date);
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); } else {
try artistalbum.put("album", @as(i64, @bitCast(album_hash))); var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
try albumsong_scrobbles.append(complete_scrobble.date);
}
for (complete_scrobble.artists_track) |artist| {
const artist_hash = std.hash.Fnv1a_64.hash(artist);
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
}
}
} }
} },
.LastFMWeb => |scrobbles| {
appends: for (scrobbles) |scrobble| {
if (scrobble.date == null) continue :appends;
const filtered_scrobble = Data.Scrobble{
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artists_album = &[_][]const u8{scrobble.artist.@"#text"},
.track = scrobble.name,
.artists_track = &[_][]const u8{scrobble.artist.@"#text"},
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
var track_hash_string = std.ArrayList(u8).init(request.allocator); const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
try track_hash_string.appendSlice(complete_scrobble.album);
try track_hash_string.appendSlice(complete_scrobble.track);
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
const albumsong_hash = pair(album_hash, track_hash); try view_params.append(row);
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); //try job_params.append(complete_scrobble);
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
var albumsong_scrobbles = albumsong.get("scrobbles");
try albumsong_scrobbles.?.append(complete_scrobble.date);
} else {
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
try albumsong_scrobbles.append(complete_scrobble.date);
}
for (complete_scrobble.artists_track) |artist| { var album_hash_string = std.ArrayList(u8).init(request.allocator);
const artist_hash = std.hash.Fnv1a_64.hash(artist); for (complete_scrobble.artists_album) |artist| {
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); try album_hash_string.appendSlice(artist);
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); const artist_hash = std.hash.Fnv1a_64.hash(artist);
const albumsongsartist_hash = pair(albumsong_hash, artist_hash); try stored_artist_hashes.append(artist_hash);
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); }
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); try album_hash_string.appendSlice(complete_scrobble.album);
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
for (stored_artist_hashes.items) |artist_hash| {
const artistalbum_hash = pair(artist_hash, album_hash);
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
if (tracks.get(signed_artistalbums_hash_string) == null) {
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
}
}
var track_hash_string = std.ArrayList(u8).init(request.allocator);
try track_hash_string.appendSlice(complete_scrobble.album);
try track_hash_string.appendSlice(complete_scrobble.track);
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
const albumsong_hash = pair(album_hash, track_hash);
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
var albumsong_scrobbles = albumsong.get("scrobbles");
try albumsong_scrobbles.?.append(complete_scrobble.date);
} else {
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
try albumsong_scrobbles.append(complete_scrobble.date);
}
for (complete_scrobble.artists_track) |artist| {
const artist_hash = std.hash.Fnv1a_64.hash(artist);
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
}
}
} }
} },
.Spotify => |scrobbles| {
appends: for (scrobbles) |scrobble| {
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
skipped_tracks += 1;
continue :appends;
}
if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
skipped_tracks += 1;
continue :appends;
}
const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) {
limited_tracks += 1;
continue :appends;
}
const filtered_scrobble = Data.Scrobble{
.album = scrobble.master_metadata_album_album_name.?,
.artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
.track = scrobble.master_metadata_track_name.?,
.artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
try view_params.append(row);
//try job_params.append(complete_scrobble);
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
var album_hash_string = std.ArrayList(u8).init(request.allocator);
for (complete_scrobble.artists_album) |artist| {
try album_hash_string.appendSlice(artist);
const artist_hash = std.hash.Fnv1a_64.hash(artist);
try stored_artist_hashes.append(artist_hash);
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
}
try album_hash_string.appendSlice(complete_scrobble.album);
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
for (stored_artist_hashes.items) |artist_hash| {
const artistalbum_hash = pair(artist_hash, album_hash);
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
if (tracks.get(signed_artistalbums_hash_string) == null) {
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
}
}
var track_hash_string = std.ArrayList(u8).init(request.allocator);
try track_hash_string.appendSlice(complete_scrobble.album);
try track_hash_string.appendSlice(complete_scrobble.track);
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
const albumsong_hash = pair(album_hash, track_hash);
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
var albumsong_scrobbles = albumsong.get("scrobbles");
try albumsong_scrobbles.?.append(complete_scrobble.date);
} else {
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
try albumsong_scrobbles.append(complete_scrobble.date);
}
for (complete_scrobble.artists_track) |artist| {
const artist_hash = std.hash.Fnv1a_64.hash(artist);
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
}
}
}
},
} }
std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks });

232
src/date_fmt.zig
View file

@ -4,7 +4,7 @@ const Data = @import("types.zig");
pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 {
var date = std.ArrayList(u8).init(allocator); var date = std.ArrayList(u8).init(allocator);
try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, std.time.ns_per_s) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M");
return date.items; return date.items;
} }
@ -50,233 +50,3 @@ pub fn urlDecode(allocator: std.mem.Allocator, str: []const u8) ![]const u8 {
} }
return decoded.toOwnedSlice(); return decoded.toOwnedSlice();
} }
const ScrobbleFields = enum {
date, // LastFM(Stats) timestamp
ts, // Spotify timestamp
name, // LastFM track name
track, // LastFMStats track name
master_metadata_track_name, // Spotify track name
artist, // LastFM(Stats) artist name
master_metadata_album_artist_name, // Spotify artist name
album, // LastFM(Stats) album name
master_metadata_album_album_name, // Spotify album name
ms_played, // Spotify playtime
reason_end, // Spotify reason end,1_000
@"@attr", // LastFM now playing
irrelevant,
};
pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.UnifiedScrobble {
var scanner = std.json.Scanner.initCompleteInput(allocator, input);
defer scanner.deinit();
var out = std.ArrayList(Data.UnifiedScrobble).init(allocator);
array: switch (try scanner.peekNextTokenType()) {
.array_begin => {
// Go into array
_ = try scanner.next();
while (try scanner.peekNextTokenType() != .array_end) {
var r: Data.UnifiedScrobble = undefined;
// Go into object
_ = try scanner.next();
while (try scanner.peekNextTokenType() != .object_end) {
const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
const field_name = std.meta.stringToEnum(ScrobbleFields, switch (key_token) {
inline .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
}) orelse .irrelevant;
switch (field_name) {
.@"@attr" => {
freeAllocated(allocator, key_token);
r = undefined;
try scanner.skipUntilStackHeight(3);
},
.ts, .date => |d| {
freeAllocated(allocator, key_token);
const date = switch (d) {
.date => blk: {
if (try scanner.peekNextTokenType() == .object_begin) {
// For now, try to just skip over the object_begin and assume the next field is uts
_ = try scanner.next();
try scanner.skipValue();
const lfw_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
try scanner.skipValue();
try scanner.skipValue();
_ = try scanner.next();
const lfw_date = try std.fmt.parseInt(i64, switch (lfw_date_token) {
inline .number, .allocated_number, .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
}, 10) * std.time.ns_per_s;
freeAllocated(allocator, lfw_date_token);
break :blk lfw_date;
} else {
const lfs_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
const lfs_date = try std.fmt.parseInt(i64, switch (lfs_date_token) {
inline .number, .allocated_number, .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
}, 10) * std.time.ns_per_ms;
freeAllocated(allocator, lfs_date_token);
break :blk lfs_date;
}
},
.ts => blk: {
// This might need to be an alloc_always, but I'm gonna try if_needed first
const spotify_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
const spotify_date = try zeit.instant(.{ .source = .{ .iso8601 = switch (spotify_date_token) {
inline .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
} } });
freeAllocated(allocator, spotify_date_token);
break :blk spotify_date.unixTimestamp() * std.time.ns_per_s;
},
else => unreachable,
};
@field(r, "date") = date;
},
.ms_played => {
freeAllocated(allocator, key_token);
const spotify_ms_played = try scanner.nextAlloc(allocator, .alloc_if_needed);
@field(r, "playtime") = try std.fmt.parseInt(u64, switch (spotify_ms_played) {
inline .number, .allocated_number, .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
}, 10);
freeAllocated(allocator, spotify_ms_played);
},
.master_metadata_track_name, .track, .name => {
freeAllocated(allocator, key_token);
const track = try scanner.nextAlloc(allocator, .alloc_always);
@field(r, "track") = switch (track) {
inline .string, .allocated_string => |slice| slice,
.null => null,
else => return error.UnexpectedToken,
};
},
.master_metadata_album_artist_name, .artist => {
freeAllocated(allocator, key_token);
const artist = if (try scanner.peekNextTokenType() == .object_begin) blk: {
// Skip object_begin, mbid key, mbid, and #text key
_ = try scanner.next();
try scanner.skipValue();
try scanner.skipValue();
try scanner.skipValue();
const lfw_artist_token = try scanner.nextAlloc(allocator, .alloc_always);
// Leave object
_ = try scanner.next();
break :blk switch (lfw_artist_token) {
inline .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
};
} else blk: {
const artist_token = try scanner.nextAlloc(allocator, .alloc_always);
break :blk switch (artist_token) {
inline .string, .allocated_string => |slice| slice,
.null => null,
else => return error.UnexpectedToken,
};
};
@field(r, "track_artist") = artist;
@field(r, "album_artist") = artist;
},
.master_metadata_album_album_name, .album => {
freeAllocated(allocator, key_token);
const album = if (try scanner.peekNextTokenType() == .object_begin) blk: {
// Skip object_begin, mbid key, mbid, and #text key
_ = try scanner.next();
try scanner.skipValue();
try scanner.skipValue();
try scanner.skipValue();
const lfw_album_token = try scanner.nextAlloc(allocator, .alloc_always);
// Leave object
_ = try scanner.next();
break :blk switch (lfw_album_token) {
inline .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
};
} else blk: {
const album_token = try scanner.nextAlloc(allocator, .alloc_always);
break :blk switch (album_token) {
inline .string, .allocated_string => |slice| slice,
.null => null,
else => return error.UnexpectedToken,
};
};
@field(r, "album") = album;
},
.reason_end => {
freeAllocated(allocator, key_token);
const reason_end = try scanner.nextAlloc(allocator, .alloc_always);
@field(r, "reason_end") = switch (reason_end) {
inline .string, .allocated_string => |slice| slice,
.null => null,
else => return error.UnexpectedToken,
};
},
else => {
freeAllocated(allocator, key_token);
try scanner.skipValue();
},
}
}
// Exit object
_ = try scanner.next();
try out.append(r);
}
},
// LastFM(stats)
.object_begin => {
_ = try scanner.next();
find_array: while (true) {
const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
const field_name = switch (key_token) {
inline .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
};
if (!std.mem.eql(u8, field_name, "scrobbles") and !std.mem.eql(u8, field_name, "recenttracks")) {
freeAllocated(allocator, key_token);
try scanner.skipValue();
} else {
freeAllocated(allocator, key_token);
break :find_array;
}
}
switch (try scanner.peekNextTokenType()) {
// LastFM Stats
.array_begin => continue :array .array_begin,
// LastFM
.object_begin => {
// Enter recenttracks
_ = try scanner.next();
while (true) {
const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
const field_name = switch (key_token) {
inline .string, .allocated_string => |slice| slice,
else => return error.UnexpectedToken,
};
if (!std.mem.eql(u8, field_name, "track")) {
freeAllocated(allocator, key_token);
try scanner.skipValue();
} else {
freeAllocated(allocator, key_token);
continue :array .array_begin;
}
}
},
else => unreachable,
}
},
else => return error.UnexpectedToken,
}
const scrobbles = try out.toOwnedSlice();
return scrobbles;
}
fn freeAllocated(allocator: std.mem.Allocator, token: std.json.Token) void {
switch (token) {
.allocated_number, .allocated_string => |slice| {
allocator.free(slice);
},
else => {},
}
}

2
src/queries.zig
View file

@ -18,7 +18,7 @@ pub fn entityQueryResult(request: *jetzig.Request, comptime query: GeneratedQuer
var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, .{}, .{ .allocator = request.allocator, .column_names = true }); var result = try request.repo.connection.?.postgresql.connection.queryOpts(query.query, .{}, .{ .allocator = request.allocator, .column_names = true });
var out: *jetzig.Data.Value = try Data.object(); var out: *jetzig.Data.Value = try Data.object();
var mapper = result.mapper(PeakResult, .{ .dupe = true, .allocator = request.allocator }); var mapper = result.mapper(PeakResult, .{ .dupe = true, .allocator = request.allocator });
var rank: u64 = comptime std.math.maxInt(u64); var rank = comptime (std.math.pow(u64, 2, 63) - 1);
var date: []const u8 = undefined; var date: []const u8 = undefined;
var scrobbles = std.AutoArrayHashMap(i64, u32).init(request.allocator); var scrobbles = std.AutoArrayHashMap(i64, u32).init(request.allocator);

14
src/types.zig
View file

@ -1,17 +1,3 @@
const std = @import("std");
pub const UnifiedScrobble = struct {
track: ?[]const u8,
// These can be null per Spotify
track_artist: ?[]const u8,
album: ?[]const u8,
album_artist: ?[]const u8,
date: i64,
// Relevant Spotify data
playtime: ?u64 = null,
reason_end: ?[]const u8 = null,
};
pub const ImportedScrobbles = union(ScrobbleSources) { pub const ImportedScrobbles = union(ScrobbleSources) {
LastFMStats: []IgnorantScrobble, LastFMStats: []IgnorantScrobble,
LastFMWeb: []LastFMWebScrobble, LastFMWeb: []LastFMWebScrobble,