Create own parsing function
I have dreamt of this for a long time. It is a minor optimization to be honest, but I previously had to choose between ugly code (what I had just prior to this) or looping through the data twice (slow). This parses the data and puts it into my intermediary type directly, along with relevant information about whetehr or not the scrobble is valid, and then I only need to loop over it once with "nice enough" code. There is still more I can do. My ultimate goal is to remove the looping entirely, and verify the data as it's being parsed, queuing up much smaller jobs that handle the individual entities (albums, artistsongs, etc.) as they come, rather than collecting it all and running it at once. I can get over the problem of needing to wit for all the LastFM responses that way as well. Furthermore, the data checking in the function expects a rather rigid structure that I'm not certain I can guarantee, but I'm pretty sure it'll stay that way. In any case, it may behoove me to make it more dynamic at some point. In any case, I am very excited about this change, and I hope I can continue improving upon it.
This commit is contained in:
parent
8af6341f95
commit
902fcd4447
3 changed files with 328 additions and 247 deletions
|
|
@ -47,22 +47,21 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
|
|||
var skipped_tracks: u64 = 0;
|
||||
var limited_tracks: u64 = 0;
|
||||
|
||||
const imported_scrobbles: Data.ImportedScrobbles = switch (source) {
|
||||
0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles },
|
||||
1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) },
|
||||
const imported_scrobbles: []Data.UnifiedScrobble = switch (source) {
|
||||
0, 1 => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable),
|
||||
2 => blk: {
|
||||
const user_agent: []const u8 = "Zuletzt/0.0.1";
|
||||
var client = Client{ .allocator = request.allocator };
|
||||
var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator);
|
||||
var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator);
|
||||
var scrobble_buffer = std.ArrayList(Data.UnifiedScrobble).init(request.allocator);
|
||||
|
||||
const username = if (params.getT(.string, "username")) |un| un else "VAOTM";
|
||||
|
||||
var page: usize = 1;
|
||||
var max_pages: ?usize = null;
|
||||
//var max_pages: ?usize = null;
|
||||
|
||||
while (true) : (page += 1) {
|
||||
if (max_pages != null and page > max_pages.?) break;
|
||||
if (page > 91) break;
|
||||
const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page });
|
||||
const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
|
||||
std.log.debug("{}: {}", .{ page, r });
|
||||
|
|
@ -72,13 +71,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
|
|||
continue;
|
||||
}
|
||||
const response_string = try lastfm_response_buffer.toOwnedSlice();
|
||||
const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true });
|
||||
//const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10);
|
||||
if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10);
|
||||
try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track);
|
||||
const parsed_lastfm_response = try Utils.scrobbleIngest(request.allocator, response_string);
|
||||
//const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true });
|
||||
//if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10);
|
||||
try scrobble_buffer.appendSlice(parsed_lastfm_response);
|
||||
}
|
||||
|
||||
break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items };
|
||||
break :blk try scrobble_buffer.toOwnedSlice();
|
||||
},
|
||||
else => unreachable,
|
||||
};
|
||||
|
|
@ -90,254 +89,92 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
|
|||
var albumsongs = try job.params.put("albumsongs", .object);
|
||||
var albumsongsartists = try job.params.put("albumsongsartists", .object);
|
||||
|
||||
// Not sure if I should be proud or feel sick
|
||||
switch (imported_scrobbles) {
|
||||
.LastFMStats => |scrobbles| {
|
||||
appends: for (scrobbles) |scrobble| {
|
||||
if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) {
|
||||
limited_tracks += 1;
|
||||
continue :appends;
|
||||
}
|
||||
const filtered_scrobble = Data.Scrobble{
|
||||
.album = scrobble.album,
|
||||
.artists_album = &[_][]const u8{scrobble.artist},
|
||||
.track = scrobble.track,
|
||||
.artists_track = &[_][]const u8{scrobble.artist},
|
||||
.date = scrobble.date * 1_000,
|
||||
};
|
||||
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
|
||||
appends: for (imported_scrobbles) |scrobble| {
|
||||
if (scrobble.date > latest_timestamp * std.time.ns_per_s or scrobble.date < earliest_timestamp * std.time.ns_per_s) {
|
||||
limited_tracks += 1;
|
||||
continue :appends;
|
||||
}
|
||||
if (scrobble.playtime != null and scrobble.playtime.? < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
|
||||
skipped_tracks += 1;
|
||||
continue :appends;
|
||||
}
|
||||
if (scrobble.track_artist == null or scrobble.album_artist == null or scrobble.track == null) {
|
||||
skipped_tracks += 1;
|
||||
continue :appends;
|
||||
}
|
||||
|
||||
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
|
||||
const filtered_scrobble = Data.Scrobble{
|
||||
.album = scrobble.album.?,
|
||||
.artists_album = &.{scrobble.album_artist.?},
|
||||
.artists_track = &.{scrobble.track_artist.?},
|
||||
.date = scrobble.date,
|
||||
.track = scrobble.track.?,
|
||||
};
|
||||
|
||||
try view_params.append(row);
|
||||
//try job_params.append(complete_scrobble);
|
||||
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
|
||||
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
|
||||
|
||||
var album_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
for (complete_scrobble.artists_album) |artist| {
|
||||
try album_hash_string.appendSlice(artist);
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
try stored_artist_hashes.append(artist_hash);
|
||||
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
|
||||
}
|
||||
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
|
||||
|
||||
try album_hash_string.appendSlice(complete_scrobble.album);
|
||||
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
|
||||
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
|
||||
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
|
||||
try view_params.append(row);
|
||||
//try job_params.append(complete_scrobble);
|
||||
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
|
||||
|
||||
for (stored_artist_hashes.items) |artist_hash| {
|
||||
const artistalbum_hash = pair(artist_hash, album_hash);
|
||||
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
|
||||
if (tracks.get(signed_artistalbums_hash_string) == null) {
|
||||
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
|
||||
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
|
||||
}
|
||||
}
|
||||
var album_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
for (complete_scrobble.artists_album) |artist| {
|
||||
try album_hash_string.appendSlice(artist);
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
try stored_artist_hashes.append(artist_hash);
|
||||
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
|
||||
}
|
||||
|
||||
var track_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
try track_hash_string.appendSlice(complete_scrobble.album);
|
||||
try track_hash_string.appendSlice(complete_scrobble.track);
|
||||
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
|
||||
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
|
||||
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
|
||||
try album_hash_string.appendSlice(complete_scrobble.album);
|
||||
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
|
||||
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
|
||||
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
|
||||
|
||||
const albumsong_hash = pair(album_hash, track_hash);
|
||||
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
|
||||
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
|
||||
var albumsong_scrobbles = albumsong.get("scrobbles");
|
||||
try albumsong_scrobbles.?.append(complete_scrobble.date);
|
||||
} else {
|
||||
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
|
||||
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
|
||||
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
|
||||
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
|
||||
try albumsong_scrobbles.append(complete_scrobble.date);
|
||||
}
|
||||
|
||||
for (complete_scrobble.artists_track) |artist| {
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
|
||||
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
|
||||
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
|
||||
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
|
||||
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
|
||||
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
|
||||
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
}
|
||||
}
|
||||
for (stored_artist_hashes.items) |artist_hash| {
|
||||
const artistalbum_hash = pair(artist_hash, album_hash);
|
||||
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
|
||||
if (tracks.get(signed_artistalbums_hash_string) == null) {
|
||||
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
|
||||
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
|
||||
}
|
||||
},
|
||||
.LastFMWeb => |scrobbles| {
|
||||
appends: for (scrobbles) |scrobble| {
|
||||
if (scrobble.date == null) continue :appends;
|
||||
const filtered_scrobble = Data.Scrobble{
|
||||
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
|
||||
.artists_album = &[_][]const u8{scrobble.artist.@"#text"},
|
||||
.track = scrobble.name,
|
||||
.artists_track = &[_][]const u8{scrobble.artist.@"#text"},
|
||||
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000,
|
||||
};
|
||||
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
|
||||
}
|
||||
|
||||
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
|
||||
var track_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
try track_hash_string.appendSlice(complete_scrobble.album);
|
||||
try track_hash_string.appendSlice(complete_scrobble.track);
|
||||
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
|
||||
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
|
||||
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
|
||||
|
||||
try view_params.append(row);
|
||||
//try job_params.append(complete_scrobble);
|
||||
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
|
||||
const albumsong_hash = pair(album_hash, track_hash);
|
||||
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
|
||||
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
|
||||
var albumsong_scrobbles = albumsong.get("scrobbles");
|
||||
try albumsong_scrobbles.?.append(complete_scrobble.date);
|
||||
} else {
|
||||
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
|
||||
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
|
||||
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
|
||||
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
|
||||
try albumsong_scrobbles.append(complete_scrobble.date);
|
||||
}
|
||||
|
||||
var album_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
for (complete_scrobble.artists_album) |artist| {
|
||||
try album_hash_string.appendSlice(artist);
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
try stored_artist_hashes.append(artist_hash);
|
||||
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
|
||||
}
|
||||
|
||||
try album_hash_string.appendSlice(complete_scrobble.album);
|
||||
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
|
||||
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
|
||||
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
|
||||
|
||||
for (stored_artist_hashes.items) |artist_hash| {
|
||||
const artistalbum_hash = pair(artist_hash, album_hash);
|
||||
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
|
||||
if (tracks.get(signed_artistalbums_hash_string) == null) {
|
||||
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
|
||||
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
|
||||
}
|
||||
}
|
||||
|
||||
var track_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
try track_hash_string.appendSlice(complete_scrobble.album);
|
||||
try track_hash_string.appendSlice(complete_scrobble.track);
|
||||
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
|
||||
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
|
||||
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
|
||||
|
||||
const albumsong_hash = pair(album_hash, track_hash);
|
||||
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
|
||||
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
|
||||
var albumsong_scrobbles = albumsong.get("scrobbles");
|
||||
try albumsong_scrobbles.?.append(complete_scrobble.date);
|
||||
} else {
|
||||
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
|
||||
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
|
||||
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
|
||||
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
|
||||
try albumsong_scrobbles.append(complete_scrobble.date);
|
||||
}
|
||||
|
||||
for (complete_scrobble.artists_track) |artist| {
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
|
||||
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
|
||||
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
|
||||
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
|
||||
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
|
||||
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
|
||||
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
}
|
||||
}
|
||||
for (complete_scrobble.artists_track) |artist| {
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
|
||||
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
|
||||
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
|
||||
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
|
||||
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
|
||||
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
|
||||
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
}
|
||||
},
|
||||
.Spotify => |scrobbles| {
|
||||
appends: for (scrobbles) |scrobble| {
|
||||
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
|
||||
skipped_tracks += 1;
|
||||
continue :appends;
|
||||
}
|
||||
if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
|
||||
skipped_tracks += 1;
|
||||
continue :appends;
|
||||
}
|
||||
|
||||
const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
|
||||
if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) {
|
||||
limited_tracks += 1;
|
||||
continue :appends;
|
||||
}
|
||||
|
||||
const filtered_scrobble = Data.Scrobble{
|
||||
.album = scrobble.master_metadata_album_album_name.?,
|
||||
.artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
|
||||
.track = scrobble.master_metadata_track_name.?,
|
||||
.artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
|
||||
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000,
|
||||
};
|
||||
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
|
||||
|
||||
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
|
||||
|
||||
try view_params.append(row);
|
||||
//try job_params.append(complete_scrobble);
|
||||
|
||||
var stored_artist_hashes = std.ArrayList(u64).init(request.allocator);
|
||||
|
||||
var album_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
for (complete_scrobble.artists_album) |artist| {
|
||||
try album_hash_string.appendSlice(artist);
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
try stored_artist_hashes.append(artist_hash);
|
||||
const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist);
|
||||
}
|
||||
|
||||
try album_hash_string.appendSlice(complete_scrobble.album);
|
||||
const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items);
|
||||
const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))});
|
||||
if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album);
|
||||
|
||||
for (stored_artist_hashes.items) |artist_hash| {
|
||||
const artistalbum_hash = pair(artist_hash, album_hash);
|
||||
const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))});
|
||||
if (tracks.get(signed_artistalbums_hash_string) == null) {
|
||||
var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object);
|
||||
try artistalbum.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
try artistalbum.put("album", @as(i64, @bitCast(album_hash)));
|
||||
}
|
||||
}
|
||||
|
||||
var track_hash_string = std.ArrayList(u8).init(request.allocator);
|
||||
try track_hash_string.appendSlice(complete_scrobble.album);
|
||||
try track_hash_string.appendSlice(complete_scrobble.track);
|
||||
const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items);
|
||||
const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))});
|
||||
if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track);
|
||||
|
||||
const albumsong_hash = pair(album_hash, track_hash);
|
||||
const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))});
|
||||
if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| {
|
||||
var albumsong_scrobbles = albumsong.get("scrobbles");
|
||||
try albumsong_scrobbles.?.append(complete_scrobble.date);
|
||||
} else {
|
||||
var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object);
|
||||
try albumsong.put("album", @as(i64, @bitCast(album_hash)));
|
||||
try albumsong.put("song", @as(i64, @bitCast(track_hash)));
|
||||
var albumsong_scrobbles = try albumsong.put("scrobbles", .array);
|
||||
try albumsong_scrobbles.append(complete_scrobble.date);
|
||||
}
|
||||
|
||||
for (complete_scrobble.artists_track) |artist| {
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(artist);
|
||||
const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))});
|
||||
if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist);
|
||||
const albumsongsartist_hash = pair(albumsong_hash, artist_hash);
|
||||
const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash});
|
||||
if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) {
|
||||
var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object);
|
||||
try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash)));
|
||||
try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash)));
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks });
|
||||
|
|
|
|||
230
src/date_fmt.zig
230
src/date_fmt.zig
|
|
@ -50,3 +50,233 @@ pub fn urlDecode(allocator: std.mem.Allocator, str: []const u8) ![]const u8 {
|
|||
}
|
||||
return decoded.toOwnedSlice();
|
||||
}
|
||||
|
||||
const ScrobbleFields = enum {
|
||||
date, // LastFM(Stats) timestamp
|
||||
ts, // Spotify timestamp
|
||||
name, // LastFM track name
|
||||
track, // LastFMStats track name
|
||||
master_metadata_track_name, // Spotify track name
|
||||
artist, // LastFM(Stats) artist name
|
||||
master_metadata_album_artist_name, // Spotify artist name
|
||||
album, // LastFM(Stats) album name
|
||||
master_metadata_album_album_name, // Spotify album name
|
||||
ms_played, // Spotify playtime
|
||||
reason_end, // Spotify reason end,1_000
|
||||
@"@attr", // LastFM now playing
|
||||
irrelevant,
|
||||
};
|
||||
|
||||
pub fn scrobbleIngest(allocator: std.mem.Allocator, input: []const u8) ![]Data.UnifiedScrobble {
|
||||
var scanner = std.json.Scanner.initCompleteInput(allocator, input);
|
||||
defer scanner.deinit();
|
||||
|
||||
var out = std.ArrayList(Data.UnifiedScrobble).init(allocator);
|
||||
|
||||
array: switch (try scanner.peekNextTokenType()) {
|
||||
.array_begin => {
|
||||
// Go into array
|
||||
_ = try scanner.next();
|
||||
while (try scanner.peekNextTokenType() != .array_end) {
|
||||
var r: Data.UnifiedScrobble = undefined;
|
||||
// Go into object
|
||||
_ = try scanner.next();
|
||||
while (try scanner.peekNextTokenType() != .object_end) {
|
||||
const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
|
||||
const field_name = std.meta.stringToEnum(ScrobbleFields, switch (key_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
}) orelse .irrelevant;
|
||||
switch (field_name) {
|
||||
.@"@attr" => {
|
||||
freeAllocated(allocator, key_token);
|
||||
r = undefined;
|
||||
try scanner.skipUntilStackHeight(3);
|
||||
},
|
||||
.ts, .date => |d| {
|
||||
freeAllocated(allocator, key_token);
|
||||
const date = switch (d) {
|
||||
.date => blk: {
|
||||
if (try scanner.peekNextTokenType() == .object_begin) {
|
||||
// For now, try to just skip over the object_begin and assume the next field is uts
|
||||
_ = try scanner.next();
|
||||
try scanner.skipValue();
|
||||
const lfw_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
|
||||
try scanner.skipValue();
|
||||
try scanner.skipValue();
|
||||
_ = try scanner.next();
|
||||
const lfw_date = try std.fmt.parseInt(i64, switch (lfw_date_token) {
|
||||
inline .number, .allocated_number, .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
}, 10) * std.time.ns_per_s;
|
||||
freeAllocated(allocator, lfw_date_token);
|
||||
break :blk lfw_date;
|
||||
} else {
|
||||
const lfs_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
|
||||
const lfs_date = try std.fmt.parseInt(i64, switch (lfs_date_token) {
|
||||
inline .number, .allocated_number, .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
}, 10) * std.time.ns_per_ms;
|
||||
freeAllocated(allocator, lfs_date_token);
|
||||
break :blk lfs_date;
|
||||
}
|
||||
},
|
||||
.ts => blk: {
|
||||
// This might need to be an alloc_always, but I'm gonna try if_needed first
|
||||
const spotify_date_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
|
||||
const spotify_date = try zeit.instant(.{ .source = .{ .iso8601 = switch (spotify_date_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
} } });
|
||||
freeAllocated(allocator, spotify_date_token);
|
||||
break :blk spotify_date.unixTimestamp() * std.time.ns_per_s;
|
||||
},
|
||||
else => unreachable,
|
||||
};
|
||||
@field(r, "date") = date;
|
||||
},
|
||||
.ms_played => {
|
||||
freeAllocated(allocator, key_token);
|
||||
const spotify_ms_played = try scanner.nextAlloc(allocator, .alloc_if_needed);
|
||||
@field(r, "playtime") = try std.fmt.parseInt(u64, switch (spotify_ms_played) {
|
||||
inline .number, .allocated_number, .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
}, 10);
|
||||
freeAllocated(allocator, spotify_ms_played);
|
||||
},
|
||||
.master_metadata_track_name, .track, .name => {
|
||||
freeAllocated(allocator, key_token);
|
||||
const track = try scanner.nextAlloc(allocator, .alloc_always);
|
||||
@field(r, "track") = switch (track) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
.null => null,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
},
|
||||
.master_metadata_album_artist_name, .artist => {
|
||||
freeAllocated(allocator, key_token);
|
||||
const artist = if (try scanner.peekNextTokenType() == .object_begin) blk: {
|
||||
// Skip object_begin, mbid key, mbid, and #text key
|
||||
_ = try scanner.next();
|
||||
try scanner.skipValue();
|
||||
try scanner.skipValue();
|
||||
try scanner.skipValue();
|
||||
const lfw_artist_token = try scanner.nextAlloc(allocator, .alloc_always);
|
||||
// Leave object
|
||||
_ = try scanner.next();
|
||||
break :blk switch (lfw_artist_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
} else blk: {
|
||||
const artist_token = try scanner.nextAlloc(allocator, .alloc_always);
|
||||
break :blk switch (artist_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
.null => null,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
};
|
||||
@field(r, "track_artist") = artist;
|
||||
@field(r, "album_artist") = artist;
|
||||
},
|
||||
.master_metadata_album_album_name, .album => {
|
||||
freeAllocated(allocator, key_token);
|
||||
const album = if (try scanner.peekNextTokenType() == .object_begin) blk: {
|
||||
// Skip object_begin, mbid key, mbid, and #text key
|
||||
_ = try scanner.next();
|
||||
try scanner.skipValue();
|
||||
try scanner.skipValue();
|
||||
try scanner.skipValue();
|
||||
const lfw_album_token = try scanner.nextAlloc(allocator, .alloc_always);
|
||||
// Leave object
|
||||
_ = try scanner.next();
|
||||
break :blk switch (lfw_album_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
} else blk: {
|
||||
const album_token = try scanner.nextAlloc(allocator, .alloc_always);
|
||||
break :blk switch (album_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
.null => null,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
};
|
||||
@field(r, "album") = album;
|
||||
},
|
||||
.reason_end => {
|
||||
freeAllocated(allocator, key_token);
|
||||
const reason_end = try scanner.nextAlloc(allocator, .alloc_always);
|
||||
@field(r, "reason_end") = switch (reason_end) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
.null => null,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
},
|
||||
else => {
|
||||
freeAllocated(allocator, key_token);
|
||||
try scanner.skipValue();
|
||||
},
|
||||
}
|
||||
}
|
||||
// Exit object
|
||||
_ = try scanner.next();
|
||||
try out.append(r);
|
||||
}
|
||||
},
|
||||
// LastFM(stats)
|
||||
.object_begin => {
|
||||
_ = try scanner.next();
|
||||
find_array: while (true) {
|
||||
const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
|
||||
const field_name = switch (key_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
if (!std.mem.eql(u8, field_name, "scrobbles") and !std.mem.eql(u8, field_name, "recenttracks")) {
|
||||
freeAllocated(allocator, key_token);
|
||||
try scanner.skipValue();
|
||||
} else {
|
||||
freeAllocated(allocator, key_token);
|
||||
break :find_array;
|
||||
}
|
||||
}
|
||||
switch (try scanner.peekNextTokenType()) {
|
||||
// LastFM Stats
|
||||
.array_begin => continue :array .array_begin,
|
||||
// LastFM
|
||||
.object_begin => {
|
||||
// Enter recenttracks
|
||||
_ = try scanner.next();
|
||||
while (true) {
|
||||
const key_token = try scanner.nextAlloc(allocator, .alloc_if_needed);
|
||||
const field_name = switch (key_token) {
|
||||
inline .string, .allocated_string => |slice| slice,
|
||||
else => return error.UnexpectedToken,
|
||||
};
|
||||
if (!std.mem.eql(u8, field_name, "track")) {
|
||||
freeAllocated(allocator, key_token);
|
||||
try scanner.skipValue();
|
||||
} else {
|
||||
freeAllocated(allocator, key_token);
|
||||
continue :array .array_begin;
|
||||
}
|
||||
}
|
||||
},
|
||||
else => unreachable,
|
||||
}
|
||||
},
|
||||
else => return error.UnexpectedToken,
|
||||
}
|
||||
const scrobbles = try out.toOwnedSlice();
|
||||
return scrobbles;
|
||||
}
|
||||
|
||||
fn freeAllocated(allocator: std.mem.Allocator, token: std.json.Token) void {
|
||||
switch (token) {
|
||||
.allocated_number, .allocated_string => |slice| {
|
||||
allocator.free(slice);
|
||||
},
|
||||
else => {},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,17 @@
|
|||
const std = @import("std");
|
||||
|
||||
pub const UnifiedScrobble = struct {
|
||||
track: ?[]const u8,
|
||||
// These can be null per Spotify
|
||||
track_artist: ?[]const u8,
|
||||
album: ?[]const u8,
|
||||
album_artist: ?[]const u8,
|
||||
date: i64,
|
||||
// Relevant Spotify data
|
||||
playtime: ?u64 = null,
|
||||
reason_end: ?[]const u8 = null,
|
||||
};
|
||||
|
||||
pub const ImportedScrobbles = union(ScrobbleSources) {
|
||||
LastFMStats: []IgnorantScrobble,
|
||||
LastFMWeb: []LastFMWebScrobble,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue