diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index c6b25b1..dcf7f1f 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -1,7 +1,6 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -const lastfm = @import("../../types.zig").LastFM; const Data = @import("../../types.zig"); const rules = @import("../../apply_rule.zig"); diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 2582ff4..c86e6f0 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -23,229 +23,141 @@ pub fn post(request: *jetzig.Request) !jetzig.View { defer rule_file.close(); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; var job = try request.job("process_scrobbles"); const source = params.getT(.integer, "t").?; // This param is required in HTML - const before_limiter: bool = if (params.get("bbool")) |_| true else false; - const after_limiter: bool = if (params.get("abool")) |_| true else false; + const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { + const date = params.getT(.string, "latest-date").?; + break :blk try zeit.Time.fromISO8601(date); + } else blk: { + break :blk (try zeit.instant(.{ .source = .now })).time(); + }; + const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { + const date = params.getT(.string, "earliest-date").?; + break :blk try zeit.Time.fromISO8601(date); + } else blk: { + break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); + }; - var scrobbles_view = try root.put("scrobbles", .array); - var scrobbles_data = try job.params.put("scrobbles", .array); + const earliest_timestamp = earliest_date.instant().unixTimestamp(); + const latest_timestamp = latest_date.instant().unixTimestamp(); + + var view_params = try root.put("scrobbles", .array); + var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - switch (source) { - 0, 1 => { - if (try request.file("upload")) |file| { - std.log.debug("{s}", .{file.filename}); - switch (source) { - 0 => { - const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); - const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; - const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; - appends: for (content.scrobbles) |scrobble| { - // We can short-circuit on the limiter bools - if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; + const imported_scrobbles: Data.ImportedScrobbles = switch (source) { + 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles }, + 1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) }, + 2 => blk: { + const user_agent: []const u8 = "Zuletzt/0.0.1"; + var client = Client{ .allocator = request.allocator }; + var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); + var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator); - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, scrobble, rl) - else - Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date, - }; + const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + var page: usize = 1; + var max_pages: ?usize = null; - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - }, - 1 => { - const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); - const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time(); - const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); - appends: for (content) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } - - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { - limited_tracks += 1; - continue :appends; - } - - const pre_formatted_scrobble: Data.ImportedScrobble = .{ - .track = scrobble.master_metadata_track_name.?, - .album = scrobble.master_metadata_album_album_name.?, - .artist = scrobble.master_metadata_album_artist_name.?, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - }, - else => unreachable, - } - try job.schedule(); - std.log.debug("Skipped {} tracks", .{skipped_tracks}); - std.log.debug("Filtered {} tracks", .{limited_tracks}); + while (true) : (page += 1) { + if (max_pages != null and page > max_pages.?) break; + const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); + const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + std.log.debug("{}", .{r}); + const response_string = try lastfm_response_buffer.toOwnedSlice(); + const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true }); + //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10); + if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10); + try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track); } - }, - 2 => { - if (params.getT(.string, "username")) |username| { - _ = username; - const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json"; - const user_agent: []const u8 = "Zuletzt/0.0.1"; - var client = Client{ .allocator = request.allocator }; - var ar = std.ArrayList(u8).init(request.allocator); - _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - const first_response = try ar.toOwnedSlice(); - const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - appends: for (json.recenttracks.track) |scrobble| { - if (scrobble.date == null) continue :appends; - const pre_formatted_scrobble = Data.ImportedScrobble{ - .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artist = scrobble.artist.@"#text", - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; - for (2..max_pages) |page| { - const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page}); - std.log.debug("{s}", .{rest_query}); - _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); - const response = try ar.toOwnedSlice(); - const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); - - appends: for (json2.recenttracks.track) |scrobble| { - if (scrobble.date == null) continue :appends; - const pre_formatted_scrobble = Data.ImportedScrobble{ - .track = scrobble.name, - .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", - .artist = scrobble.artist.@"#text", - .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10), - }; - - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) - else - Data.Scrobble{ - .album = pre_formatted_scrobble.album, - .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, - .track = pre_formatted_scrobble.track, - .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, - .date = pre_formatted_scrobble.date, - }; - - const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - - try scrobbles_view.append(row); - //try scrobbles_data.append(formatted_scrobble); - var scrobble_data = try scrobbles_data.append(.object); - try scrobble_data.put("album", formatted_scrobble.album); - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("date", formatted_scrobble.date); - - var taa = try scrobble_data.put("artists_track", .array); - for (formatted_scrobble.artists_track) |a| { - try taa.append(a); - } - - var aaa = try scrobble_data.put("artists_album", .array); - for (formatted_scrobble.artists_album) |a| { - try aaa.append(a); - } - } - } - try job.schedule(); - } + break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items }; }, else => unreachable, + }; + + // Not sure if I should be proud or feel sick + switch (imported_scrobbles) { + inline else => |scrobbles| { + appends: for (scrobbles) |scrobble| { + const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) { + Data.IgnorantScrobble => { + if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) { + limited_tracks += 1; + continue :appends; + } + break :blk Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date * 1_000, + }; + }, + Data.SpotifyScrobble => { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } + + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) { + limited_tracks += 1; + continue :appends; + } + + break :blk Data.Scrobble{ + .album = scrobble.master_metadata_album_album_name.?, + .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .track = scrobble.master_metadata_track_name.?, + .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?}, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000, + }; + }, + Data.LastFMWebScrobble => { + break :blk Data.Scrobble{ + .album = if (scrobble.album) |album| album.@"#text" else "Not Provided", + .artists_album = &[_][]const u8{scrobble.artist.@"#text"}, + .track = scrobble.name, + .artists_track = &[_][]const u8{scrobble.artist.@"#text"}, + .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000, + }; + }, + else => unreachable, + }; + + const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble; + + const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); + + try view_params.append(row); + var job_param = try job_params.append(.object); + try job_param.put("album", complete_scrobble.album); + try job_param.put("track", complete_scrobble.track); + try job_param.put("date", complete_scrobble.date); + + var track_artists_array = try job_param.put("artists_track", .array); + for (complete_scrobble.artists_track) |a| { + try track_artists_array.append(a); + } + + var album_artists_array = try job_param.put("artists_album", .array); + for (complete_scrobble.artists_album) |a| { + try album_artists_array.append(a); + } + } + }, } + + std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); + try job.schedule(); + return request.render(.created); } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 31151e1..420e08c 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -17,8 +17,10 @@ Last.fm Spotify Last.fm (WebAuth) - Limit to Scrobbles before: - Limit to Scrobbles after: + + Advanced Options + Limit to Scrobbles before: + Limit to Scrobbles after: diff --git a/src/apply_rule.zig b/src/apply_rule.zig index 6163e93..fc4c0bc 100644 --- a/src/apply_rule.zig +++ b/src/apply_rule.zig @@ -1,5 +1,5 @@ const std = @import("std"); -const Rules = @import("./types.zig").Rules; +const Rule = @import("./types.zig").Rule; const Data = @import("./types.zig"); // Wrapper for containsAtLeast to make the switch below to work @@ -11,18 +11,10 @@ fn eqlWrapper(haystack: []const u8, needle: []const u8) bool { return std.mem.eql(u8, haystack, needle); } -pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) !Data.Scrobble { - const artists = try allocator.alloc([]const u8, 1); - artists[0] = scrobble.artist; - var output_scrobble = Data.Scrobble{ - .track = scrobble.track, - .artists_track = artists, - .album = scrobble.album, - .artists_album = artists, - .date = scrobble.date, - }; +pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble, rules: []Rule) !Data.Scrobble { + var output_scrobble = scrobble; - for (rules.rules) |rule| { + for (rules) |rule| { var match_found: bool = switch (rule.cond_req) { .any => false, .all => true, @@ -34,10 +26,16 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc }; switch (rule.cond_req) { .any => switch (cond.match_on) { - inline else => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .album, .track => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .artists_album, .artists_track => |on| { + for (@field(scrobble, @tagName(on))) |artist| match_found = match_found or match_fn(artist, cond.match_txt); + }, }, .all => switch (cond.match_on) { - inline else => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .album, .track => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt), + inline .artists_album, .artists_track => |on| { + for (@field(scrobble, @tagName(on))) |artist| match_found = match_found and match_fn(artist, cond.match_txt); + }, }, } } diff --git a/src/date_fmt.zig b/src/date_fmt.zig index 00fe93b..7e44da7 100644 --- a/src/date_fmt.zig +++ b/src/date_fmt.zig @@ -4,7 +4,7 @@ const Data = @import("types.zig"); pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 { var date = std.ArrayList(u8).init(allocator); - try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); + try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M"); return date.items; } diff --git a/src/types.zig b/src/types.zig index 073b2d3..3b2e4ff 100644 --- a/src/types.zig +++ b/src/types.zig @@ -1,7 +1,20 @@ -pub const ImportedScrobble = struct { +pub const ImportedScrobbles = union(ScrobbleSources) { + LastFMStats: []IgnorantScrobble, + LastFMWeb: []LastFMWebScrobble, + Spotify: []SpotifyScrobble, +}; + +const ScrobbleSources = enum { + LastFMStats, + LastFMWeb, + Spotify, +}; + +pub const IgnorantScrobble = struct { track: []const u8, artist: []const u8, - album: []const u8 = "", + album: []const u8 = "Not Provided", + //albumId: []const u8, date: i64, }; @@ -14,7 +27,7 @@ pub const Scrobble = struct { }; // From lastfmstats.com -pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble }; +pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble }; // I derived whether or not these values were optional from searching // the respective fields for null in Vim, so there may be some fields @@ -45,46 +58,48 @@ pub const SpotifyScrobble = struct { pub const LastFMWeb = struct { recenttracks: struct { - track: []struct { - artist: LastFMWebHyperlinkData, - album: ?LastFMWebHyperlinkData = null, - name: []const u8, - mbid: ?[]const u8 = null, - image: []struct { - size: []const u8, - @"#text": []const u8, - }, - date: ?struct { - uts: []const u8, - @"#text": []const u8, - } = null, - @"@attr": ?struct { - nowplaying: []const u8, - } = null, - url: []const u8, - }, - @"@attr": LastFMWebAttr, + track: []LastFMWebScrobble, + @"@attr": LastFMWebQueryInfo, }, }; -pub const LastFMWebAttr = struct { - perPage: []const u8, - totalPages: []const u8, - page: []const u8, - user: []const u8, - total: []const u8, -}; - pub const LastFMWebHyperlinkData = struct { mbid: []const u8, @"#text": []const u8, }; +pub const LastFMWebScrobble = struct { + artist: LastFMWebHyperlinkData, + album: ?LastFMWebHyperlinkData = null, + name: []const u8, + mbid: ?[]const u8 = null, + image: []struct { + size: []const u8, + @"#text": []const u8, + }, + date: ?struct { + uts: []const u8, + @"#text": []const u8, + } = null, + @"@attr": ?struct { + nowplaying: []const u8, + } = null, + url: []const u8, +}; + +pub const LastFMWebQueryInfo = struct { + perPage: []const u8, + totalPages: []const u8, + page: []const u8, + user: []const u8, + total: []const u8, +}; + pub const Rule = struct { name: []const u8, cond_req: enum { any, all }, conditionals: []struct { - match_on: enum { artist, album, track }, + match_on: enum { artists_album, artists_track, album, track }, match_cond: enum { is, contains }, match_txt: []const u8, },