diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig
index c6b25b1..dcf7f1f 100644
--- a/src/app/jobs/process_scrobbles.zig
+++ b/src/app/jobs/process_scrobbles.zig
@@ -1,7 +1,6 @@
const std = @import("std");
const jetzig = @import("jetzig");
const jetquery = @import("jetzig").jetquery;
-const lastfm = @import("../../types.zig").LastFM;
const Data = @import("../../types.zig");
const rules = @import("../../apply_rule.zig");
diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig
index 2582ff4..c86e6f0 100644
--- a/src/app/views/upload.zig
+++ b/src/app/views/upload.zig
@@ -23,229 +23,141 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
defer rule_file.close();
const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000);
- const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null;
+ const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null;
var job = try request.job("process_scrobbles");
const source = params.getT(.integer, "t").?; // This param is required in HTML
- const before_limiter: bool = if (params.get("bbool")) |_| true else false;
- const after_limiter: bool = if (params.get("abool")) |_| true else false;
+ const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: {
+ const date = params.getT(.string, "latest-date").?;
+ break :blk try zeit.Time.fromISO8601(date);
+ } else blk: {
+ break :blk (try zeit.instant(.{ .source = .now })).time();
+ };
+ const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: {
+ const date = params.getT(.string, "earliest-date").?;
+ break :blk try zeit.Time.fromISO8601(date);
+ } else blk: {
+ break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time();
+ };
- var scrobbles_view = try root.put("scrobbles", .array);
- var scrobbles_data = try job.params.put("scrobbles", .array);
+ const earliest_timestamp = earliest_date.instant().unixTimestamp();
+ const latest_timestamp = latest_date.instant().unixTimestamp();
+
+ var view_params = try root.put("scrobbles", .array);
+ var job_params = try job.params.put("scrobbles", .array);
var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0;
- switch (source) {
- 0, 1 => {
- if (try request.file("upload")) |file| {
- std.log.debug("{s}", .{file.filename});
- switch (source) {
- 0 => {
- const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{});
- const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0;
- const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807;
- appends: for (content.scrobbles) |scrobble| {
- // We can short-circuit on the limiter bools
- if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends;
+ const imported_scrobbles: Data.ImportedScrobbles = switch (source) {
+ 0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles },
+ 1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) },
+ 2 => blk: {
+ const user_agent: []const u8 = "Zuletzt/0.0.1";
+ var client = Client{ .allocator = request.allocator };
+ var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator);
+ var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator);
- const formatted_scrobble = if (rule_list) |rl|
- try rules.applyScrobbleRule(request.allocator, scrobble, rl)
- else
- Data.Scrobble{
- .album = scrobble.album,
- .artists_album = &[_][]const u8{scrobble.artist},
- .track = scrobble.track,
- .artists_track = &[_][]const u8{scrobble.artist},
- .date = scrobble.date,
- };
+ const username = if (params.getT(.string, "username")) |un| un else "VAOTM";
- const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
+ var page: usize = 1;
+ var max_pages: ?usize = null;
- try scrobbles_view.append(row);
- //try scrobbles_data.append(formatted_scrobble);
- var scrobble_data = try scrobbles_data.append(.object);
- try scrobble_data.put("album", formatted_scrobble.album);
- try scrobble_data.put("track", formatted_scrobble.track);
- try scrobble_data.put("date", formatted_scrobble.date);
-
- var taa = try scrobble_data.put("artists_track", .array);
- for (formatted_scrobble.artists_track) |a| {
- try taa.append(a);
- }
-
- var aaa = try scrobble_data.put("artists_album", .array);
- for (formatted_scrobble.artists_album) |a| {
- try aaa.append(a);
- }
- }
- },
- 1 => {
- const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true });
- const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time();
- const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time();
- appends: for (content) |scrobble| {
- if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
- skipped_tracks += 1;
- continue :appends;
- }
- if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
- skipped_tracks += 1;
- continue :appends;
- }
-
- const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
- if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) {
- limited_tracks += 1;
- continue :appends;
- }
-
- const pre_formatted_scrobble: Data.ImportedScrobble = .{
- .track = scrobble.master_metadata_track_name.?,
- .album = scrobble.master_metadata_album_album_name.?,
- .artist = scrobble.master_metadata_album_artist_name.?,
- .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(),
- };
-
- const formatted_scrobble = if (rule_list) |rl|
- try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
- else
- Data.Scrobble{
- .album = pre_formatted_scrobble.album,
- .artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
- .track = pre_formatted_scrobble.track,
- .artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
- .date = pre_formatted_scrobble.date,
- };
-
- const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
-
- try scrobbles_view.append(row);
- //try scrobbles_data.append(formatted_scrobble);
- var scrobble_data = try scrobbles_data.append(.object);
- try scrobble_data.put("album", formatted_scrobble.album);
- try scrobble_data.put("track", formatted_scrobble.track);
- try scrobble_data.put("date", formatted_scrobble.date);
-
- var taa = try scrobble_data.put("artists_track", .array);
- for (formatted_scrobble.artists_track) |a| {
- try taa.append(a);
- }
-
- var aaa = try scrobble_data.put("artists_album", .array);
- for (formatted_scrobble.artists_album) |a| {
- try aaa.append(a);
- }
- }
- },
- else => unreachable,
- }
- try job.schedule();
- std.log.debug("Skipped {} tracks", .{skipped_tracks});
- std.log.debug("Filtered {} tracks", .{limited_tracks});
+ while (true) : (page += 1) {
+ if (max_pages != null and page > max_pages.?) break;
+ const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page });
+ const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
+ std.log.debug("{}", .{r});
+ const response_string = try lastfm_response_buffer.toOwnedSlice();
+ const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true });
+ //const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10);
+ if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10);
+ try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track);
}
- },
- 2 => {
- if (params.getT(.string, "username")) |username| {
- _ = username;
- const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json";
- const user_agent: []const u8 = "Zuletzt/0.0.1";
- var client = Client{ .allocator = request.allocator };
- var ar = std.ArrayList(u8).init(request.allocator);
- _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
- const first_response = try ar.toOwnedSlice();
- const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true });
- appends: for (json.recenttracks.track) |scrobble| {
- if (scrobble.date == null) continue :appends;
- const pre_formatted_scrobble = Data.ImportedScrobble{
- .track = scrobble.name,
- .album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
- .artist = scrobble.artist.@"#text",
- .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10),
- };
-
- const formatted_scrobble = if (rule_list) |rl|
- try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
- else
- Data.Scrobble{
- .album = pre_formatted_scrobble.album,
- .artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
- .track = pre_formatted_scrobble.track,
- .artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
- .date = pre_formatted_scrobble.date,
- };
-
- const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
-
- try scrobbles_view.append(row);
- //try scrobbles_data.append(formatted_scrobble);
- var scrobble_data = try scrobbles_data.append(.object);
- try scrobble_data.put("album", formatted_scrobble.album);
- try scrobble_data.put("track", formatted_scrobble.track);
- try scrobble_data.put("date", formatted_scrobble.date);
-
- var taa = try scrobble_data.put("artists_track", .array);
- for (formatted_scrobble.artists_track) |a| {
- try taa.append(a);
- }
-
- var aaa = try scrobble_data.put("artists_album", .array);
- for (formatted_scrobble.artists_album) |a| {
- try aaa.append(a);
- }
- }
- const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1;
- for (2..max_pages) |page| {
- const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page});
- std.log.debug("{s}", .{rest_query});
- _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
- const response = try ar.toOwnedSlice();
- const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true });
-
- appends: for (json2.recenttracks.track) |scrobble| {
- if (scrobble.date == null) continue :appends;
- const pre_formatted_scrobble = Data.ImportedScrobble{
- .track = scrobble.name,
- .album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
- .artist = scrobble.artist.@"#text",
- .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10),
- };
-
- const formatted_scrobble = if (rule_list) |rl|
- try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
- else
- Data.Scrobble{
- .album = pre_formatted_scrobble.album,
- .artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
- .track = pre_formatted_scrobble.track,
- .artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
- .date = pre_formatted_scrobble.date,
- };
-
- const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
-
- try scrobbles_view.append(row);
- //try scrobbles_data.append(formatted_scrobble);
- var scrobble_data = try scrobbles_data.append(.object);
- try scrobble_data.put("album", formatted_scrobble.album);
- try scrobble_data.put("track", formatted_scrobble.track);
- try scrobble_data.put("date", formatted_scrobble.date);
-
- var taa = try scrobble_data.put("artists_track", .array);
- for (formatted_scrobble.artists_track) |a| {
- try taa.append(a);
- }
-
- var aaa = try scrobble_data.put("artists_album", .array);
- for (formatted_scrobble.artists_album) |a| {
- try aaa.append(a);
- }
- }
- }
- try job.schedule();
- }
+ break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items };
},
else => unreachable,
+ };
+
+ // Not sure if I should be proud or feel sick
+ switch (imported_scrobbles) {
+ inline else => |scrobbles| {
+ appends: for (scrobbles) |scrobble| {
+ const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) {
+ Data.IgnorantScrobble => {
+ if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) {
+ limited_tracks += 1;
+ continue :appends;
+ }
+ break :blk Data.Scrobble{
+ .album = scrobble.album,
+ .artists_album = &[_][]const u8{scrobble.artist},
+ .track = scrobble.track,
+ .artists_track = &[_][]const u8{scrobble.artist},
+ .date = scrobble.date * 1_000,
+ };
+ },
+ Data.SpotifyScrobble => {
+ if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
+ skipped_tracks += 1;
+ continue :appends;
+ }
+ if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
+ skipped_tracks += 1;
+ continue :appends;
+ }
+
+ const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
+ if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) {
+ limited_tracks += 1;
+ continue :appends;
+ }
+
+ break :blk Data.Scrobble{
+ .album = scrobble.master_metadata_album_album_name.?,
+ .artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
+ .track = scrobble.master_metadata_track_name.?,
+ .artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
+ .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000,
+ };
+ },
+ Data.LastFMWebScrobble => {
+ break :blk Data.Scrobble{
+ .album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
+ .artists_album = &[_][]const u8{scrobble.artist.@"#text"},
+ .track = scrobble.name,
+ .artists_track = &[_][]const u8{scrobble.artist.@"#text"},
+ .date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000,
+ };
+ },
+ else => unreachable,
+ };
+
+ const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
+
+ const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
+
+ try view_params.append(row);
+ var job_param = try job_params.append(.object);
+ try job_param.put("album", complete_scrobble.album);
+ try job_param.put("track", complete_scrobble.track);
+ try job_param.put("date", complete_scrobble.date);
+
+ var track_artists_array = try job_param.put("artists_track", .array);
+ for (complete_scrobble.artists_track) |a| {
+ try track_artists_array.append(a);
+ }
+
+ var album_artists_array = try job_param.put("artists_album", .array);
+ for (complete_scrobble.artists_album) |a| {
+ try album_artists_array.append(a);
+ }
+ }
+ },
}
+
+ std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks });
+ try job.schedule();
+
return request.render(.created);
}
diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl
index 31151e1..420e08c 100644
--- a/src/app/views/upload/index.zmpl
+++ b/src/app/views/upload/index.zmpl
@@ -17,8 +17,10 @@
Last.fm
Spotify
Last.fm (WebAuth)
- Limit to Scrobbles before:
- Limit to Scrobbles after:
+
+ Advanced Options
+ Limit to Scrobbles before:
+ Limit to Scrobbles after: