From 89e98c7a47f907c507df88cd74d429090b1de26a Mon Sep 17 00:00:00 2001 From: mitteneer Date: Thu, 15 May 2025 20:23:12 -0400 Subject: [PATCH] Allow uploads from LastFM API Very slow at the moment. Look into ways to speed this up --- src/app/views/upload.zig | 313 ++++++++++++++++++++------------ src/app/views/upload/index.zmpl | 7 +- 2 files changed, 205 insertions(+), 115 deletions(-) diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 25a9128..220e2c9 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -4,6 +4,8 @@ const jetquery = @import("jetzig").jetquery; const zeit = @import("zeit"); const rules = @import("../../apply_rule.zig"); const Data = @import("../../types.zig"); +const Utils = @import("../../date_fmt.zig"); +const Client = std.http.Client; pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { _ = data; @@ -13,99 +15,152 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - if (try request.file("upload")) |file| { - const params = try request.params(); - const source = try std.fmt.parseInt(u8, params.get("t").?.string.value, 10); // This param is required in HTML - const before_limiter: bool = if (params.get("bbool")) |_| true else false; - const after_limiter: bool = if (params.get("abool")) |_| true else false; + const params = try request.params(); + const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), + else => err, + }); - var scrobbles_view = try root.put("scrobbles", .array); - var job = try request.job("process_scrobbles"); - var scrobbles_data = try job.params.put("scrobbles", .array); + defer rule_file.close(); + const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); + const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + var job = try request.job("process_scrobbles"); + const source = params.getT(.integer, "t").?; // This param is required in HTML + const before_limiter: bool = if (params.get("bbool")) |_| true else false; + const after_limiter: bool = if (params.get("abool")) |_| true else false; - var skipped_tracks: u64 = 0; - var limited_tracks: u64 = 0; + var scrobbles_view = try root.put("scrobbles", .array); + var scrobbles_data = try job.params.put("scrobbles", .array); - const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), - else => err, - }); + var skipped_tracks: u64 = 0; + var limited_tracks: u64 = 0; - defer rule_file.close(); - const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); - const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; + switch (source) { + 0, 1 => { + if (try request.file("upload")) |file| { + std.log.debug("{s}", .{file.filename}); + switch (source) { + 0 => { + const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); + const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; + const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; + appends: for (content.scrobbles) |scrobble| { + // We can short-circuit on the limiter bools + if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; - switch (source) { - 0 => { - const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); - const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0; - const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807; - appends: for (content.scrobbles) |scrobble| { - // We can short-circuit on the limiter bools - if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends; + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, scrobble, rl) + else + Data.Scrobble{ + .album = scrobble.album, + .artists_album = &[_][]const u8{scrobble.artist}, + .track = scrobble.track, + .artists_track = &[_][]const u8{scrobble.artist}, + .date = scrobble.date, + }; - const formatted_scrobble = if (rule_list) |rl| - try rules.applyScrobbleRule(request.allocator, scrobble, rl) - else - Data.Scrobble{ - .album = scrobble.album, - .artists_album = &[_][]const u8{scrobble.artist}, - .track = scrobble.track, - .artists_track = &[_][]const u8{scrobble.artist}, - .date = scrobble.date, - }; + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); - var scrobble_view = try scrobbles_view.append(.object); - var artists = try scrobble_view.put("artists", .array); + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); - try scrobble_view.put("track", formatted_scrobble.track); - try scrobble_view.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_track) |artist| { - try artists.append(artist); - } - try scrobble_view.put("date", formatted_scrobble.date); + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } - var scrobble_data = try scrobbles_data.append(.object); - var artists_album = try scrobble_data.put("artists_album", .array); - var artists_track = try scrobble_data.put("artists_track", .array); + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + }, + 1 => { + const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); + const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time(); + const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); + appends: for (content) |scrobble| { + if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { + skipped_tracks += 1; + continue :appends; + } + if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { + skipped_tracks += 1; + continue :appends; + } - try scrobble_data.put("track", formatted_scrobble.track); - try scrobble_data.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_album) |artist| { - try artists_album.append(artist); - } + const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); + if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { + limited_tracks += 1; + continue :appends; + } - for (formatted_scrobble.artists_track) |artist| { - try artists_track.append(artist); - } - try scrobble_data.put("date", formatted_scrobble.date); + const pre_formatted_scrobble: Data.ImportedScrobble = .{ + .track = scrobble.master_metadata_track_name.?, + .album = scrobble.master_metadata_album_album_name.?, + .artist = scrobble.master_metadata_album_artist_name.?, + .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(), + }; + + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; + + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + }, + else => unreachable, } - }, - 1 => { - const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); - const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else (try zeit.instant(.{})).time(); - const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); - appends: for (content) |scrobble| { - if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { - skipped_tracks += 1; - continue :appends; - } - if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) { - skipped_tracks += 1; - continue :appends; - } + try job.schedule(); + std.log.debug("Skipped {} tracks", .{skipped_tracks}); + std.log.debug("Filtered {} tracks", .{limited_tracks}); + } + }, + 2 => { + if (params.getT(.string, "username")) |username| { + _ = username; + const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&format=json"; + const user_agent: []const u8 = "Zuletzt/0.0.1"; + var client = Client{ .allocator = request.allocator }; + var ar = std.ArrayList(u8).init(request.allocator); + _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + const first_response = try ar.toOwnedSlice(); + const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true }); - const iso_ts = try zeit.Time.fromISO8601(scrobble.ts); - if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) { - limited_tracks += 1; - continue :appends; - } - - const pre_formatted_scrobble: Data.ImportedScrobble = .{ - .track = scrobble.master_metadata_track_name.?, - .album = scrobble.master_metadata_album_album_name.?, - .artist = scrobble.master_metadata_album_artist_name.?, - .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000, + for (json.recenttracks.track) |scrobble| { + const pre_formatted_scrobble = Data.ImportedScrobble{ + .track = scrobble.name, + .album = if (scrobble.album) |album| album.@"#text".? else "", + .artist = scrobble.artist.@"#text".?, + .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), }; const formatted_scrobble = if (rule_list) |rl| @@ -119,44 +174,76 @@ pub fn post(request: *jetzig.Request) !jetzig.View { .date = pre_formatted_scrobble.date, }; - var scrobble_view = try scrobbles_view.append(.object); - var artists = try scrobble_view.put("artists", .array); - - try scrobble_view.put("track", formatted_scrobble.track); - try scrobble_view.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_track) |artist| { - try artists.append(artist); - } - try scrobble_view.put("date", formatted_scrobble.date); + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); var scrobble_data = try scrobbles_data.append(.object); - var artists_album = try scrobble_data.put("artists_album", .array); - var artists_track = try scrobble_data.put("artists_track", .array); - - try scrobble_data.put("track", formatted_scrobble.track); try scrobble_data.put("album", formatted_scrobble.album); - for (formatted_scrobble.artists_album) |artist| { - try artists_album.append(artist); - } - - for (formatted_scrobble.artists_track) |artist| { - try artists_track.append(artist); - } + try scrobble_data.put("track", formatted_scrobble.track); try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } } - }, - else => unreachable, - } - try job.schedule(); - std.log.debug("Skipped {} tracks", .{skipped_tracks}); - std.log.debug("Filtered {} tracks", .{limited_tracks}); + const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1; + for (2..max_pages) |page| { + const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&page={}&format=json", .{page}); + std.log.debug("{s}", .{rest_query}); + _ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); + const response = try ar.toOwnedSlice(); + const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true }); + + for (json2.recenttracks.track) |scrobble| { + const pre_formatted_scrobble = Data.ImportedScrobble{ + .track = scrobble.name, + .album = if (scrobble.album) |album| album.@"#text".? else "", + .artist = scrobble.artist.@"#text".?, + .date = try std.fmt.parseInt(i64, scrobble.date.uts, 10), + }; + + const formatted_scrobble = if (rule_list) |rl| + try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl) + else + Data.Scrobble{ + .album = pre_formatted_scrobble.album, + .artists_album = &[_][]const u8{pre_formatted_scrobble.artist}, + .track = pre_formatted_scrobble.track, + .artists_track = &[_][]const u8{pre_formatted_scrobble.artist}, + .date = pre_formatted_scrobble.date, + }; + + const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble); + + try scrobbles_view.append(row); + //try scrobbles_data.append(formatted_scrobble); + var scrobble_data = try scrobbles_data.append(.object); + try scrobble_data.put("album", formatted_scrobble.album); + try scrobble_data.put("track", formatted_scrobble.track); + try scrobble_data.put("date", formatted_scrobble.date); + + var taa = try scrobble_data.put("artists_track", .array); + for (formatted_scrobble.artists_track) |a| { + try taa.append(a); + } + + var aaa = try scrobble_data.put("artists_album", .array); + for (formatted_scrobble.artists_album) |a| { + try aaa.append(a); + } + } + } + try job.schedule(); + } + }, + else => unreachable, } - - var upload_table = try root.put("upload_table", .array); - try upload_table.append("Track"); - try upload_table.append("Artist"); - try upload_table.append("Album"); - try upload_table.append("Date"); - return request.render(.created); } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 9043a5b..31151e1 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -9,11 +9,14 @@
- - + + + +
Last.fm Spotify + Last.fm (WebAuth) Limit to Scrobbles before: Limit to Scrobbles after: