mitteneer/zuletzt
1
0
Fork 0
Code Issues 14 Pull requests Projects Releases Packages Wiki Activity Actions

Allow uploads from LastFM API

Browse source 
Very slow at the moment. Look into ways to speed this up
This commit is contained in:
mitteneer 2025-05-15 20:23:12 -04:00
parent f69ffb2b37
commit 89e98c7a47
2 changed files with 205 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

199
src/app/views/upload.zig
View file

@ -4,6 +4,8 @@ const jetquery = @import("jetzig").jetquery;
const zeit = @import("zeit"); const zeit = @import("zeit");
const rules = @import("../../apply_rule.zig"); const rules = @import("../../apply_rule.zig");
const Data = @import("../../types.zig"); const Data = @import("../../types.zig");
const Utils = @import("../../date_fmt.zig");
const Client = std.http.Client;
pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View {
_ = data; _ = data;
@ -13,19 +15,7 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View {
pub fn post(request: *jetzig.Request) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View {
var root = try request.data(.object); var root = try request.data(.object);
if (try request.file("upload")) |file| {
const params = try request.params(); const params = try request.params();
const source = try std.fmt.parseInt(u8, params.get("t").?.string.value, 10); // This param is required in HTML
const before_limiter: bool = if (params.get("bbool")) |_| true else false;
const after_limiter: bool = if (params.get("abool")) |_| true else false;
var scrobbles_view = try root.put("scrobbles", .array);
var job = try request.job("process_scrobbles");
var scrobbles_data = try job.params.put("scrobbles", .array);
var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0;
const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) {
error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }),
else => err, else => err,
@ -34,7 +24,21 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
defer rule_file.close(); defer rule_file.close();
const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000);
const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null; const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null;
var job = try request.job("process_scrobbles");
const source = params.getT(.integer, "t").?; // This param is required in HTML
const before_limiter: bool = if (params.get("bbool")) |_| true else false;
const after_limiter: bool = if (params.get("abool")) |_| true else false;
var scrobbles_view = try root.put("scrobbles", .array);
var scrobbles_data = try job.params.put("scrobbles", .array);
var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0;
switch (source) {
0, 1 => {
if (try request.file("upload")) |file| {
std.log.debug("{s}", .{file.filename});
switch (source) { switch (source) {
0 => { 0 => {
const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{}); const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{});
@ -55,36 +59,30 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
.date = scrobble.date, .date = scrobble.date,
}; };
var scrobble_view = try scrobbles_view.append(.object); const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
var artists = try scrobble_view.put("artists", .array);
try scrobble_view.put("track", formatted_scrobble.track);
try scrobble_view.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_track) |artist| {
try artists.append(artist);
}
try scrobble_view.put("date", formatted_scrobble.date);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object); var scrobble_data = try scrobbles_data.append(.object);
var artists_album = try scrobble_data.put("artists_album", .array);
var artists_track = try scrobble_data.put("artists_track", .array);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("album", formatted_scrobble.album); try scrobble_data.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_album) |artist| { try scrobble_data.put("track", formatted_scrobble.track);
try artists_album.append(artist); try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
} }
for (formatted_scrobble.artists_track) |artist| { var aaa = try scrobble_data.put("artists_album", .array);
try artists_track.append(artist); for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
} }
try scrobble_data.put("date", formatted_scrobble.date);
} }
}, },
1 => { 1 => {
const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true }); const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true });
const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else (try zeit.instant(.{})).time(); const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time();
const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time(); const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time();
appends: for (content) |scrobble| { appends: for (content) |scrobble| {
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) { if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
skipped_tracks += 1; skipped_tracks += 1;
@ -105,7 +103,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
.track = scrobble.master_metadata_track_name.?, .track = scrobble.master_metadata_track_name.?,
.album = scrobble.master_metadata_album_album_name.?, .album = scrobble.master_metadata_album_album_name.?,
.artist = scrobble.master_metadata_album_artist_name.?, .artist = scrobble.master_metadata_album_artist_name.?,
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(),
}; };
const formatted_scrobble = if (rule_list) |rl| const formatted_scrobble = if (rule_list) |rl|
@ -119,30 +117,24 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
.date = pre_formatted_scrobble.date, .date = pre_formatted_scrobble.date,
}; };
var scrobble_view = try scrobbles_view.append(.object); const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
var artists = try scrobble_view.put("artists", .array);
try scrobble_view.put("track", formatted_scrobble.track);
try scrobble_view.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_track) |artist| {
try artists.append(artist);
}
try scrobble_view.put("date", formatted_scrobble.date);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object); var scrobble_data = try scrobbles_data.append(.object);
var artists_album = try scrobble_data.put("artists_album", .array);
var artists_track = try scrobble_data.put("artists_track", .array);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("album", formatted_scrobble.album); try scrobble_data.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_album) |artist| { try scrobble_data.put("track", formatted_scrobble.track);
try artists_album.append(artist); try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
} }
for (formatted_scrobble.artists_track) |artist| { var aaa = try scrobble_data.put("artists_album", .array);
try artists_track.append(artist); for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
} }
try scrobble_data.put("date", formatted_scrobble.date);
} }
}, },
else => unreachable, else => unreachable,
@ -151,12 +143,107 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
std.log.debug("Skipped {} tracks", .{skipped_tracks}); std.log.debug("Skipped {} tracks", .{skipped_tracks});
std.log.debug("Filtered {} tracks", .{limited_tracks}); std.log.debug("Filtered {} tracks", .{limited_tracks});
} }
},
2 => {
if (params.getT(.string, "username")) |username| {
_ = username;
const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&format=json";
const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator };
var ar = std.ArrayList(u8).init(request.allocator);
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const first_response = try ar.toOwnedSlice();
const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true });
var upload_table = try root.put("upload_table", .array); for (json.recenttracks.track) |scrobble| {
try upload_table.append("Track"); const pre_formatted_scrobble = Data.ImportedScrobble{
try upload_table.append("Artist"); .track = scrobble.name,
try upload_table.append("Album"); .album = if (scrobble.album) |album| album.@"#text".? else "",
try upload_table.append("Date"); .artist = scrobble.artist.@"#text".?,
.date = try std.fmt.parseInt(i64, scrobble.date.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1;
for (2..max_pages) |page| {
const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&page={}&format=json", .{page});
std.log.debug("{s}", .{rest_query});
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const response = try ar.toOwnedSlice();
const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true });
for (json2.recenttracks.track) |scrobble| {
const pre_formatted_scrobble = Data.ImportedScrobble{
.track = scrobble.name,
.album = if (scrobble.album) |album| album.@"#text".? else "",
.artist = scrobble.artist.@"#text".?,
.date = try std.fmt.parseInt(i64, scrobble.date.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
}
try job.schedule();
}
},
else => unreachable,
}
return request.render(.created); return request.render(.created);
} }

3
src/app/views/upload/index.zmpl
View file

@ -11,9 +11,12 @@
<label>File</label> <label>File</label>
<input type="file" name="upload"/> <input type="file" name="upload"/>
<input type="submit" value="Submit"/> <input type="submit" value="Submit"/>
<label>Username</label>
<input type="text" name="username"/>
<fieldset> <fieldset>
<input type="radio" name="t" label="Last.fm" value="0" required>Last.fm</input> <input type="radio" name="t" label="Last.fm" value="0" required>Last.fm</input>
<input type="radio" name="t" label="Spotify" value="1" required>Spotify</input> <input type="radio" name="t" label="Spotify" value="1" required>Spotify</input>
<input type="radio" name="t" label="Last.fm (Web Auth)" value="2" required>Last.fm (WebAuth)</input>
<input type="checkbox" name="bbool" id="bbool" value="false"></input>Limit to Scrobbles before: <input type="datetime-local" name="b" label="date-before"></input> <input type="checkbox" name="bbool" id="bbool" value="false"></input>Limit to Scrobbles before: <input type="datetime-local" name="b" label="date-before"></input>
<input type="checkbox" name="abool" id="abool" value="false"></input>Limit to Scrobbles after: <input type="datetime-local" name="a" label="date-after"></input> <input type="checkbox" name="abool" id="abool" value="false"></input>Limit to Scrobbles after: <input type="datetime-local" name="a" label="date-after"></input>
</fieldset> </fieldset>