mitteneer/zuletzt
1
0
Fork 0
Code Issues 14 Pull requests Projects Releases Packages Wiki Activity Actions

Refactor upload.zig

Browse source 
I have been unhappy with the branches, but didn't quite know what to do about it. THis feels much nicer. Also fixes datetime stuff with jetquery.  The HTML element parsing isn't quite where I want it to be, but it works for the time being.
This commit is contained in:
mitteneer 2025-05-20 15:07:51 -04:00
parent 6494bbdf60
commit a2a739bc9c
6 changed files with 184 additions and 258 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/app/jobs/process_scrobbles.zig
View file

@ -1,7 +1,6 @@
const std = @import("std");
const jetzig = @import("jetzig");
const jetquery = @import("jetzig").jetquery;
const lastfm = @import("../../types.zig").LastFM;
const Data = @import("../../types.zig");
const rules = @import("../../apply_rule.zig");

282
src/app/views/upload.zig
View file

@ -23,67 +23,81 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
defer rule_file.close();
const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000);
const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null;
const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null;
var job = try request.job("process_scrobbles");
const source = params.getT(.integer, "t").?; // This param is required in HTML
const before_limiter: bool = if (params.get("bbool")) |_| true else false;
const after_limiter: bool = if (params.get("abool")) |_| true else false;
const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: {
const date = params.getT(.string, "latest-date").?;
break :blk try zeit.Time.fromISO8601(date);
} else blk: {
break :blk (try zeit.instant(.{ .source = .now })).time();
};
const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: {
const date = params.getT(.string, "earliest-date").?;
break :blk try zeit.Time.fromISO8601(date);
} else blk: {
break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time();
};
var scrobbles_view = try root.put("scrobbles", .array);
var scrobbles_data = try job.params.put("scrobbles", .array);
const earliest_timestamp = earliest_date.instant().unixTimestamp();
const latest_timestamp = latest_date.instant().unixTimestamp();
var view_params = try root.put("scrobbles", .array);
var job_params = try job.params.put("scrobbles", .array);
var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0;
switch (source) {
0, 1 => {
if (try request.file("upload")) |file| {
std.log.debug("{s}", .{file.filename});
switch (source) {
0 => {
const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{});
const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0;
const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807;
appends: for (content.scrobbles) |scrobble| {
// We can short-circuit on the limiter bools
if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends;
const imported_scrobbles: Data.ImportedScrobbles = switch (source) {
0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles },
1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) },
2 => blk: {
const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator };
var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator);
var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator);
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, scrobble, rl)
else
Data.Scrobble{
const username = if (params.getT(.string, "username")) |un| un else "VAOTM";
var page: usize = 1;
var max_pages: ?usize = null;
while (true) : (page += 1) {
if (max_pages != null and page > max_pages.?) break;
const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page });
const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
std.log.debug("{}", .{r});
const response_string = try lastfm_response_buffer.toOwnedSlice();
const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true });
//const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10);
if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10);
try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track);
}
break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items };
},
else => unreachable,
};
// Not sure if I should be proud or feel sick
switch (imported_scrobbles) {
inline else => |scrobbles| {
appends: for (scrobbles) |scrobble| {
const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) {
Data.IgnorantScrobble => {
if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) {
limited_tracks += 1;
continue :appends;
}
break :blk Data.Scrobble{
.album = scrobble.album,
.artists_album = &[_][]const u8{scrobble.artist},
.track = scrobble.track,
.artists_track = &[_][]const u8{scrobble.artist},
.date = scrobble.date,
.date = scrobble.date * 1_000,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
},
1 => {
const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true });
const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time();
const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time();
appends: for (content) |scrobble| {
Data.SpotifyScrobble => {
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
skipped_tracks += 1;
continue :appends;
@ -94,158 +108,56 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
}
const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) {
if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) {
limited_tracks += 1;
continue :appends;
}
const pre_formatted_scrobble: Data.ImportedScrobble = .{
.track = scrobble.master_metadata_track_name.?,
break :blk Data.Scrobble{
.album = scrobble.master_metadata_album_album_name.?,
.artist = scrobble.master_metadata_album_artist_name.?,
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(),
.artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
.track = scrobble.master_metadata_track_name.?,
.artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000,
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
},
Data.LastFMWebScrobble => {
break :blk Data.Scrobble{
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artists_album = &[_][]const u8{scrobble.artist.@"#text"},
.track = scrobble.name,
.artists_track = &[_][]const u8{scrobble.artist.@"#text"},
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
},
else => unreachable,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
try view_params.append(row);
var job_param = try job_params.append(.object);
try job_param.put("album", complete_scrobble.album);
try job_param.put("track", complete_scrobble.track);
try job_param.put("date", complete_scrobble.date);
var track_artists_array = try job_param.put("artists_track", .array);
for (complete_scrobble.artists_track) |a| {
try track_artists_array.append(a);
}
var album_artists_array = try job_param.put("artists_album", .array);
for (complete_scrobble.artists_album) |a| {
try album_artists_array.append(a);
}
try job.schedule();
std.log.debug("Skipped {} tracks", .{skipped_tracks});
std.log.debug("Filtered {} tracks", .{limited_tracks});
}
},
2 => {
if (params.getT(.string, "username")) |username| {
_ = username;
const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json";
const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator };
var ar = std.ArrayList(u8).init(request.allocator);
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const first_response = try ar.toOwnedSlice();
const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true });
appends: for (json.recenttracks.track) |scrobble| {
if (scrobble.date == null) continue :appends;
const pre_formatted_scrobble = Data.ImportedScrobble{
.track = scrobble.name,
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artist = scrobble.artist.@"#text",
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1;
for (2..max_pages) |page| {
const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page});
std.log.debug("{s}", .{rest_query});
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const response = try ar.toOwnedSlice();
const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true });
appends: for (json2.recenttracks.track) |scrobble| {
if (scrobble.date == null) continue :appends;
const pre_formatted_scrobble = Data.ImportedScrobble{
.track = scrobble.name,
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artist = scrobble.artist.@"#text",
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
}
std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks });
try job.schedule();
}
},
else => unreachable,
}
return request.render(.created);
}

6
src/app/views/upload/index.zmpl
View file

@ -17,8 +17,10 @@
<input type="radio" name="t" label="Last.fm" value="0" required>Last.fm</input>
<input type="radio" name="t" label="Spotify" value="1" required>Spotify</input>
<input type="radio" name="t" label="Last.fm (Web Auth)" value="2" required>Last.fm (WebAuth)</input>
<input type="checkbox" name="bbool" id="bbool" value="false"></input>Limit to Scrobbles before: <input type="datetime-local" name="b" label="date-before"></input>
<input type="checkbox" name="abool" id="abool" value="false"></input>Limit to Scrobbles after: <input type="datetime-local" name="a" label="date-after"></input>
<input type="checkbox" name="adv-opt" label="Advanced Options">Advanced Options</input>
Limit to Scrobbles before: <input type="datetime-local" name="latest-date" label="date-before"></input>
Limit to Scrobbles after: <input type="datetime-local" name="earliest-date" label="date-after"></input>
</fieldset>
</form>
</body>

26
src/apply_rule.zig
View file

@ -1,5 +1,5 @@
const std = @import("std");
const Rules = @import("./types.zig").Rules;
const Rule = @import("./types.zig").Rule;
const Data = @import("./types.zig");
// Wrapper for containsAtLeast to make the switch below to work
@ -11,18 +11,10 @@ fn eqlWrapper(haystack: []const u8, needle: []const u8) bool {
return std.mem.eql(u8, haystack, needle);
}
pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) !Data.Scrobble {
const artists = try allocator.alloc([]const u8, 1);
artists[0] = scrobble.artist;
var output_scrobble = Data.Scrobble{
.track = scrobble.track,
.artists_track = artists,
.album = scrobble.album,
.artists_album = artists,
.date = scrobble.date,
};
pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble, rules: []Rule) !Data.Scrobble {
var output_scrobble = scrobble;
for (rules.rules) |rule| {
for (rules) |rule| {
var match_found: bool = switch (rule.cond_req) {
.any => false,
.all => true,
@ -34,10 +26,16 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedSc
};
switch (rule.cond_req) {
.any => switch (cond.match_on) {
inline else => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
inline .album, .track => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
inline .artists_album, .artists_track => |on| {
for (@field(scrobble, @tagName(on))) |artist| match_found = match_found or match_fn(artist, cond.match_txt);
},
},
.all => switch (cond.match_on) {
inline else => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
inline .album, .track => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
inline .artists_album, .artists_track => |on| {
for (@field(scrobble, @tagName(on))) |artist| match_found = match_found and match_fn(artist, cond.match_txt);
},
},
}
}

2
src/date_fmt.zig
View file

@ -4,7 +4,7 @@ const Data = @import("types.zig");
pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 {
var date = std.ArrayList(u8).init(allocator);
try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M");
try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M");
return date.items;
}

43
src/types.zig
View file

@ -1,7 +1,20 @@
pub const ImportedScrobble = struct {
pub const ImportedScrobbles = union(ScrobbleSources) {
LastFMStats: []IgnorantScrobble,
LastFMWeb: []LastFMWebScrobble,
Spotify: []SpotifyScrobble,
};
const ScrobbleSources = enum {
LastFMStats,
LastFMWeb,
Spotify,
};
pub const IgnorantScrobble = struct {
track: []const u8,
artist: []const u8,
album: []const u8 = "",
album: []const u8 = "Not Provided",
//albumId: []const u8,
date: i64,
};
@ -14,7 +27,7 @@ pub const Scrobble = struct {
};
// From lastfmstats.com
pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble };
pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble };
// I derived whether or not these values were optional from searching
// the respective fields for null in Vim, so there may be some fields
@ -45,7 +58,17 @@ pub const SpotifyScrobble = struct {
pub const LastFMWeb = struct {
recenttracks: struct {
track: []struct {
track: []LastFMWebScrobble,
@"@attr": LastFMWebQueryInfo,
},
};
pub const LastFMWebHyperlinkData = struct {
mbid: []const u8,
@"#text": []const u8,
};
pub const LastFMWebScrobble = struct {
artist: LastFMWebHyperlinkData,
album: ?LastFMWebHyperlinkData = null,
name: []const u8,
@ -62,12 +85,9 @@ pub const LastFMWeb = struct {
nowplaying: []const u8,
} = null,
url: []const u8,
},
@"@attr": LastFMWebAttr,
},
};
pub const LastFMWebAttr = struct {
pub const LastFMWebQueryInfo = struct {
perPage: []const u8,
totalPages: []const u8,
page: []const u8,
@ -75,16 +95,11 @@ pub const LastFMWebAttr = struct {
total: []const u8,
};
pub const LastFMWebHyperlinkData = struct {
mbid: []const u8,
@"#text": []const u8,
};
pub const Rule = struct {
name: []const u8,
cond_req: enum { any, all },
conditionals: []struct {
match_on: enum { artist, album, track },
match_on: enum { artists_album, artists_track, album, track },
match_cond: enum { is, contains },
match_txt: []const u8,
},