Compare commits

..

No commits in common. "d6a638bf2703cd4c5c15441c9fc4a2f8b3621028" and "4c759433d25f2ec6e961eafd505c8d163e51a29d" have entirely different histories.

7 changed files with 259 additions and 180 deletions

View file

@ -16,7 +16,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig
return;
},
};
const out_rules = &[_]Data.Rule{rule};
const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} };
const out = try std.json.stringifyAlloc(allocator, out_rules, .{});
try file.writeAll(out);
file.close();
@ -35,17 +35,18 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig
const file_write: std.fs.File = try std.fs.cwd().openFile("rules.json", .{ .mode = .write_only });
if (file_content.len == 0) {
const out_rules = &[_]Data.Rule{rule};
const out_rules = Data.Rules{ .rules = &[_]Data.Rule{rule} };
const out = try std.json.stringifyAlloc(allocator, out_rules, .{});
try file_write.writeAll(out);
file_write.close();
return;
}
const content: []Data.Rule = try std.json.parseFromSliceLeaky([]Data.Rule, allocator, file_content, .{});
try rules.appendSlice(content);
const content: Data.Rules = try std.json.parseFromSliceLeaky(Data.Rules, allocator, file_content, .{});
try rules.appendSlice(content.rules);
try rules.append(rule);
const out = try std.json.stringifyAlloc(allocator, rules.items, .{});
const out_rules = Data.Rules{ .rules = rules.items };
const out = try std.json.stringifyAlloc(allocator, out_rules, .{});
try file_write.writeAll(out);
file_write.close();

View file

@ -1,6 +1,7 @@
const std = @import("std");
const jetzig = @import("jetzig");
const jetquery = @import("jetzig").jetquery;
const lastfm = @import("../../types.zig").LastFM;
const Data = @import("../../types.zig");
const rules = @import("../../apply_rule.zig");

View file

@ -23,81 +23,67 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
defer rule_file.close();
const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000);
const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null;
const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null;
var job = try request.job("process_scrobbles");
const source = params.getT(.integer, "t").?; // This param is required in HTML
const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: {
const date = params.getT(.string, "latest-date").?;
break :blk try zeit.Time.fromISO8601(date);
} else blk: {
break :blk (try zeit.instant(.{ .source = .now })).time();
};
const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: {
const date = params.getT(.string, "earliest-date").?;
break :blk try zeit.Time.fromISO8601(date);
} else blk: {
break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time();
};
const before_limiter: bool = if (params.get("bbool")) |_| true else false;
const after_limiter: bool = if (params.get("abool")) |_| true else false;
const earliest_timestamp = earliest_date.instant().unixTimestamp();
const latest_timestamp = latest_date.instant().unixTimestamp();
var view_params = try root.put("scrobbles", .array);
var job_params = try job.params.put("scrobbles", .array);
var scrobbles_view = try root.put("scrobbles", .array);
var scrobbles_data = try job.params.put("scrobbles", .array);
var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0;
const imported_scrobbles: Data.ImportedScrobbles = switch (source) {
0 => Data.ImportedScrobbles{ .LastFMStats = (try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true })).scrobbles },
1 => Data.ImportedScrobbles{ .Spotify = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, if (try request.file("upload")) |file| file.content else unreachable, .{ .ignore_unknown_fields = true }) },
2 => blk: {
const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator };
var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator);
var scrobble_buffer = std.ArrayList(Data.LastFMWebScrobble).init(request.allocator);
switch (source) {
0, 1 => {
if (try request.file("upload")) |file| {
std.log.debug("{s}", .{file.filename});
switch (source) {
0 => {
const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{});
const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0;
const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807;
appends: for (content.scrobbles) |scrobble| {
// We can short-circuit on the limiter bools
if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends;
const username = if (params.getT(.string, "username")) |un| un else "VAOTM";
var page: usize = 1;
var max_pages: ?usize = null;
while (true) : (page += 1) {
if (max_pages != null and page > max_pages.?) break;
const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page });
const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
std.log.debug("{}", .{r});
const response_string = try lastfm_response_buffer.toOwnedSlice();
const parsed_lastfm_response = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response_string, .{ .ignore_unknown_fields = true });
//const current_page = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".page, 10);
if (max_pages == null) max_pages = try std.fmt.parseInt(usize, parsed_lastfm_response.recenttracks.@"@attr".totalPages, 10);
try scrobble_buffer.appendSlice(parsed_lastfm_response.recenttracks.track);
}
break :blk Data.ImportedScrobbles{ .LastFMWeb = scrobble_buffer.items };
},
else => unreachable,
};
// Not sure if I should be proud or feel sick
switch (imported_scrobbles) {
inline else => |scrobbles| {
appends: for (scrobbles) |scrobble| {
const filtered_scrobble: Data.Scrobble = blk: switch (@TypeOf(scrobble)) {
Data.IgnorantScrobble => {
if (scrobble.date > latest_timestamp * 1_000 or scrobble.date < earliest_timestamp * 1_000) {
limited_tracks += 1;
continue :appends;
}
break :blk Data.Scrobble{
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, scrobble, rl)
else
Data.Scrobble{
.album = scrobble.album,
.artists_album = &[_][]const u8{scrobble.artist},
.track = scrobble.track,
.artists_track = &[_][]const u8{scrobble.artist},
.date = scrobble.date * 1_000,
.date = scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
},
Data.SpotifyScrobble => {
1 => {
const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true });
const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time();
const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time();
appends: for (content) |scrobble| {
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
skipped_tracks += 1;
continue :appends;
@ -108,56 +94,158 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
}
const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
if ((iso_ts.after(latest_date) or iso_ts.before(earliest_date))) {
if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) {
limited_tracks += 1;
continue :appends;
}
break :blk Data.Scrobble{
.album = scrobble.master_metadata_album_album_name.?,
.artists_album = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
const pre_formatted_scrobble: Data.ImportedScrobble = .{
.track = scrobble.master_metadata_track_name.?,
.artists_track = &[_][]const u8{scrobble.master_metadata_album_artist_name.?},
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1_000_000,
.album = scrobble.master_metadata_album_album_name.?,
.artist = scrobble.master_metadata_album_artist_name.?,
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(),
};
},
Data.LastFMWebScrobble => {
break :blk Data.Scrobble{
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artists_album = &[_][]const u8{scrobble.artist.@"#text"},
.track = scrobble.name,
.artists_track = &[_][]const u8{scrobble.artist.@"#text"},
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10) * 1_000_000,
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
},
else => unreachable,
};
const complete_scrobble = if (rule_list) |rl| try rules.applyScrobbleRule(request.allocator, filtered_scrobble, rl) else filtered_scrobble;
const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble);
try view_params.append(row);
var job_param = try job_params.append(.object);
try job_param.put("album", complete_scrobble.album);
try job_param.put("track", complete_scrobble.track);
try job_param.put("date", complete_scrobble.date);
var track_artists_array = try job_param.put("artists_track", .array);
for (complete_scrobble.artists_track) |a| {
try track_artists_array.append(a);
}
var album_artists_array = try job_param.put("artists_album", .array);
for (complete_scrobble.artists_album) |a| {
try album_artists_array.append(a);
}
try job.schedule();
std.log.debug("Skipped {} tracks", .{skipped_tracks});
std.log.debug("Filtered {} tracks", .{limited_tracks});
}
},
2 => {
if (params.getT(.string, "username")) |username| {
_ = username;
const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&format=json";
const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator };
var ar = std.ArrayList(u8).init(request.allocator);
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const first_response = try ar.toOwnedSlice();
const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true });
appends: for (json.recenttracks.track) |scrobble| {
if (scrobble.date == null) continue :appends;
const pre_formatted_scrobble = Data.ImportedScrobble{
.track = scrobble.name,
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artist = scrobble.artist.@"#text",
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks });
try job.schedule();
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1;
for (2..max_pages) |page| {
const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=1000&page={}&format=json", .{page});
std.log.debug("{s}", .{rest_query});
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const response = try ar.toOwnedSlice();
const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true });
appends: for (json2.recenttracks.track) |scrobble| {
if (scrobble.date == null) continue :appends;
const pre_formatted_scrobble = Data.ImportedScrobble{
.track = scrobble.name,
.album = if (scrobble.album) |album| album.@"#text" else "Not Provided",
.artist = scrobble.artist.@"#text",
.date = try std.fmt.parseInt(i64, scrobble.date.?.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
}
try job.schedule();
}
},
else => unreachable,
}
return request.render(.created);
}

View file

@ -17,10 +17,8 @@
<input type="radio" name="t" label="Last.fm" value="0" required>Last.fm</input>
<input type="radio" name="t" label="Spotify" value="1" required>Spotify</input>
<input type="radio" name="t" label="Last.fm (Web Auth)" value="2" required>Last.fm (WebAuth)</input>
<input type="checkbox" name="adv-opt" label="Advanced Options">Advanced Options</input>
Limit to Scrobbles before: <input type="datetime-local" name="latest-date" label="date-before"></input>
Limit to Scrobbles after: <input type="datetime-local" name="earliest-date" label="date-after"></input>
<input type="checkbox" name="bbool" id="bbool" value="false"></input>Limit to Scrobbles before: <input type="datetime-local" name="b" label="date-before"></input>
<input type="checkbox" name="abool" id="abool" value="false"></input>Limit to Scrobbles after: <input type="datetime-local" name="a" label="date-after"></input>
</fieldset>
</form>
</body>

View file

@ -1,5 +1,5 @@
const std = @import("std");
const Rule = @import("./types.zig").Rule;
const Rules = @import("./types.zig").Rules;
const Data = @import("./types.zig");
// Wrapper for containsAtLeast to make the switch below to work
@ -11,10 +11,18 @@ fn eqlWrapper(haystack: []const u8, needle: []const u8) bool {
return std.mem.eql(u8, haystack, needle);
}
pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble, rules: []Rule) !Data.Scrobble {
var output_scrobble = scrobble;
pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.ImportedScrobble, rules: Rules) !Data.Scrobble {
const artists = try allocator.alloc([]const u8, 1);
artists[0] = scrobble.artist;
var output_scrobble = Data.Scrobble{
.track = scrobble.track,
.artists_track = artists,
.album = scrobble.album,
.artists_album = artists,
.date = scrobble.date,
};
for (rules) |rule| {
for (rules.rules) |rule| {
var match_found: bool = switch (rule.cond_req) {
.any => false,
.all => true,
@ -26,16 +34,10 @@ pub fn applyScrobbleRule(allocator: std.mem.Allocator, scrobble: Data.Scrobble,
};
switch (rule.cond_req) {
.any => switch (cond.match_on) {
inline .album, .track => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
inline .artists_album, .artists_track => |on| {
for (@field(scrobble, @tagName(on))) |artist| match_found = match_found or match_fn(artist, cond.match_txt);
},
inline else => |on| match_found = match_found or match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
},
.all => switch (cond.match_on) {
inline .album, .track => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
inline .artists_album, .artists_track => |on| {
for (@field(scrobble, @tagName(on))) |artist| match_found = match_found and match_fn(artist, cond.match_txt);
},
inline else => |on| match_found = match_found and match_fn(@field(scrobble, @tagName(on)), cond.match_txt),
},
}
}

View file

@ -4,7 +4,7 @@ const Data = @import("types.zig");
pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 {
var date = std.ArrayList(u8).init(allocator);
try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M");
try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M");
return date.items;
}

View file

@ -1,20 +1,7 @@
pub const ImportedScrobbles = union(ScrobbleSources) {
LastFMStats: []IgnorantScrobble,
LastFMWeb: []LastFMWebScrobble,
Spotify: []SpotifyScrobble,
};
const ScrobbleSources = enum {
LastFMStats,
LastFMWeb,
Spotify,
};
pub const IgnorantScrobble = struct {
pub const ImportedScrobble = struct {
track: []const u8,
artist: []const u8,
album: []const u8 = "Not Provided",
//albumId: []const u8,
album: []const u8 = "",
date: i64,
};
@ -27,7 +14,7 @@ pub const Scrobble = struct {
};
// From lastfmstats.com
pub const LastFMStats = struct { username: []const u8, scrobbles: []IgnorantScrobble };
pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble };
// I derived whether or not these values were optional from searching
// the respective fields for null in Vim, so there may be some fields
@ -58,17 +45,7 @@ pub const SpotifyScrobble = struct {
pub const LastFMWeb = struct {
recenttracks: struct {
track: []LastFMWebScrobble,
@"@attr": LastFMWebQueryInfo,
},
};
pub const LastFMWebHyperlinkData = struct {
mbid: []const u8,
@"#text": []const u8,
};
pub const LastFMWebScrobble = struct {
track: []struct {
artist: LastFMWebHyperlinkData,
album: ?LastFMWebHyperlinkData = null,
name: []const u8,
@ -85,9 +62,12 @@ pub const LastFMWebScrobble = struct {
nowplaying: []const u8,
} = null,
url: []const u8,
},
@"@attr": LastFMWebAttr,
},
};
pub const LastFMWebQueryInfo = struct {
pub const LastFMWebAttr = struct {
perPage: []const u8,
totalPages: []const u8,
page: []const u8,
@ -95,11 +75,16 @@ pub const LastFMWebQueryInfo = struct {
total: []const u8,
};
pub const LastFMWebHyperlinkData = struct {
mbid: []const u8,
@"#text": []const u8,
};
pub const Rule = struct {
name: []const u8,
cond_req: enum { any, all },
conditionals: []struct {
match_on: enum { artists_album, artists_track, album, track },
match_on: enum { artist, album, track },
match_cond: enum { is, contains },
match_txt: []const u8,
},
@ -110,6 +95,10 @@ pub const Rule = struct {
},
};
pub const Rules = struct {
rules: []const Rule,
};
// Can't import types in .zmpl files, so defining this here
// doesn't really do much (except maybe in the .zig file for views?)
//pub const HeaderTypes = []enum {