Compare commits

...

6 commits

Author SHA1 Message Date
5697f95355 Fix LastFM uploading errors
Not sure which of these actually made it work, will probably work backwards at some point to reverse engineer it
2025-05-15 20:23:53 -04:00
89e98c7a47 Allow uploads from LastFM API
Very slow at the moment. Look into ways to speed this up
2025-05-15 20:23:12 -04:00
f69ffb2b37 Move upload.zig to the new table partial 2025-05-15 20:22:34 -04:00
52fefc9ba5 Create dateCompare function
Will eventually try to move away from zeit. Don't need all of it's functionality as long as SQL can format dates
2025-05-15 20:22:09 -04:00
4991bac9a4 Add LastFM scrobble type
In preparation for importing via LastFM api
2025-05-15 15:39:21 -04:00
c42b8d24dd Fix typo 2025-05-15 15:37:21 -04:00
9 changed files with 281 additions and 126 deletions

View file

@ -1,7 +1,7 @@
const std = @import("std");
const jetzig = @import("jetzig");
const jetquery = @import("jetzig").jetquery;
const TableRow = @import("../../types.zig").TableRows;
const TableRow = @import("../../types.zig").TableRow;
const HyperlinkData = @import("../../types.zig").HyperlinkData;
pub fn index(request: *jetzig.Request) !jetzig.View {

View file

@ -1,7 +1,7 @@
const std = @import("std");
const jetzig = @import("jetzig");
const jetquery = @import("jetzig").jetquery;
const TableRow = @import("../../types.zig").TableRows;
const TableRow = @import("../../types.zig").TableRow;
const dateFmt = @import("../../date_fmt.zig").dateFmt;
const ordinalFmt = @import("../../ordinal_fmt.zig").ordinalFmt;

View file

@ -1,7 +1,7 @@
const std = @import("std");
const jetzig = @import("jetzig");
const zeit = @import("zeit");
const TableRow = @import("../../types.zig").TableRows;
const TableRow = @import("../../types.zig").TableRow;
const HyperlinkData = @import("../../types.zig").HyperlinkData;
pub fn index(request: *jetzig.Request) !jetzig.View {

View file

@ -1,6 +1,6 @@
const std = @import("std");
const jetzig = @import("jetzig");
const TableRow = @import("../../types.zig").TableRows;
const TableRow = @import("../../types.zig").TableRow;
const HyperlinkData = @import("../../types.zig").HyperlinkData;
pub fn index(request: *jetzig.Request) !jetzig.View {

View file

@ -4,6 +4,8 @@ const jetquery = @import("jetzig").jetquery;
const zeit = @import("zeit");
const rules = @import("../../apply_rule.zig");
const Data = @import("../../types.zig");
const Utils = @import("../../date_fmt.zig");
const Client = std.http.Client;
pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View {
_ = data;
@ -13,99 +15,152 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View {
pub fn post(request: *jetzig.Request) !jetzig.View {
var root = try request.data(.object);
if (try request.file("upload")) |file| {
const params = try request.params();
const source = try std.fmt.parseInt(u8, params.get("t").?.string.value, 10); // This param is required in HTML
const before_limiter: bool = if (params.get("bbool")) |_| true else false;
const after_limiter: bool = if (params.get("abool")) |_| true else false;
const params = try request.params();
const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) {
error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }),
else => err,
});
var scrobbles_view = try root.put("scrobbles", .array);
var job = try request.job("process_scrobbles");
var scrobbles_data = try job.params.put("scrobbles", .array);
defer rule_file.close();
const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000);
const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null;
var job = try request.job("process_scrobbles");
const source = params.getT(.integer, "t").?; // This param is required in HTML
const before_limiter: bool = if (params.get("bbool")) |_| true else false;
const after_limiter: bool = if (params.get("abool")) |_| true else false;
var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0;
var scrobbles_view = try root.put("scrobbles", .array);
var scrobbles_data = try job.params.put("scrobbles", .array);
const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) {
error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }),
else => err,
});
var skipped_tracks: u64 = 0;
var limited_tracks: u64 = 0;
defer rule_file.close();
const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000);
const rule_list = std.json.parseFromSliceLeaky(Data.Rules, request.allocator, rule_file_content, .{}) catch null;
switch (source) {
0, 1 => {
if (try request.file("upload")) |file| {
std.log.debug("{s}", .{file.filename});
switch (source) {
0 => {
const content: Data.LastFMStats = try std.json.parseFromSliceLeaky(Data.LastFMStats, request.allocator, file.content, .{});
const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0;
const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807;
appends: for (content.scrobbles) |scrobble| {
// We can short-circuit on the limiter bools
if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends;
switch (source) {
0 => {
const content: Data.LastFM = try std.json.parseFromSliceLeaky(Data.LastFM, request.allocator, file.content, .{});
const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0;
const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807;
appends: for (content.scrobbles) |scrobble| {
// We can short-circuit on the limiter bools
if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends;
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, scrobble, rl)
else
Data.Scrobble{
.album = scrobble.album,
.artists_album = &[_][]const u8{scrobble.artist},
.track = scrobble.track,
.artists_track = &[_][]const u8{scrobble.artist},
.date = scrobble.date,
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, scrobble, rl)
else
Data.Scrobble{
.album = scrobble.album,
.artists_album = &[_][]const u8{scrobble.artist},
.track = scrobble.track,
.artists_track = &[_][]const u8{scrobble.artist},
.date = scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
var scrobble_view = try scrobbles_view.append(.object);
var artists = try scrobble_view.put("artists", .array);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
try scrobble_view.put("track", formatted_scrobble.track);
try scrobble_view.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_track) |artist| {
try artists.append(artist);
}
try scrobble_view.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var scrobble_data = try scrobbles_data.append(.object);
var artists_album = try scrobble_data.put("artists_album", .array);
var artists_track = try scrobble_data.put("artists_track", .array);
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
},
1 => {
const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true });
const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "b").?)) else (try zeit.instant(.{})).time();
const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.getT(.string, "a").?)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time();
appends: for (content) |scrobble| {
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
skipped_tracks += 1;
continue :appends;
}
if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
skipped_tracks += 1;
continue :appends;
}
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_album) |artist| {
try artists_album.append(artist);
}
const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) {
limited_tracks += 1;
continue :appends;
}
for (formatted_scrobble.artists_track) |artist| {
try artists_track.append(artist);
}
try scrobble_data.put("date", formatted_scrobble.date);
const pre_formatted_scrobble: Data.ImportedScrobble = .{
.track = scrobble.master_metadata_track_name.?,
.album = scrobble.master_metadata_album_album_name.?,
.artist = scrobble.master_metadata_album_artist_name.?,
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp(),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
},
else => unreachable,
}
},
1 => {
const content: []Data.SpotifyScrobble = try std.json.parseFromSliceLeaky([]Data.SpotifyScrobble, request.allocator, file.content, .{ .ignore_unknown_fields = true });
const before_limiting_date: zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else (try zeit.instant(.{})).time();
const after_limiting_date: zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else (try zeit.instant(.{ .source = .{ .unix_nano = 0 } })).time();
appends: for (content) |scrobble| {
if (scrobble.ms_played < 30_000 and (scrobble.reason_end == null or !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
skipped_tracks += 1;
continue :appends;
}
if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
skipped_tracks += 1;
continue :appends;
}
try job.schedule();
std.log.debug("Skipped {} tracks", .{skipped_tracks});
std.log.debug("Filtered {} tracks", .{limited_tracks});
}
},
2 => {
if (params.getT(.string, "username")) |username| {
_ = username;
const query: []const u8 = "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&format=json";
const user_agent: []const u8 = "Zuletzt/0.0.1";
var client = Client{ .allocator = request.allocator };
var ar = std.ArrayList(u8).init(request.allocator);
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const first_response = try ar.toOwnedSlice();
const json = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, first_response, .{ .ignore_unknown_fields = true });
const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date) or iso_ts.before(after_limiting_date))) {
limited_tracks += 1;
continue :appends;
}
const pre_formatted_scrobble: Data.ImportedScrobble = .{
.track = scrobble.master_metadata_track_name.?,
.album = scrobble.master_metadata_album_album_name.?,
.artist = scrobble.master_metadata_album_artist_name.?,
.date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000,
for (json.recenttracks.track) |scrobble| {
const pre_formatted_scrobble = Data.ImportedScrobble{
.track = scrobble.name,
.album = if (scrobble.album) |album| album.@"#text".? else "",
.artist = scrobble.artist.@"#text".?,
.date = try std.fmt.parseInt(i64, scrobble.date.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
@ -119,44 +174,76 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
.date = pre_formatted_scrobble.date,
};
var scrobble_view = try scrobbles_view.append(.object);
var artists = try scrobble_view.put("artists", .array);
try scrobble_view.put("track", formatted_scrobble.track);
try scrobble_view.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_track) |artist| {
try artists.append(artist);
}
try scrobble_view.put("date", formatted_scrobble.date);
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
var artists_album = try scrobble_data.put("artists_album", .array);
var artists_track = try scrobble_data.put("artists_track", .array);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("album", formatted_scrobble.album);
for (formatted_scrobble.artists_album) |artist| {
try artists_album.append(artist);
}
for (formatted_scrobble.artists_track) |artist| {
try artists_track.append(artist);
}
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
},
else => unreachable,
}
try job.schedule();
std.log.debug("Skipped {} tracks", .{skipped_tracks});
std.log.debug("Filtered {} tracks", .{limited_tracks});
const max_pages = (try std.fmt.parseInt(usize, json.recenttracks.@"@attr".totalPages, 10)) + 1;
for (2..max_pages) |page| {
const rest_query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=VAOTM&api_key=b0c410a48a6078a651e0832699e3cd41&limit=200&page={}&format=json", .{page});
std.log.debug("{s}", .{rest_query});
_ = try client.fetch(.{ .response_storage = .{ .dynamic = &ar }, .location = .{ .url = rest_query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } });
const response = try ar.toOwnedSlice();
const json2 = try std.json.parseFromSliceLeaky(Data.LastFMWeb, request.allocator, response, .{ .ignore_unknown_fields = true });
for (json2.recenttracks.track) |scrobble| {
const pre_formatted_scrobble = Data.ImportedScrobble{
.track = scrobble.name,
.album = if (scrobble.album) |album| album.@"#text".? else "",
.artist = scrobble.artist.@"#text".?,
.date = try std.fmt.parseInt(i64, scrobble.date.uts, 10),
};
const formatted_scrobble = if (rule_list) |rl|
try rules.applyScrobbleRule(request.allocator, pre_formatted_scrobble, rl)
else
Data.Scrobble{
.album = pre_formatted_scrobble.album,
.artists_album = &[_][]const u8{pre_formatted_scrobble.artist},
.track = pre_formatted_scrobble.track,
.artists_track = &[_][]const u8{pre_formatted_scrobble.artist},
.date = pre_formatted_scrobble.date,
};
const row = try Utils.scrobbleToRow(request.allocator, formatted_scrobble);
try scrobbles_view.append(row);
//try scrobbles_data.append(formatted_scrobble);
var scrobble_data = try scrobbles_data.append(.object);
try scrobble_data.put("album", formatted_scrobble.album);
try scrobble_data.put("track", formatted_scrobble.track);
try scrobble_data.put("date", formatted_scrobble.date);
var taa = try scrobble_data.put("artists_track", .array);
for (formatted_scrobble.artists_track) |a| {
try taa.append(a);
}
var aaa = try scrobble_data.put("artists_album", .array);
for (formatted_scrobble.artists_album) |a| {
try aaa.append(a);
}
}
}
try job.schedule();
}
},
else => unreachable,
}
var upload_table = try root.put("upload_table", .array);
try upload_table.append("Track");
try upload_table.append("Artist");
try upload_table.append("Album");
try upload_table.append("Date");
return request.render(.created);
}

View file

@ -9,11 +9,14 @@
</div>
<form action="/upload" enctype="multipart/form-data" method="POST">
<label>File</label>
<input type="file" name="upload" />
<input type="submit" value="Submit" />
<input type="file" name="upload"/>
<input type="submit" value="Submit"/>
<label>Username</label>
<input type="text" name="username"/>
<fieldset>
<input type="radio" name="t" label="Last.fm" value="0" required>Last.fm</input>
<input type="radio" name="t" label="Spotify" value="1" required>Spotify</input>
<input type="radio" name="t" label="Last.fm (Web Auth)" value="2" required>Last.fm (WebAuth)</input>
<input type="checkbox" name="bbool" id="bbool" value="false"></input>Limit to Scrobbles before: <input type="datetime-local" name="b" label="date-before"></input>
<input type="checkbox" name="abool" id="abool" value="false"></input>Limit to Scrobbles after: <input type="datetime-local" name="a" label="date-after"></input>
</fieldset>

View file

@ -1,15 +1,16 @@
@zig {
const ColumnChoices = []const enum{song, album, artist, artistlist, scrobbles, date};
const columns: ColumnChoices = &.{.song, .artistlist, .album, .date};
}
<html>
<head>
<link rel="stylesheet" href="styles.css">
<meta charset="UTF-8">
</head>
<body>
@partial partials/header
<h1>File Uploaded Successfully</h1>
<h2>Scrobbles Added</h2>
@partial partials/table(table_data: .scrobbles, table_headers: .upload_table, table_context: .context)
@partial partials/newtable(T: ColumnChoices, table_data: .scrobbles, columns: columns)
</body>
</html>

View file

@ -1,8 +1,35 @@
const std = @import("std");
const zeit = @import("zeit");
const Data = @import("types.zig");
pub fn dateFmt(allocator: std.mem.Allocator, epoch: i64) ![]const u8 {
var date = std.ArrayList(u8).init(allocator);
try (try zeit.instant(.{ .source = .{ .unix_timestamp = @divFloor(epoch, 1_000) } })).time().strftime(date.writer(), "%d %b %Y, %H:%M");
return date.items;
}
pub fn dateCompare(self: *[]const u8, earliest: []const u8, latest: []const u8) bool {
const range = if (self.len == 19) 4 else 2;
const time = std.fmt.parseInt(u32, self[0..range], 10) catch 0;
const etime = std.fmt.parseInt(u32, earliest[0..range], 10) catch 0;
const ltime = std.fmt.parseInt(u32, latest[0..range], 10) catch 0;
if (time != etime and time != ltime) {
return (time > etime and time < ltime);
} else {
return dateCompare(self[range + 1 ..], earliest[range + 1 ..], latest[range + 1 ..]);
}
}
pub fn scrobbleToRow(allocator: std.mem.Allocator, scrobble: Data.Scrobble) !Data.TableRow {
var artistlist = std.ArrayList(Data.HyperlinkData).init(allocator);
for (scrobble.artists_track) |a| {
try artistlist.append(Data.HyperlinkData{ .name = a, .id = 0 });
}
return Data.TableRow{
.song = .{ .name = scrobble.track, .id = 0 },
.artistlist = artistlist.items,
.album = .{ .name = scrobble.album, .id = 0 },
.date = try dateFmt(allocator, scrobble.date),
};
}

View file

@ -14,7 +14,7 @@ pub const Scrobble = struct {
};
// From lastfmstats.com
pub const LastFM = struct { username: []const u8, scrobbles: []ImportedScrobble };
pub const LastFMStats = struct { username: []const u8, scrobbles: []ImportedScrobble };
// I derived whether or not these values were optional from searching
// the respective fields for null in Vim, so there may be some fields
@ -43,6 +43,43 @@ pub const SpotifyScrobble = struct {
//incognito_mode: ?bool,
};
pub const LastFMWeb = struct {
recenttracks: struct {
track: []struct {
artist: LastFMWebHyperlinkData,
album: ?LastFMWebHyperlinkData = null,
name: []const u8,
mbid: ?[]const u8 = null,
image: ?[]struct {
size: []const u8,
@"#text": []const u8,
} = null,
date: struct {
uts: []const u8,
@"#text": []const u8,
},
@"@attr": ?struct {
nowplaying: ?[]const u8 = null,
} = null,
url: ?[]const u8 = null,
},
@"@attr": LastFMWebAttr,
},
};
pub const LastFMWebAttr = struct {
perPage: ?[]const u8 = null,
totalPages: []const u8,
page: ?[]const u8 = null,
user: ?[]const u8 = null,
total: ?[]const u8 = null,
};
pub const LastFMWebHyperlinkData = struct {
mbid: ?[]const u8 = null,
@"#text": ?[]const u8 = null,
};
pub const Rule = struct {
name: []const u8,
cond_req: enum { any, all },
@ -74,7 +111,7 @@ pub const Rules = struct {
//};
//
pub const TableRows = struct {
pub const TableRow = struct {
song: ?HyperlinkData = null,
album: ?HyperlinkData = null,
artist: ?HyperlinkData = null,