diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 15deee3..0beaf95 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -15,7 +15,17 @@ pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { pub fn post(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); - const params = try request.params(); + //const params = try request.params(); + + const UploadParams = struct { + source: enum { LFMW, LFMS, Spotify }, + earliest_date: ?[]const u8, + latest_date: ?[]const u8, + username: ?[]const u8, + }; + + const params = (try request.expectParams(UploadParams)).?; + const rule_file = try (std.fs.cwd().openFile("rules.json", .{ .mode = .read_only }) catch |err| switch (err) { error.FileNotFound => std.fs.cwd().createFile("rules.json", .{ .read = true }), else => err, @@ -26,43 +36,39 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; //var job = try request.job("process_scrobbles"); var job = try request.job("process_scrobbles2"); - const source = params.getT(.integer, "t").?; // This param is required in HTML - const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { - const date = params.getT(.string, "latest-date").?; - break :blk try zeit.Time.fromISO8601(date); - } else (try zeit.instant(.{ .source = .now })).time(); + // We can parse the dates better + const latest_ts = if (params.latest_date) |ld| + (try zeit.instant(.{ .source = .{ .iso8601 = ld } })).timestamp + else + (try zeit.instant(.{ .source = .now })).timestamp; - const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { - const date = params.getT(.string, "earliest-date").?; - break :blk try zeit.Time.fromISO8601(date); - } else (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); - - const earliest_timestamp = earliest_date.instant().unixTimestamp(); - const latest_timestamp = latest_date.instant().unixTimestamp(); + const earliest_ts = if (params.earliest_date) |ed| + (try zeit.instant(.{ .source = .{ .iso8601 = ed } })).timestamp + else + (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).timestamp; var view_params = try root.put("scrobbles", .array); - //var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; - const imported_scrobbles: []Data.UnifiedScrobble = switch (source) { - 0, 1 => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), - 2 => blk: { + const imported_scrobbles: []Data.UnifiedScrobble = switch (params.source) { + .LFMS, .Spotify => try Utils.scrobbleIngest(request.allocator, if (try request.file("upload")) |file| file.content else unreachable), + .LFMW => blk: { const user_agent: []const u8 = "Zuletzt/0.0.1"; var client = Client{ .allocator = request.allocator }; var lastfm_response_buffer = std.ArrayList(u8).init(request.allocator); var scrobble_buffer = std.ArrayList(Data.UnifiedScrobble).init(request.allocator); - const username = if (params.getT(.string, "username")) |un| un else "VAOTM"; + const username = if (params.username) |un| un else "VAOTM"; var page: usize = 1; //var max_pages: ?usize = null; while (true) : (page += 1) { if (page > 91) break; - const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, earliest_timestamp, latest_timestamp, page }); + const query: []const u8 = try std.fmt.allocPrint(request.allocator, "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user={s}&api_key=b0c410a48a6078a651e0832699e3cd41&from={}&to={}&page={}&limit=1000&format=json", .{ username, @divFloor(earliest_ts, std.time.ns_per_s), @divFloor(latest_ts, std.time.ns_per_s), page }); const r = try client.fetch(.{ .response_storage = .{ .dynamic = &lastfm_response_buffer }, .location = .{ .url = query }, .method = .GET, .headers = .{ .user_agent = .{ .override = user_agent } } }); std.log.debug("{}: {}", .{ page, r }); if (@intFromEnum(r.status) == 500) { @@ -79,7 +85,6 @@ pub fn post(request: *jetzig.Request) !jetzig.View { break :blk try scrobble_buffer.toOwnedSlice(); }, - else => unreachable, }; var artists = try job.params.put("artists", .object); @@ -89,8 +94,10 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var albumsongs = try job.params.put("albumsongs", .object); var albumsongsartists = try job.params.put("albumsongsartists", .object); + var hash_buffer = [_]u8{undefined} ** 20; // A minimum i64 needs 19 digits + 1 negative sign + appends: for (imported_scrobbles) |scrobble| { - if (scrobble.date > latest_timestamp * std.time.ns_per_s or scrobble.date < earliest_timestamp * std.time.ns_per_s) { + if (scrobble.date > latest_ts or scrobble.date < earliest_ts) { limited_tracks += 1; continue :appends; } @@ -124,18 +131,20 @@ pub fn post(request: *jetzig.Request) !jetzig.View { try album_hash_string.appendSlice(artist); const artist_hash = std.hash.Fnv1a_64.hash(artist); try stored_artist_hashes.append(artist_hash); - const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + //const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + const signed_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artist_hash))}); if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); } try album_hash_string.appendSlice(complete_scrobble.album); const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); - const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + const signed_album_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(album_hash))}); if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); for (stored_artist_hashes.items) |artist_hash| { const artistalbum_hash = pair(artist_hash, album_hash); - const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + //const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + const signed_artistalbums_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); if (tracks.get(signed_artistalbums_hash_string) == null) { var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); @@ -147,28 +156,28 @@ pub fn post(request: *jetzig.Request) !jetzig.View { try track_hash_string.appendSlice(complete_scrobble.album); try track_hash_string.appendSlice(complete_scrobble.track); const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); - const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + const signed_track_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(track_hash))}); if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); const albumsong_hash = pair(album_hash, track_hash); - const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + const signed_albumsong_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(albumsong_hash))}); if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { var albumsong_scrobbles = albumsong.get("scrobbles"); - try albumsong_scrobbles.?.append(complete_scrobble.date); + try albumsong_scrobbles.?.append(@divFloor(complete_scrobble.date, std.time.ns_per_us)); // MICROSECONDS } else { var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); try albumsong.put("album", @as(i64, @bitCast(album_hash))); try albumsong.put("song", @as(i64, @bitCast(track_hash))); var albumsong_scrobbles = try albumsong.put("scrobbles", .array); - try albumsong_scrobbles.append(complete_scrobble.date); + try albumsong_scrobbles.append(@divFloor(complete_scrobble.date, std.time.ns_per_us)); // MICROSECONDS } for (complete_scrobble.artists_track) |artist| { const artist_hash = std.hash.Fnv1a_64.hash(artist); - const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + const signed_artist_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(artist_hash))}); if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); const albumsongsartist_hash = pair(albumsong_hash, artist_hash); - const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + const signed_albumsongsartist_hash_string = try std.fmt.bufPrint(&hash_buffer, "{}", .{@as(i64, @bitCast(albumsongsartist_hash))}); if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); @@ -177,12 +186,14 @@ pub fn post(request: *jetzig.Request) !jetzig.View { } } - std.log.debug("\nSkipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); + std.log.debug("Skipped {} tracks\nFiltered {} tracks by date", .{ skipped_tracks, limited_tracks }); try job.schedule(); return request.render(.created); } +// Cantor Pairing Function +// https://en.wikipedia.org/wiki/Pairing_function fn pair(a: u64, b: u64) u64 { - return @divFloor((a +% b) *% (a +% b +% 1) +% b, 2); + return @mod(@divFloor((a +% b) *% (a +% b +% 1), 2) +% b, std.math.maxInt(u64)); } diff --git a/src/app/views/upload/index.zmpl b/src/app/views/upload/index.zmpl index 420e08c..bdeaacb 100644 --- a/src/app/views/upload/index.zmpl +++ b/src/app/views/upload/index.zmpl @@ -14,14 +14,29 @@