diff --git a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig index c429e19..c3bcd12 100644 --- a/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig +++ b/src/app/database/migrations/2025-04-07_14-35-53_create_scrobbles.zig @@ -6,7 +6,7 @@ pub fn up(repo: anytype) !void { try repo.createTable( "scrobbles", &.{ - t.primaryKey("id", .{ .type = .bigint }), + t.primaryKey("id", .{}), t.column("albumsong", .bigint, .{ .reference = .{ "albumsongs", "id" } }), t.column("datetime", .datetime, .{}), t.timestamps(.{}), diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index ea743b9..770bde7 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -49,15 +49,15 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig for (album_artists, 0..album_artists.len) |artist, i| { const artist_name = try artist.coerce([]const u8); album_artist_name_buffer[i] = artist_name; - album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(artist_name))); + album_artist_id_buffer[i] = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(artist_name))); try album_hash_string.appendSlice(artist_name); } try album_hash_string.appendSlice(scrobble.album); try track_hash_string.appendSlice(scrobble.album); - const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(album_hash_string.items))); + const album_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(album_hash_string.items))); try track_hash_string.appendSlice(scrobble.track); - const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_32.hash(track_hash_string.items))); + const track_hash = @as(i64, @bitCast(std.hash.Fnv1a_64.hash(track_hash_string.items))); var albumsong_id = try jetzig.database.Query(.Albumsong) .find(album_hash ^ track_hash) @@ -120,7 +120,7 @@ pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig .select(.{.id}).execute(env.repo); if (artist_id == null) artist_id = try jetzig.database.Query(.Artist) - .insert(.{ .id = artist_id, .name = scrobble_album_artist, .disambiguation = null }) + .insert(.{ .id = album_artist_hash, .name = scrobble_album_artist, .disambiguation = null }) .returning(.{.id}).execute(env.repo); try jetzig.database.Query(.Artistalbum) .insert(.{ .album_id = album_id.?.id, .artist_id = artist_id.?.id }).execute(env.repo); diff --git a/src/app/jobs/process_scrobbles2.zig b/src/app/jobs/process_scrobbles2.zig new file mode 100644 index 0000000..8b3ae66 --- /dev/null +++ b/src/app/jobs/process_scrobbles2.zig @@ -0,0 +1,131 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +// The `run` function for a job is invoked every time the job is processed by a queue worker +// (or by the Jetzig server if the job is processed in-line). +// +// Arguments: +// * allocator: Arena allocator for use during the job execution process. +// * params: Params assigned to a job (from a request, values added to response data). +// * env: Provides the following fields: +// - logger: Logger attached to the same stream as the Jetzig server. +// - environment: Enum of `{ production, development }`. +pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { + _ = allocator; + + for (params.getT(.object, "tracks").?.items()) |track| { + const id = try std.fmt.parseInt(i64, track.key, 10); + + const track_query = try jetzig.database.Query(.Song) + .find(id).execute(env.repo); + + if (track_query == null) { + const name = try track.value.coerce([]const u8); + try jetzig.database.Query(.Song) + .insert(.{ .id = id, .name = name, .length = null, .hidden = false }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albums").?.items()) |album| { + const id = try std.fmt.parseInt(i64, album.key, 10); + + const album_query = try jetzig.database.Query(.Album) + .find(id).execute(env.repo); + + if (album_query == null) { + const name = try album.value.coerce([]const u8); + try jetzig.database.Query(.Album) + .insert(.{ .id = id, .name = name, .length = null }) + .execute(env.repo); + } + } + + for (params.getT(.object, "artists").?.items()) |artist| { + const id = try std.fmt.parseInt(i64, artist.key, 10); + + const artist_query = try jetzig.database.Query(.Artist) + .find(id).execute(env.repo); + + if (artist_query == null) { + const name = try artist.value.coerce([]const u8); + try jetzig.database.Query(.Artist) + .insert(.{ .id = id, .name = name }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albumsongs").?.items()) |as| { + const id = try std.fmt.parseInt(i64, as.key, 10); + + const as_query = try jetzig.database.Query(.Albumsong) + .find(id).execute(env.repo); + + if (as_query == null) { + const track_id = @as(i64, @intCast(as.value.getT(.integer, "song").?)); + const album_id = @as(i64, @intCast(as.value.getT(.integer, "album").?)); + try jetzig.database.Query(.Albumsong) + .insert(.{ .id = id, .song_id = track_id, .album_id = album_id }) + .execute(env.repo); + } + + const scrobbles = as.value.getT(.array, "scrobbles").?; + for (scrobbles.items()) |date| { + try jetzig.database.Query(.Scrobble).insert(.{ .albumsong = id, .datetime = date }) + .execute(env.repo); + } + } + + for (params.getT(.object, "artistalbums").?.items()) |aa| { + const id = try std.fmt.parseInt(i64, aa.key, 10); + + const aa_query = try jetzig.database.Query(.Artistalbum) + .find(id).execute(env.repo); + + if (aa_query == null) { + const artist_id = @as(i64, @intCast(aa.value.getT(.integer, "artist").?)); + const album_id = @as(i64, @intCast(aa.value.getT(.integer, "album").?)); + try jetzig.database.Query(.Artistalbum) + .insert(.{ .id = id, .artist_id = artist_id, .album_id = album_id }) + .execute(env.repo); + } + } + + for (params.getT(.object, "albumsongsartists").?.items()) |asa| { + const id = try std.fmt.parseInt(i64, asa.key, 10); + + const asa_query = try jetzig.database.Query(.Albumsongsartist) + .find(id).execute(env.repo); + + if (asa_query == null) { + const albumsong_id = @as(i64, @intCast(asa.value.getT(.integer, "albumsong").?)); + const artist_id = @as(i64, @intCast(asa.value.getT(.integer, "artist").?)); + try jetzig.database.Query(.Albumsongsartist) + .insert(.{ .id = id, .albumsong_id = albumsong_id, .artist_id = artist_id }) + .execute(env.repo); + } + } + + //for ((params.getT(.object, "albumsongsartists")).?.items(.object)) |asa| { + // const id = try std.fmt.parseInt(i64, asa.key, 10); + // const albumsong_id = asa.value.getT(.integer, "albumsong"); + // const track_artist_id = asa.value.getT(.integer, "artist"); + + // const albumsongartist = try jetzig.database.Query(.Albumsongsartist) + // .find(id) + // .select(.{.id}).execute(env.repo); + + // if (albumsongartist == null) { + // var artist_id = try jetzig.database.Query(.Artist) + // .find(track_artist_id) + // .select(.{.id}).execute(env.repo); + // + // if (artist_id == null) { + // const artist = params.chain(.{"artists",}) + // artist_id = try jetzig.database.Query(.Artist) + // .insert(.{ .id = track_artist_hash, .name = scrobble_track_artist, .disambiguation = null }) + // .execute(env.repo); + // } + // } + //} +} diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index ebe9395..ac2c665 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -24,26 +24,25 @@ pub fn post(request: *jetzig.Request) !jetzig.View { defer rule_file.close(); const rule_file_content = try rule_file.readToEndAlloc(request.allocator, 16_000_000); const rule_list = std.json.parseFromSliceLeaky([]Data.Rule, request.allocator, rule_file_content, .{}) catch null; - var job = try request.job("process_scrobbles"); + //var job = try request.job("process_scrobbles"); + var job = try request.job("process_scrobbles2"); const source = params.getT(.integer, "t").?; // This param is required in HTML + const latest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { const date = params.getT(.string, "latest-date").?; break :blk try zeit.Time.fromISO8601(date); - } else blk: { - break :blk (try zeit.instant(.{ .source = .now })).time(); - }; + } else (try zeit.instant(.{ .source = .now })).time(); + const earliest_date = if (params.getT(.boolean, "adv-opt")) |_| blk: { const date = params.getT(.string, "earliest-date").?; break :blk try zeit.Time.fromISO8601(date); - } else blk: { - break :blk (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); - }; + } else (try zeit.instant(.{ .source = .{ .unix_timestamp = 0 } })).time(); const earliest_timestamp = earliest_date.instant().unixTimestamp(); const latest_timestamp = latest_date.instant().unixTimestamp(); var view_params = try root.put("scrobbles", .array); - var job_params = try job.params.put("scrobbles", .array); + //var job_params = try job.params.put("scrobbles", .array); var skipped_tracks: u64 = 0; var limited_tracks: u64 = 0; @@ -79,6 +78,13 @@ pub fn post(request: *jetzig.Request) !jetzig.View { else => unreachable, }; + var artists = try job.params.put("artists", .object); + var albums = try job.params.put("albums", .object); + var tracks = try job.params.put("tracks", .object); + var artistalbums = try job.params.put("artistalbums", .object); + var albumsongs = try job.params.put("albumsongs", .object); + var albumsongsartists = try job.params.put("albumsongsartists", .object); + // Not sure if I should be proud or feel sick switch (imported_scrobbles) { .LastFMStats => |scrobbles| { @@ -99,7 +105,65 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, .LastFMWeb => |scrobbles| { @@ -117,7 +181,65 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, .Spotify => |scrobbles| { @@ -149,7 +271,66 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const row = try Utils.scrobbleToRow(request.allocator, complete_scrobble); try view_params.append(row); - try job_params.append(complete_scrobble); + //try job_params.append(complete_scrobble); + + var stored_artist_hashes = std.ArrayList(u64).init(request.allocator); + + var album_hash_string = std.ArrayList(u8).init(request.allocator); + for (complete_scrobble.artists_album) |artist| { + try album_hash_string.appendSlice(artist); + const artist_hash = std.hash.Fnv1a_64.hash(artist); + try stored_artist_hashes.append(artist_hash); + const signed_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_hash_string) == null) try artists.put(signed_hash_string, artist); + } + + try album_hash_string.appendSlice(complete_scrobble.album); + const album_hash = std.hash.Fnv1a_64.hash(album_hash_string.items); + const signed_album_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(album_hash))}); + if (albums.get(signed_album_hash_string) == null) try albums.put(signed_album_hash_string, complete_scrobble.album); + + for (stored_artist_hashes.items) |artist_hash| { + const artistalbum_hash = pair(artist_hash, album_hash); + const signed_artistalbums_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artistalbum_hash))}); + if (tracks.get(signed_artistalbums_hash_string) == null) { + var artistalbum = try artistalbums.put(signed_artistalbums_hash_string, .object); + try artistalbum.put("artist", @as(i64, @bitCast(artist_hash))); + try artistalbum.put("album", @as(i64, @bitCast(album_hash))); + } + } + + var track_hash_string = std.ArrayList(u8).init(request.allocator); + try track_hash_string.appendSlice(complete_scrobble.album); + try track_hash_string.appendSlice(complete_scrobble.track); + const track_hash = std.hash.Fnv1a_64.hash(track_hash_string.items); + const signed_track_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(track_hash))}); + if (tracks.get(signed_track_hash_string) == null) try tracks.put(signed_track_hash_string, complete_scrobble.track); + + const albumsong_hash = pair(album_hash, track_hash); + const signed_albumsong_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(albumsong_hash))}); + if (albumsongs.get(signed_albumsong_hash_string)) |albumsong| { + var albumsong_scrobbles = albumsong.get("scrobbles"); + try albumsong_scrobbles.?.append(complete_scrobble.date); + } else { + var albumsong = try albumsongs.put(signed_albumsong_hash_string, .object); + try albumsong.put("album", @as(i64, @bitCast(album_hash))); + try albumsong.put("song", @as(i64, @bitCast(track_hash))); + var albumsong_scrobbles = try albumsong.put("scrobbles", .array); + try albumsong_scrobbles.append(complete_scrobble.date); + } + + for (complete_scrobble.artists_track) |artist| { + const artist_hash = std.hash.Fnv1a_64.hash(artist); + const signed_artist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{@as(i64, @bitCast(artist_hash))}); + if (artists.get(signed_artist_hash_string) == null) try artists.put(signed_artist_hash_string, artist); + const albumsongsartist_hash = pair(albumsong_hash, artist_hash); + const signed_albumsongsartist_hash_string = try std.fmt.allocPrint(request.allocator, "{}", .{albumsongsartist_hash}); + if (albumsongsartists.get(signed_albumsongsartist_hash_string) == null) { + var albumsongartist = try albumsongsartists.put(signed_albumsongsartist_hash_string, .object); + try albumsongartist.put("albumsong", @as(i64, @bitCast(albumsong_hash))); + try albumsongartist.put("artist", @as(i64, @bitCast(artist_hash))); + } + } } }, } @@ -159,3 +340,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { return request.render(.created); } + +fn pair(a: u64, b: u64) u64 { + return @divFloor((a +% b) *% (a +% b +% 1) +% b, 2); +}