diff --git a/build.zig.zon b/build.zig.zon index ac0178f..6abba12 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -16,8 +16,8 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/ae356c0ebf71b41a75e81e81cdeed043686b5c6c.tar.gz", - .hash = "12208504968da1049c81c1e7d28fdb32282d5f7f8226f69f5d37eebe45b580cbc97b", + .url = "https://github.com/jetzig-framework/jetzig/archive/da2978ed04c1248faa06cbcbf1d0a284afeddb5e.tar.gz", + .hash = "1220d2bf337c4a878e88087cc56b44d4a71b0a33e7b57eaedd1b765e3a775865f18a", }, .zeit = .{ .url = "https://github.com/rockorager/zeit/archive/refs/heads/main.tar.gz", diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index c26a8a7..6e0c73d 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -1,13 +1,9 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; +const Scrobble = @import("../../types.zig").LastFMScrobble; +const lastfm = @import("../../types.zig").LastFM; -const Scrobble = struct { - track: []u8, - artist: []u8, - album: ?[]u8, - date: u64, -}; // The `run` function for a job is invoked every time the job is processed by a queue worker // (or by the Jetzig server if the job is processed in-line). // @@ -19,63 +15,74 @@ const Scrobble = struct { // - environment: Enum of `{ production, development }`. pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { _ = allocator; - _ = env; - if (params.getT(.array, "data")) |scrobbles| { - //var scrobble = params.pop(); - for (scrobbles.items()) |val| { - std.log.debug("{s}", .{val}); - // const scrobble = val.coerce(Scrobble); - // // Make hashes - // const album_hash = std.hash.Fnv1a_64.hash(scrobble.album); - // const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist); - // const song_hash = std.hash.Fnv1a_64.hash(scrobble.track); + if (params.getT(.array, "scrobbles")) |scrobbles| { + for (scrobbles.items()) |item| { + const scrobble: Scrobble = .{ .track = item.track.?, .artist = item.artist.?, .album = item.album.?, .date = item.date.? }; - // var album_id: u64 = 0; - // const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989; - // if (artist_hash == album_hash) { - // album_id = album_hash % 99999989; - // } else { - // album_id = (artist_hash ^ album_hash) % 99999989; - // } - // const artist_id = artist_hash % 99999989; + // Make hashes + const album_hash = std.hash.Fnv1a_64.hash(scrobble.album); + const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist); + const song_hash = std.hash.Fnv1a_64.hash(scrobble.track); - // // ID start - I think we can use SERIAL for this - // // We don't compare intermediate IDs to anything, - // // so keeping it a SERIAL is probably fine - // const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable; - // const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable; - // const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable; + // Make IDs + // Song: Song hash XOR artist hash XOR album hash + // This way, if two songs share a name, then + // the IDs also depend on the hash of the album + // they're on, as well as the artist name. As far + // as I can tell, this is only as issue for Sufjan + // Steven's `Songs for Christmas`. - // // Inserts - // const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id }); - // const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id }); - // const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id }); - // const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false }); - // const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 }); - // const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 }); + // Album: If the album is not self-titled, then + // album hash XOR artist hash. This way, if two + // artists have an album of the same name, then + // the IDs also depend on the hash of the artist + // name. As far as I can tell, this is only an + // issue for Weezer. - // // Checks - // const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo); - // const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); - // const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); - // const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo); - // const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo); - // const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo); + // Artist Artist hash. If two artists have the same name, + // then a descriptive string can be provided to + // differentiate after the fact, or in a rule. + var album_id: u64 = 0; + const song_id = (song_hash ^ artist_hash ^ album_hash); + if (artist_hash == album_hash) { + album_id = album_hash; + } else { + album_id = (artist_hash ^ album_hash); + } + const artist_id = artist_hash; - // // Insert into Intermediate Tables - // if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert); - // if (albumsong_check == 0) try env.repo.execute(albumsong_insert); - // if (artistsong_check == 0) try env.repo.execute(artistsong_insert); + const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable; + const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable; + const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable; - // if (album_check == 0) try env.repo.execute(album_insert); - // if (artist_check == 0) try env.repo.execute(artist_insert); - // if (song_check == 0) try env.repo.execute(song_insert); + // Inserts + const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id }); + const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id }); + const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id }); + const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false }); + const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 }); + const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 }); - // const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable; - // try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo); + // Checks + const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo); + const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); + const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); + const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo); + const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo); + const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo); - //scrobble = params.pop(); + // Insert into Intermediate Tables + if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert); + if (albumsong_check == 0) try env.repo.execute(albumsong_insert); + if (artistsong_check == 0) try env.repo.execute(artistsong_insert); + + if (album_check == 0) try env.repo.execute(album_insert); + if (artist_check == 0) try env.repo.execute(artist_insert); + if (song_check == 0) try env.repo.execute(song_insert); + + const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable; + try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo); } } } diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index cb76636..f6e19d3 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -1,6 +1,8 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; +const Scrobble = @import("../../types.zig").LastFMScrobble; +const lastfm = @import("../../types.zig").LastFM; pub fn index(request: *jetzig.Request, data: *jetzig.Data) !jetzig.View { _ = data; @@ -14,35 +16,29 @@ pub fn get(id: []const u8, request: *jetzig.Request, data: *jetzig.Data) !jetzig } pub fn post(request: *jetzig.Request) !jetzig.View { - const Scrobble = struct { - track: []u8, - artist: []u8, - album: ?[]u8, - date: u64, - }; - - const lastfm = struct { - username: []u8, - scrobbles: []Scrobble, - }; - var root = try request.data(.object); - var job = try request.job("process_scrobbles"); - var uploaded_scrobbles = try job.params.put("data", .array); if (try request.file("upload")) |file| { const content = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{}); defer content.deinit(); const history = content.value; - var scrobbles = try root.put("scrobbles", .array); - for (history.scrobbles) |scrobble| { - try scrobbles.append(scrobble); - try uploaded_scrobbles.append(scrobble); - } - } + var scrobbles_view = try root.put("scrobbles", .array); - try job.schedule(); + var job = try request.job("process_scrobbles"); + var scrobbles_data = try job.params.put("scrobbles", .array); + + for (history.scrobbles) |scrobble| { + var value = try scrobbles_data.append(.object); + // This is so unnecessary, probably useful once I start doing Spotify integration though + inline for (std.meta.fields(Scrobble)) |f| { + try value.put(f.name, @as(f.type, @field(scrobble, f.name))); + } + // Note sure why this works for ZMPL, but not for jobs. + try scrobbles_view.append(scrobble); + } + try job.schedule(); + } var upload_table = try root.put("upload_table", .array); try upload_table.append("Track"); diff --git a/src/types.zig b/src/types.zig new file mode 100644 index 0000000..e0905e7 --- /dev/null +++ b/src/types.zig @@ -0,0 +1,33 @@ +pub const LastFMScrobble = struct { + track: []u8, + artist: []u8, + album: ?[]u8, + date: u64, +}; + +// From lastfmstats.com +pub const LastFM = struct { username: []u8, scrobbles: []LastFMScrobble }; + +pub const SpotifyScrobble = struct { + ts: []u8, + username: []u8, + platform: []u8, + ms_played: u64, + conn_country: []u8, + ip_addr_decrypted: []u8, + user_agent_decrypted: []u8, + master_metadata_track_name: []u8, + master_metadata_artist_name: []u8, + master_metadata_album_name: []u8, + spotify_track_uri: []u8, + episode_name: []u8, + reason_start: []u8, + reason_end: []u8, + shuffle: bool, + skipped: bool, + offline: bool, + offline_timestamp: u64, + incognito_mode: bool, +}; + +pub const Spotify = struct { scrobbles: []SpotifyScrobble };