From 03f5941615829cb2c0baac40110916674901798d Mon Sep 17 00:00:00 2001 From: mitteneer Date: Sat, 15 Feb 2025 14:04:16 -0500 Subject: [PATCH] Working on process_scrobbles.zig --- build.zig.zon | 8 +- src/app/jobs/process_scrobbles.zig | 119 ++++++++++++++--------------- src/app/views/upload.zig | 20 ++--- 3 files changed, 69 insertions(+), 78 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index a85ac4d..ac0178f 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -16,8 +16,12 @@ // internet connectivity. .dependencies = .{ .jetzig = .{ - .url = "https://github.com/jetzig-framework/jetzig/archive/475ed269525624a67004594ddca44dc8ebea1919.tar.gz", - .hash = "1220bc060ba2320fa9fed8e554a8b692a93ef73fa3ab40617b9ed1d928d2029297fb", + .url = "https://github.com/jetzig-framework/jetzig/archive/ae356c0ebf71b41a75e81e81cdeed043686b5c6c.tar.gz", + .hash = "12208504968da1049c81c1e7d28fdb32282d5f7f8226f69f5d37eebe45b580cbc97b", + }, + .zeit = .{ + .url = "https://github.com/rockorager/zeit/archive/refs/heads/main.tar.gz", + .hash = "122022233835adc719535a8e7cefdd2902a67bbfb7ef198441ca9ce89c0593f488c2", }, }, .paths = .{ diff --git a/src/app/jobs/process_scrobbles.zig b/src/app/jobs/process_scrobbles.zig index e0d276a..c26a8a7 100644 --- a/src/app/jobs/process_scrobbles.zig +++ b/src/app/jobs/process_scrobbles.zig @@ -1,10 +1,13 @@ const std = @import("std"); const jetzig = @import("jetzig"); const jetquery = @import("jetzig").jetquery; -//const time = @cImport({ -// @cInclude("time.h"); -//}); +const Scrobble = struct { + track: []u8, + artist: []u8, + album: ?[]u8, + date: u64, +}; // The `run` function for a job is invoked every time the job is processed by a queue worker // (or by the Jetzig server if the job is processed in-line). // @@ -16,71 +19,63 @@ const jetquery = @import("jetzig").jetquery; // - environment: Enum of `{ production, development }`. pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { _ = allocator; - _ = params; + _ = env; + if (params.getT(.array, "data")) |scrobbles| { - //const memory = try allocator.alloc(u8, 19); + //var scrobble = params.pop(); + for (scrobbles.items()) |val| { + std.log.debug("{s}", .{val}); + // const scrobble = val.coerce(Scrobble); + // // Make hashes + // const album_hash = std.hash.Fnv1a_64.hash(scrobble.album); + // const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist); + // const song_hash = std.hash.Fnv1a_64.hash(scrobble.track); - // Get all scrobbles from the RawScrobbles table + // var album_id: u64 = 0; + // const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989; + // if (artist_hash == album_hash) { + // album_id = album_hash % 99999989; + // } else { + // album_id = (artist_hash ^ album_hash) % 99999989; + // } + // const artist_id = artist_hash % 99999989; - const query = jetzig.database.Query(.RawScrobble).select(.{}); - const scrobbles = try env.repo.all(query); - defer env.repo.free(scrobbles); + // // ID start - I think we can use SERIAL for this + // // We don't compare intermediate IDs to anything, + // // so keeping it a SERIAL is probably fine + // const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable; + // const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable; + // const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable; - for (scrobbles) |scrobble| { - //const date = [19]u8{}; - //time.strftime{ date, 19, "%Y-%m-%d %H:%M:%D", scrobbles.date }; - //time.strftime(memory, 19, "%Y-%m-%d %H:%M:%S", scrobbles.date); - //std.debug.print("{s}", .{memory}); + // // Inserts + // const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id }); + // const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id }); + // const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id }); + // const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false }); + // const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 }); + // const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 }); - // Make hashes - const album_hash = std.hash.Fnv1a_64.hash(scrobble.album); - const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist); - const song_hash = std.hash.Fnv1a_64.hash(scrobble.track); + // // Checks + // const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo); + // const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); + // const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); + // const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo); + // const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo); + // const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo); - var album_id: u64 = 0; - const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989; - if (artist_hash == album_hash) { - album_id = album_hash % 99999989; - } else { - album_id = (artist_hash ^ album_hash) % 99999989; + // // Insert into Intermediate Tables + // if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert); + // if (albumsong_check == 0) try env.repo.execute(albumsong_insert); + // if (artistsong_check == 0) try env.repo.execute(artistsong_insert); + + // if (album_check == 0) try env.repo.execute(album_insert); + // if (artist_check == 0) try env.repo.execute(artist_insert); + // if (song_check == 0) try env.repo.execute(song_insert); + + // const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable; + // try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo); + + //scrobble = params.pop(); } - const artist_id = artist_hash % 99999989; - - // ID start - I think we can use SERIAL for this - // We don't compare intermediate IDs to anything, - // so keeping it a SERIAL is probably fine - const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable; - const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable; - const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable; - - // Inserts - const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id }); - const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id }); - const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id }); - const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false }); - const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 }); - const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 }); - - // Checks - const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo); - const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); - const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo); - const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo); - const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo); - const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo); - - // Insert into Intermediate Tables - if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert); - if (albumsong_check == 0) try env.repo.execute(albumsong_insert); - if (artistsong_check == 0) try env.repo.execute(artistsong_insert); - - if (album_check == 0) try env.repo.execute(album_insert); - if (artist_check == 0) try env.repo.execute(artist_insert); - if (song_check == 0) try env.repo.execute(song_insert); - - const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable; - try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo); } - // Clear RawScrobbles when done processing - try jetzig.database.Query(.RawScrobble).deleteAll().execute(env.repo); } diff --git a/src/app/views/upload.zig b/src/app/views/upload.zig index 80b8a47..cb76636 100644 --- a/src/app/views/upload.zig +++ b/src/app/views/upload.zig @@ -17,7 +17,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View { const Scrobble = struct { track: []u8, artist: []u8, - album: []u8, + album: ?[]u8, date: u64, }; @@ -28,25 +28,17 @@ pub fn post(request: *jetzig.Request) !jetzig.View { var root = try request.data(.object); var job = try request.job("process_scrobbles"); - var counter: u16 = 0; + var uploaded_scrobbles = try job.params.put("data", .array); if (try request.file("upload")) |file| { - const parsed = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{}); - - const history = parsed.value; + const content = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{}); + defer content.deinit(); + const history = content.value; var scrobbles = try root.put("scrobbles", .array); for (history.scrobbles) |scrobble| { try scrobbles.append(scrobble); - //const song_hash: u64 = std.hash.Fnv1a_64.hash(scrobble.track) % 99999989; - //job.params.put(scrobble.song, song_hash); - //std.debug.print("{d}\n", .{song_hash}); - - const database_update = jetzig.database.Query(.RawScrobble) - .insert(.{ .id = counter, .track = scrobble.track, .album = scrobble.album, .artist = scrobble.artist, .date = (scrobble.date * 1000) }); - - try request.repo.execute(database_update); - counter += 1; + try uploaded_scrobbles.append(scrobble); } }