Start process_scrobbles job

src/app/database/Schema.zig

@@ -20,7 +20,7 @@ pub const Album = jetquery.Model(
         id: i32,
         title: []const u8,
         song_num: i32,
-        length: f64,
+        length: f32,
         play_count: i32,
         holiday: bool,
         compilation: bool,
@@ -95,6 +95,9 @@ pub const Scrobble = jetquery.Model(
     struct {
         id: i32,
         date: jetquery.DateTime,
+        song_id: i32,
+        album_id: ?i32,
+        artist_id: i32,
         created_at: jetquery.DateTime,
         updated_at: jetquery.DateTime,
     },
@@ -113,7 +116,7 @@ pub const Song = jetquery.Model(
     struct {
         id: i32,
         title: []const u8,
-        length: f64,
+        length: f32,
         hidden: bool,
         holiday: bool,
         play_count: i32,
@@ -163,7 +166,7 @@ pub const Rating = jetquery.Model(
     struct {
         id: i32,
         reference_id: i32,
-        score: f64,
+        score: f32,
         date: jetquery.DateTime,
         created_at: jetquery.DateTime,
         updated_at: jetquery.DateTime,
@@ -171,12 +174,17 @@ pub const Rating = jetquery.Model(
     .{},
 );
 
-pub const RawScrobble = jetquery.Model(@This(), "raw_scrobbles", struct {
-    id: i32,
-    track: []const u8,
-    artist: []const u8,
-    album: []const u8,
-    date: i32,
-    created_at: jetquery.DateTime,
-    updated_at: jetquery.DateTime,
-}, .{});
+pub const RawScrobble = jetquery.Model(
+    @This(),
+    "raw_scrobbles",
+    struct {
+        id: i32,
+        track: []const u8,
+        artist: []const u8,
+        album: []const u8,
+        date: i32,
+        created_at: jetquery.DateTime,
+        updated_at: jetquery.DateTime,
+    },
+    .{},
+);

src/app/database/migrations/2024-11-30_15-40-39_create_scrobbles.zig

@@ -8,6 +8,9 @@ pub fn up(repo: anytype) !void {
         &.{
             t.primaryKey("id", .{}),
             t.column("date", .datetime, .{}),
+            t.column("song_id", .integer, .{}),
+            t.column("album_id", .integer, .{ .optional = true }),
+            t.column("artist_id", .integer, .{}),
             t.timestamps(.{}),
         },
         .{},

src/app/jobs/process_scrobbles.zig

@@ -1,18 +1,77 @@
 const std = @import("std");
 const jetzig = @import("jetzig");
+const jetquery = @import("jetzig").jetquery;
 
 // The `run` function for a job is invoked every time the job is processed by a queue worker
 // (or by the Jetzig server if the job is processed in-line).
 //
 // Arguments:
 // * allocator: Arena allocator for use during the job execution process.
-// * params:    Params assigned to a job (from a request, any values added to `data`).
+// * params:    Params assigned to a job (from a request, values added to response data).
 // * env:       Provides the following fields:
 //              - logger:      Logger attached to the same stream as the Jetzig server.
 //              - environment: Enum of `{ production, development }`.
 pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void {
     _ = allocator;
     _ = params;
-    // Job execution code goes here. Add any code that you would like to run in the background.
-    try env.logger.INFO("Running a job.", .{});
+
+    // Get all scrobbles from the RawScrobbles table
+
+    const query = jetzig.database.Query(.RawScrobble).select(.{});
+    const scrobbles = try env.repo.all(query);
+    defer env.repo.free(scrobbles);
+
+    for (scrobbles) |scrobble| {
+
+        // Make hashes
+        const album_hash = std.hash.Fnv1a_64.hash(scrobble.album);
+        const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist);
+        const song_hash = std.hash.Fnv1a_64.hash(scrobble.track);
+
+        var album_id: u64 = 0;
+        const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989;
+        if (artist_hash == album_hash) {
+            album_id = album_hash % 99999989;
+        } else {
+            album_id = (artist_hash ^ album_hash) % 99999989;
+        }
+        const artist_id = artist_hash % 99999989;
+
+        // ID start - I think we can use SERIAL for this
+        // We don't compare intermediate IDs to anything,
+        // so keeping it a SERIAL is probably fine
+        const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable;
+        const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable;
+        const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable;
+
+        // Inserts
+        const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id });
+        const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id });
+        const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id });
+        const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false });
+        const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 });
+        const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 });
+
+        // Checks
+        const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo);
+        const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
+        const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
+        const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo);
+        const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo);
+        const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo);
+
+        // Insert into Intermediate Tables
+        if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert);
+        if (albumsong_check == 0) try env.repo.execute(albumsong_insert);
+        if (artistsong_check == 0) try env.repo.execute(artistsong_insert);
+
+        if (album_check == 0) try env.repo.execute(album_insert);
+        if (artist_check == 0) try env.repo.execute(artist_insert);
+        if (song_check == 0) try env.repo.execute(song_insert);
+
+        const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable;
+        try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id }).execute(env.repo);
+    }
+    // Clear RawScrobbles when done processing
+    try jetzig.database.Query(.RawScrobble).deleteAll().execute(env.repo);
 }

src/app/middleware/lastfmMiddleware.zig

@@ -0,0 +1,71 @@
+/// Demo middleware. Assign middleware by declaring `pub const middleware` in the
+/// `jetzig_options` defined in your application's `src/main.zig`.
+///
+/// Middleware is called before and after the request, providing full access to the active
+/// request, allowing you to execute any custom code for logging, tracking, inserting response
+/// headers, etc.
+///
+/// This middleware is configured in the demo app's `src/main.zig`:
+///
+/// ```
+/// pub const jetzig_options = struct {
+///    pub const middleware: []const type = &.{@import("app/middleware/DemoMiddleware.zig")};
+/// };
+/// ```
+const std = @import("std");
+const jetzig = @import("jetzig");
+const jetquery = @import("jetzig").jetquery;
+
+/// Define any custom data fields you want to store here. Assigning to these fields in the `init`
+/// function allows you to access them in various middleware callbacks defined below, where they
+/// can also be modified.
+my_custom_value: []const u8,
+
+const Self = @This();
+
+/// Initialize middleware.
+pub fn init(request: *jetzig.http.Request) !*Self {
+    var middleware = try request.allocator.create(Self);
+    middleware.my_custom_value = "initial value";
+    return middleware;
+}
+
+/// Invoked immediately after the request is received but before it has started processing.
+/// Any calls to `request.render` or `request.redirect` will prevent further processing of the
+/// request, including any other middleware in the chain.
+//pub fn afterRequest(self: *Self, request: *jetzig.http.Request) !void {
+//    const Scrobble = struct {
+//        track: []u8,
+//        artist: []u8,
+//        album: []u8,
+//        date: i64
+//    };
+//
+//    const lastfm = struct {
+//        username: u8,
+//        scrobbles: []Scrobble,
+//    };
+//
+//    var root = try request.data(.object);
+//
+//    if (try request.file("upload")) |file| {
+//        const parsed = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{});
+//        const history = parsed.value;
+//        var scrobbles = try root.put("scrobbles", .array);
+//
+//        for (history.scrobbles) |scrobble| {
+//            try scrobbles.append(scrobble);
+//        }
+//
+//        //var scrobble = try root.put("scrobbles", .array);
+//    }
+//}
+
+/// Invoked immediately before the response renders to the client.
+/// The response can be modified here if needed.
+pub fn beforeResponse(self: *Self, request: *jetzig.http.Request, response: *jetzig.http.Response) !void {
+    try request.server.logger.DEBUG(
+        "[DemoMiddleware:beforeResponse] my_custom_value: {s}, response status: {s}",
+        .{ self.my_custom_value, @tagName(response.status_code) },
+    );
+}

src/app/views/upload.zig

@@ -27,25 +27,31 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
     };
 
     var root = try request.data(.object);
+    var job = try request.job("process_scrobbles");
+    var counter: u16 = 0;
 
     if (try request.file("upload")) |file| {
         const parsed = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{});
 
         const history = parsed.value;
 
-        //std.debug.print("{s}", .{history.scrobbles[19].artist});
-
         var scrobbles = try root.put("scrobbles", .array);
         for (history.scrobbles) |scrobble| {
             try scrobbles.append(scrobble);
+            //const song_hash: u64 = std.hash.Fnv1a_64.hash(scrobble.track) % 99999989;
+            //job.params.put(scrobble.song, song_hash);
+            //std.debug.print("{d}\n", .{song_hash});
 
             const database_update = jetzig.database.Query(.RawScrobble)
-                .insert(.{ .track = scrobble.track, .album = scrobble.album, .artist = scrobble.artist, .date = @divFloor(scrobble.date, 1000) });
+                .insert(.{ .id = counter, .track = scrobble.track, .album = scrobble.album, .artist = scrobble.artist, .date = @divFloor(scrobble.date, 1000) });
 
             try request.repo.execute(database_update);
+            counter += 1;
         }
     }
 
+    try job.schedule();
+
     var upload_table = try root.put("upload_table", .array);
     try upload_table.append("Track");
     try upload_table.append("Artist");

src/main.zig

@@ -16,11 +16,10 @@ pub const jetzig_options = struct {
         jetzig.middleware.HtmxMiddleware,
         // Demo middleware included with new projects. Remove once you are familiar with Jetzig's
         // middleware system.
-        @import("app/middleware/DemoMiddleware.zig"),
     };
 
     // Maximum bytes to allow in request body.
-    // pub const max_bytes_request_body: usize = std.math.pow(usize, 2, 16);
+    pub const max_bytes_request_body: usize = std.math.pow(usize, 2, 32);
 
     // Maximum filesize for `public/` content.
     // pub const max_bytes_public_content: usize = std.math.pow(usize, 2, 20);