mitteneer/zuletzt
1
0
Fork 0
Code Issues 14 Pull requests Projects Releases Packages Wiki Activity Actions

Working on process_scrobbles.zig

Browse source
This commit is contained in:
mitteneer 2025-02-15 14:04:16 -05:00
parent f860b09674
commit 03f5941615
3 changed files with 69 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

8
build.zig.zon
View file

@ -16,8 +16,12 @@
// internet connectivity. // internet connectivity.
.dependencies = .{ .dependencies = .{
.jetzig = .{ .jetzig = .{
.url = "https://github.com/jetzig-framework/jetzig/archive/475ed269525624a67004594ddca44dc8ebea1919.tar.gz", .url = "https://github.com/jetzig-framework/jetzig/archive/ae356c0ebf71b41a75e81e81cdeed043686b5c6c.tar.gz",
.hash = "1220bc060ba2320fa9fed8e554a8b692a93ef73fa3ab40617b9ed1d928d2029297fb", .hash = "12208504968da1049c81c1e7d28fdb32282d5f7f8226f69f5d37eebe45b580cbc97b",
},
.zeit = .{
.url = "https://github.com/rockorager/zeit/archive/refs/heads/main.tar.gz",
.hash = "122022233835adc719535a8e7cefdd2902a67bbfb7ef198441ca9ce89c0593f488c2",
}, },
}, },
.paths = .{ .paths = .{

119
src/app/jobs/process_scrobbles.zig
View file

@ -1,10 +1,13 @@
const std = @import("std"); const std = @import("std");
const jetzig = @import("jetzig"); const jetzig = @import("jetzig");
const jetquery = @import("jetzig").jetquery; const jetquery = @import("jetzig").jetquery;
//const time = @cImport({
// @cInclude("time.h");
//});
const Scrobble = struct {
track: []u8,
artist: []u8,
album: ?[]u8,
date: u64,
};
// The `run` function for a job is invoked every time the job is processed by a queue worker // The `run` function for a job is invoked every time the job is processed by a queue worker
// (or by the Jetzig server if the job is processed in-line). // (or by the Jetzig server if the job is processed in-line).
// //
@ -16,71 +19,63 @@ const jetquery = @import("jetzig").jetquery;
// - environment: Enum of `{ production, development }`. // - environment: Enum of `{ production, development }`.
pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void { pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void {
_ = allocator; _ = allocator;
_ = params; _ = env;
if (params.getT(.array, "data")) |scrobbles| {
//const memory = try allocator.alloc(u8, 19); //var scrobble = params.pop();
for (scrobbles.items()) |val| {
std.log.debug("{s}", .{val});
// const scrobble = val.coerce(Scrobble);
// // Make hashes
// const album_hash = std.hash.Fnv1a_64.hash(scrobble.album);
// const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist);
// const song_hash = std.hash.Fnv1a_64.hash(scrobble.track);
// Get all scrobbles from the RawScrobbles table // var album_id: u64 = 0;
// const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989;
// if (artist_hash == album_hash) {
// album_id = album_hash % 99999989;
// } else {
// album_id = (artist_hash ^ album_hash) % 99999989;
// }
// const artist_id = artist_hash % 99999989;
const query = jetzig.database.Query(.RawScrobble).select(.{}); // // ID start - I think we can use SERIAL for this
const scrobbles = try env.repo.all(query); // // We don't compare intermediate IDs to anything,
defer env.repo.free(scrobbles); // // so keeping it a SERIAL is probably fine
// const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable;
// const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable;
// const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable;
for (scrobbles) |scrobble| { // // Inserts
//const date = [19]u8{}; // const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id });
//time.strftime{ date, 19, "%Y-%m-%d %H:%M:%D", scrobbles.date }; // const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id });
//time.strftime(memory, 19, "%Y-%m-%d %H:%M:%S", scrobbles.date); // const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id });
//std.debug.print("{s}", .{memory}); // const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false });
// const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 });
// const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 });
// Make hashes // // Checks
const album_hash = std.hash.Fnv1a_64.hash(scrobble.album); // const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo);
const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist); // const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
const song_hash = std.hash.Fnv1a_64.hash(scrobble.track); // const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
// const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo);
// const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo);
// const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo);
var album_id: u64 = 0; // // Insert into Intermediate Tables
const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989; // if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert);
if (artist_hash == album_hash) { // if (albumsong_check == 0) try env.repo.execute(albumsong_insert);
album_id = album_hash % 99999989; // if (artistsong_check == 0) try env.repo.execute(artistsong_insert);
} else {
album_id = (artist_hash ^ album_hash) % 99999989; // if (album_check == 0) try env.repo.execute(album_insert);
// if (artist_check == 0) try env.repo.execute(artist_insert);
// if (song_check == 0) try env.repo.execute(song_insert);
// const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable;
// try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo);
//scrobble = params.pop();
} }
const artist_id = artist_hash % 99999989;
// ID start - I think we can use SERIAL for this
// We don't compare intermediate IDs to anything,
// so keeping it a SERIAL is probably fine
const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable;
const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable;
const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable;
// Inserts
const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id });
const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id });
const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id });
const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false });
const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 });
const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 });
// Checks
const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo);
const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo);
const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo);
const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo);
// Insert into Intermediate Tables
if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert);
if (albumsong_check == 0) try env.repo.execute(albumsong_insert);
if (artistsong_check == 0) try env.repo.execute(artistsong_insert);
if (album_check == 0) try env.repo.execute(album_insert);
if (artist_check == 0) try env.repo.execute(artist_insert);
if (song_check == 0) try env.repo.execute(song_insert);
const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable;
try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo);
} }
// Clear RawScrobbles when done processing
try jetzig.database.Query(.RawScrobble).deleteAll().execute(env.repo);
} }

20
src/app/views/upload.zig
View file

@ -17,7 +17,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
const Scrobble = struct { const Scrobble = struct {
track: []u8, track: []u8,
artist: []u8, artist: []u8,
album: []u8, album: ?[]u8,
date: u64, date: u64,
}; };
@ -28,25 +28,17 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
var root = try request.data(.object); var root = try request.data(.object);
var job = try request.job("process_scrobbles"); var job = try request.job("process_scrobbles");
var counter: u16 = 0; var uploaded_scrobbles = try job.params.put("data", .array);
if (try request.file("upload")) |file| { if (try request.file("upload")) |file| {
const parsed = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{}); const content = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{});
defer content.deinit();
const history = parsed.value; const history = content.value;
var scrobbles = try root.put("scrobbles", .array); var scrobbles = try root.put("scrobbles", .array);
for (history.scrobbles) |scrobble| { for (history.scrobbles) |scrobble| {
try scrobbles.append(scrobble); try scrobbles.append(scrobble);
//const song_hash: u64 = std.hash.Fnv1a_64.hash(scrobble.track) % 99999989; try uploaded_scrobbles.append(scrobble);
//job.params.put(scrobble.song, song_hash);
//std.debug.print("{d}\n", .{song_hash});
const database_update = jetzig.database.Query(.RawScrobble)
.insert(.{ .id = counter, .track = scrobble.track, .album = scrobble.album, .artist = scrobble.artist, .date = (scrobble.date * 1000) });
try request.repo.execute(database_update);
counter += 1;
} }
} }