Working on process_scrobbles.zig
This commit is contained in:
parent
f860b09674
commit
03f5941615
3 changed files with 69 additions and 78 deletions
|
|
@ -16,8 +16,12 @@
|
|||
// internet connectivity.
|
||||
.dependencies = .{
|
||||
.jetzig = .{
|
||||
.url = "https://github.com/jetzig-framework/jetzig/archive/475ed269525624a67004594ddca44dc8ebea1919.tar.gz",
|
||||
.hash = "1220bc060ba2320fa9fed8e554a8b692a93ef73fa3ab40617b9ed1d928d2029297fb",
|
||||
.url = "https://github.com/jetzig-framework/jetzig/archive/ae356c0ebf71b41a75e81e81cdeed043686b5c6c.tar.gz",
|
||||
.hash = "12208504968da1049c81c1e7d28fdb32282d5f7f8226f69f5d37eebe45b580cbc97b",
|
||||
},
|
||||
.zeit = .{
|
||||
.url = "https://github.com/rockorager/zeit/archive/refs/heads/main.tar.gz",
|
||||
.hash = "122022233835adc719535a8e7cefdd2902a67bbfb7ef198441ca9ce89c0593f488c2",
|
||||
},
|
||||
},
|
||||
.paths = .{
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
const std = @import("std");
|
||||
const jetzig = @import("jetzig");
|
||||
const jetquery = @import("jetzig").jetquery;
|
||||
//const time = @cImport({
|
||||
// @cInclude("time.h");
|
||||
//});
|
||||
|
||||
const Scrobble = struct {
|
||||
track: []u8,
|
||||
artist: []u8,
|
||||
album: ?[]u8,
|
||||
date: u64,
|
||||
};
|
||||
// The `run` function for a job is invoked every time the job is processed by a queue worker
|
||||
// (or by the Jetzig server if the job is processed in-line).
|
||||
//
|
||||
|
|
@ -16,71 +19,63 @@ const jetquery = @import("jetzig").jetquery;
|
|||
// - environment: Enum of `{ production, development }`.
|
||||
pub fn run(allocator: std.mem.Allocator, params: *jetzig.data.Value, env: jetzig.jobs.JobEnv) !void {
|
||||
_ = allocator;
|
||||
_ = params;
|
||||
_ = env;
|
||||
if (params.getT(.array, "data")) |scrobbles| {
|
||||
|
||||
//const memory = try allocator.alloc(u8, 19);
|
||||
//var scrobble = params.pop();
|
||||
for (scrobbles.items()) |val| {
|
||||
std.log.debug("{s}", .{val});
|
||||
// const scrobble = val.coerce(Scrobble);
|
||||
// // Make hashes
|
||||
// const album_hash = std.hash.Fnv1a_64.hash(scrobble.album);
|
||||
// const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist);
|
||||
// const song_hash = std.hash.Fnv1a_64.hash(scrobble.track);
|
||||
|
||||
// Get all scrobbles from the RawScrobbles table
|
||||
// var album_id: u64 = 0;
|
||||
// const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989;
|
||||
// if (artist_hash == album_hash) {
|
||||
// album_id = album_hash % 99999989;
|
||||
// } else {
|
||||
// album_id = (artist_hash ^ album_hash) % 99999989;
|
||||
// }
|
||||
// const artist_id = artist_hash % 99999989;
|
||||
|
||||
const query = jetzig.database.Query(.RawScrobble).select(.{});
|
||||
const scrobbles = try env.repo.all(query);
|
||||
defer env.repo.free(scrobbles);
|
||||
// // ID start - I think we can use SERIAL for this
|
||||
// // We don't compare intermediate IDs to anything,
|
||||
// // so keeping it a SERIAL is probably fine
|
||||
// const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
// const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
// const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
|
||||
for (scrobbles) |scrobble| {
|
||||
//const date = [19]u8{};
|
||||
//time.strftime{ date, 19, "%Y-%m-%d %H:%M:%D", scrobbles.date };
|
||||
//time.strftime(memory, 19, "%Y-%m-%d %H:%M:%S", scrobbles.date);
|
||||
//std.debug.print("{s}", .{memory});
|
||||
// // Inserts
|
||||
// const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id });
|
||||
// const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id });
|
||||
// const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id });
|
||||
// const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false });
|
||||
// const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 });
|
||||
// const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 });
|
||||
|
||||
// Make hashes
|
||||
const album_hash = std.hash.Fnv1a_64.hash(scrobble.album);
|
||||
const artist_hash = std.hash.Fnv1a_64.hash(scrobble.artist);
|
||||
const song_hash = std.hash.Fnv1a_64.hash(scrobble.track);
|
||||
// // Checks
|
||||
// const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo);
|
||||
// const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
|
||||
// const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
|
||||
// const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo);
|
||||
// const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo);
|
||||
// const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo);
|
||||
|
||||
var album_id: u64 = 0;
|
||||
const song_id = (song_hash ^ artist_hash ^ album_hash) % 99999989;
|
||||
if (artist_hash == album_hash) {
|
||||
album_id = album_hash % 99999989;
|
||||
} else {
|
||||
album_id = (artist_hash ^ album_hash) % 99999989;
|
||||
// // Insert into Intermediate Tables
|
||||
// if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert);
|
||||
// if (albumsong_check == 0) try env.repo.execute(albumsong_insert);
|
||||
// if (artistsong_check == 0) try env.repo.execute(artistsong_insert);
|
||||
|
||||
// if (album_check == 0) try env.repo.execute(album_insert);
|
||||
// if (artist_check == 0) try env.repo.execute(artist_insert);
|
||||
// if (song_check == 0) try env.repo.execute(song_insert);
|
||||
|
||||
// const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
// try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo);
|
||||
|
||||
//scrobble = params.pop();
|
||||
}
|
||||
const artist_id = artist_hash % 99999989;
|
||||
|
||||
// ID start - I think we can use SERIAL for this
|
||||
// We don't compare intermediate IDs to anything,
|
||||
// so keeping it a SERIAL is probably fine
|
||||
const artistalbum_offset = try jetzig.database.Query(.ArtistAlbum).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
const albumsong_offset = try jetzig.database.Query(.AlbumSong).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
const artistsong_offset = try jetzig.database.Query(.ArtistSong).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
|
||||
// Inserts
|
||||
const artistalbum_insert = jetzig.database.Query(.ArtistAlbum).insert(.{ .id = 1 + artistalbum_offset, .artist_id = artist_id, .album_id = album_id });
|
||||
const albumsong_insert = jetzig.database.Query(.AlbumSong).insert(.{ .id = 1 + albumsong_offset, .song_id = song_id, .album_id = album_id });
|
||||
const artistsong_insert = jetzig.database.Query(.ArtistSong).insert(.{ .id = 1 + artistsong_offset, .artist_id = artist_id, .song_id = song_id });
|
||||
const album_insert = jetzig.database.Query(.Album).insert(.{ .id = album_id, .title = scrobble.album, .song_num = 0, .length = 0.0, .play_count = 0, .holiday = false, .compilation = false, .deluxe = false, .live = false });
|
||||
const artist_insert = jetzig.database.Query(.Artist).insert(.{ .id = artist_id, .name = scrobble.artist, .album_num = 0, .song_num = 0, .play_count = 0 });
|
||||
const song_insert = jetzig.database.Query(.Song).insert(.{ .id = song_id, .title = scrobble.track, .length = 0.0, .hidden = false, .holiday = false, .play_count = 0 });
|
||||
|
||||
// Checks
|
||||
const artistalbum_check = try jetzig.database.Query(.ArtistAlbum).where(.{ .{ .artist_id = artist_id }, .AND, .{ .album_id = album_id } }).count().execute(env.repo);
|
||||
const albumsong_check = try jetzig.database.Query(.AlbumSong).where(.{ .{ .album_id = album_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
|
||||
const artistsong_check = try jetzig.database.Query(.ArtistSong).where(.{ .{ .artist_id = artist_id }, .AND, .{ .song_id = song_id } }).count().execute(env.repo);
|
||||
const album_check = try jetzig.database.Query(.Album).where(.{.{ .id = album_id }}).count().execute(env.repo);
|
||||
const artist_check = try jetzig.database.Query(.Artist).where(.{.{ .id = artist_id }}).count().execute(env.repo);
|
||||
const song_check = try jetzig.database.Query(.Song).where(.{.{ .id = song_id }}).count().execute(env.repo);
|
||||
|
||||
// Insert into Intermediate Tables
|
||||
if (artistalbum_check == 0) try env.repo.execute(artistalbum_insert);
|
||||
if (albumsong_check == 0) try env.repo.execute(albumsong_insert);
|
||||
if (artistsong_check == 0) try env.repo.execute(artistsong_insert);
|
||||
|
||||
if (album_check == 0) try env.repo.execute(album_insert);
|
||||
if (artist_check == 0) try env.repo.execute(artist_insert);
|
||||
if (song_check == 0) try env.repo.execute(song_insert);
|
||||
|
||||
const scrobble_offset = try jetzig.database.Query(.Scrobble).select(.{}).count().execute(env.repo) orelse unreachable;
|
||||
try jetzig.database.Query(.Scrobble).insert(.{ .id = scrobble_offset + 1, .song_id = song_id, .album_id = album_id, .artist_id = artist_id, .date = scrobble.date }).execute(env.repo);
|
||||
}
|
||||
// Clear RawScrobbles when done processing
|
||||
try jetzig.database.Query(.RawScrobble).deleteAll().execute(env.repo);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
|
|||
const Scrobble = struct {
|
||||
track: []u8,
|
||||
artist: []u8,
|
||||
album: []u8,
|
||||
album: ?[]u8,
|
||||
date: u64,
|
||||
};
|
||||
|
||||
|
|
@ -28,25 +28,17 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
|
|||
|
||||
var root = try request.data(.object);
|
||||
var job = try request.job("process_scrobbles");
|
||||
var counter: u16 = 0;
|
||||
var uploaded_scrobbles = try job.params.put("data", .array);
|
||||
|
||||
if (try request.file("upload")) |file| {
|
||||
const parsed = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{});
|
||||
|
||||
const history = parsed.value;
|
||||
const content = try std.json.parseFromSlice(lastfm, request.allocator, file.content, .{});
|
||||
defer content.deinit();
|
||||
const history = content.value;
|
||||
|
||||
var scrobbles = try root.put("scrobbles", .array);
|
||||
for (history.scrobbles) |scrobble| {
|
||||
try scrobbles.append(scrobble);
|
||||
//const song_hash: u64 = std.hash.Fnv1a_64.hash(scrobble.track) % 99999989;
|
||||
//job.params.put(scrobble.song, song_hash);
|
||||
//std.debug.print("{d}\n", .{song_hash});
|
||||
|
||||
const database_update = jetzig.database.Query(.RawScrobble)
|
||||
.insert(.{ .id = counter, .track = scrobble.track, .album = scrobble.album, .artist = scrobble.artist, .date = (scrobble.date * 1000) });
|
||||
|
||||
try request.repo.execute(database_update);
|
||||
counter += 1;
|
||||
try uploaded_scrobbles.append(scrobble);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue