Allow Spotify history uploads

It works B) Just need to fix date filtering, and I should be able to start writing views for this kind of stuff. After that, editing the data in the browser.

.gitignore

@@ -5,4 +5,5 @@ static/
 .jetzig
 src/app/database/data.db-journal
 src/app/database/old_migrations/
-src/lib/
+src/lib/time.h
+

src/app/views/upload.zig

@@ -23,41 +23,94 @@ pub fn post(request: *jetzig.Request) !jetzig.View {
 
     if (try request.file("upload")) |file| {
         const params = try request.params();
-
+        const source = try std.fmt.parseInt(u8, params.get("t").?.string.value, 10); // This param is required in HTML
-        var content: ScrobbleTypes.UploadData = undefined;
+        // Date limiting is broken atm
-
+        const before_limiter: bool = if (params.get("bbool")) |_| true else false;
-        const source = std.fmt.parseInt(u8, params.get("t").string.value, 10).?;
+        const after_limiter: bool = if (params.get("abool")) |_| true else false;
-        //if (params.get("t")) |param| {
-        //    const source = std.fmt.parseInt(u8, param.string.value, 10);
-        switch (source) {
-            0 => content = try std.json.parseFromSliceLeaky(ScrobbleTypes.LastFM, request.allocator, file.content, .{}),
-            1 => content = try std.json.parseFromSliceLeaky(ScrobbleTypes.Spotify, request.allocator, file.content, .{}),
-            else => unreachable,
-        }
-
-        //const content = try std.json.parseFromSliceLeaky(lastfm, request.allocator, file.content, .{ .ignore_unknown_fields = true });
 
         var scrobbles_view = try root.put("scrobbles", .array);
-
         var job = try request.job("process_scrobbles");
         var scrobbles_data = try job.params.put("scrobbles", .array);
 
-        // This is seconds from Unix epoch
+        var skipped_tracks: u64 = 0;
-        const limiting_date = if (params.get("l")) |param| (try zeit.instant(.{ .source = .{ .iso8601 = param.string.value } })).unixTimestamp() else 9_223_372_036_854_775_807;
+        var limited_tracks: u64 = 0;
 
+        // The only difference between a LastFM scrobble and a Spotify scrobble is the format.
+        // I've made a branches for each, because doing it all in one made the readability terrible,
+        // and formatting the date in particular was challenging. I could probably pull out the
+        // actual appending at some point, since that's the same process for each, but I'm not
+        // sure how to do that yet.
+        switch (source) {
+            0 => {
+                const content: ScrobbleTypes.LastFM = try std.json.parseFromSliceLeaky(ScrobbleTypes.LastFM, request.allocator, file.content, .{});
+                const before_limiting_date = if (before_limiter and params.get("b") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("b").?.string.value } })).unixTimestamp() * 1000 else 0;
+                const after_limiting_date = if (after_limiter and params.get("a") != null) (try zeit.instant(.{ .source = .{ .iso8601 = params.get("a").?.string.value } })).unixTimestamp() * 1000 else 9_223_372_036_854_775_807;
                 appends: for (content.scrobbles) |scrobble| {
-            if (std.mem.eql(u8, @typeName(scrobble), "SpotifyScrobble")) scrobble = scrobble.scrobblize();
+                    // We can short-circuit on the limiter bools
-            // Scrobble.date is in milliseconds from Unix epoch
+                    if ((before_limiter or after_limiter) and (scrobble.date > before_limiting_date or scrobble.date < after_limiting_date)) continue :appends;
-            if (scrobble.date < limiting_date * 1000) continue :appends;
                     var value = try scrobbles_data.append(.object);
+
                     // This is so unnecessary, probably useful once I start doing Spotify integration though
-            inline for (std.meta.fields(ScrobbleTypes.Scrobble)) |f| {
+                    inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| {
                         try value.put(f.name, @as(f.type, @field(scrobble, f.name)));
                     }
                     // Note sure why this works for ZMPL, but not for jobs.
                     try scrobbles_view.append(scrobble);
                 }
+            },
+            1 => {
+                const content: []ScrobbleTypes.SpotifyScrobble = try std.json.parseFromSliceLeaky([]ScrobbleTypes.SpotifyScrobble, request.allocator, file.content, .{});
+                const before_limiting_date: ?zeit.Time = if (before_limiter) (try zeit.Time.fromISO8601(params.get("b").?.string.value)) else null;
+                const after_limiting_date: ?zeit.Time = if (after_limiter) (try zeit.Time.fromISO8601(params.get("a").?.string.value)) else null;
+                appends: for (content) |scrobble| {
+                    // A LastFM Scrobble occurs when half a song has been played
+                    // or the song plays for 4 minutes, whichever happens first.
+                    // Spotify considers a song played if it plays for 30 seconds.
+                    // Ideally, I would go with the LastFM convention, but Spotify
+                    // history data only gives us so much information. I'm okay
+                    // with the 30 second convention, but eventually I would prefer
+                    // to get the song length from MusicBrainz and check if it meets
+                    // the requirement. Until then, if it goes 30 seconds, or the
+                    // reason_end field reads "trackdone", then it counts as a Scrobble.
+                    // May consider giving user control to the minimum millisecond requirement.
+                    if (scrobble.reason_end != null and (scrobble.ms_played < 30_000 and !std.mem.eql(u8, scrobble.reason_end.?, "trackdone"))) {
+                        skipped_tracks += 1;
+                        continue :appends;
+                    }
+                    // In the case where the artist is null, but there's other metadata, I could
+                    // probably let the user edit it in themselves, although I'm not sure if that
+                    // situation happens.
+                    if (scrobble.master_metadata_album_artist_name == null or scrobble.master_metadata_track_name == null) {
+                        skipped_tracks += 1;
+                        continue :appends;
+                    }
+
+                    // I'm separating these on account of the above comment, as well as
+                    // this part being kinda complicated
+                    const iso_ts = try zeit.Time.fromISO8601(scrobble.ts);
+                    if ((before_limiter or after_limiter) and (iso_ts.after(before_limiting_date.?) or iso_ts.before(after_limiting_date.?))) {
+                        limited_tracks += 1;
+                        continue :appends;
+                    }
+
+                    // Turn SpotifyScrobble into a LastFM scrobble
+                    const formatted_scrobble: ScrobbleTypes.LastFMScrobble = .{ .track = scrobble.master_metadata_track_name.?, .album = scrobble.master_metadata_album_album_name.?, .artist = scrobble.master_metadata_album_artist_name.?, .date = (try zeit.instant(.{ .source = .{ .iso8601 = scrobble.ts } })).unixTimestamp() * 1000 };
+
+                    var value = try scrobbles_data.append(.object);
+
+                    // This is so unnecessary, probably useful once I start doing Spotify integration though
+                    inline for (std.meta.fields(ScrobbleTypes.LastFMScrobble)) |f| {
+                        try value.put(f.name, @as(f.type, @field(formatted_scrobble, f.name)));
+                    }
+                    // Note sure why this works for ZMPL, but not for jobs.
+                    try scrobbles_view.append(formatted_scrobble);
+                }
+            },
+            else => unreachable,
+        }
         try job.schedule();
+        std.log.debug("Skipped {} tracks", .{skipped_tracks});
+        std.log.debug("Filtered {} tracks", .{limited_tracks});
     }
 
     var upload_table = try root.put("upload_table", .array);

src/app/views/upload/index.zmpl

@@ -14,9 +14,10 @@
   <input type="file" name="upload" />
   <input type="submit" value="Submit" />
   <fieldset>
-  <input type="radio" name="t" label="Last.fm" value="0">Last.fm</input>
+  <input type="radio" name="t" label="Last.fm" value="0" required>Last.fm</input>
-  <input type="radio" name="t" label="Spotify" value="1">Spotify</input>
+  <input type="radio" name="t" label="Spotify" value="1" required>Spotify</input>
-  Upload Scrobbles after: <input type="datetime-local" name="l" label="date"></input>
+  <input type="checkbox" name="bbool" id="bbool" value="false" onclick="document.getElementById('abool').setAttribute('required', 'required')"></input>Limit to Scrobbles before: <input type="datetime-local" name="b" label="date-before"></input>
+  <input type="checkbox" name="abool" id="abool" value="false" onclick="document.getElementById('bbool').setAttribute('required', 'required')"></input>Limit to Scrobbles after: <input type="datetime-local" name="a" label="date-after"></input>
   </fieldset>
 </form>
 </body>

src/main.zig

@@ -86,36 +86,6 @@ pub const jetzig_options = struct {
 };
 
 pub fn main() !void {
-    //var db = try sqlite.Db.init(.{
-    //    .mode = sqlite.Db.Mode{ .File = "/home/swebb/Source/zuletzt/src/app/database/data.db" },
-    //    .open_flags = .{
-    //        .write = true,
-    //        .create = true,
-    //    },
-    //    .threading_mode = .MultiThread,
-    //});
-
-    //const create =
-    //    \\CREATE TABLE artists ('artist', 'plays')
-    //;
-
-    //const query =
-    //    \\INSERT INTO artists ('artist', 'plays') VALUES (?,?)
-    //;
-
-    //var build = try db.prepare(create);
-    //defer build.deinit();
-
-    //try build.exec(.{},.{});
-
-    //var stmt = try db.prepare(query);
-    //defer stmt.deinit();
-
-    //try stmt.exec(.{}, .{
-    //    .artist = "Wilco",
-    //    .plays = 2500,
-    //});
-
     var gpa = std.heap.GeneralPurposeAllocator(.{}){};
     defer std.debug.assert(gpa.deinit() == .ok);
     const allocator = gpa.allocator();

src/types.zig

@@ -10,34 +10,29 @@ pub const LastFMScrobble = struct {
 // From lastfmstats.com
 pub const LastFM = struct { username: []const u8, scrobbles: []LastFMScrobble };
 
+// I derived whether or not these values were optional from searching
+// the respective fields for null in Vim, so there may be some fields
+// that can be optional that I haven't run into yet
 pub const SpotifyScrobble = struct {
-    ts: []u8,
+    ts: []const u8,
-    username: []u8,
+    username: []const u8,
-    platform: []u8,
+    platform: []const u8,
     ms_played: u64,
-    conn_country: []u8,
+    conn_country: []const u8,
-    ip_addr_decrypted: []u8,
+    ip_addr_decrypted: ?[]const u8,
-    user_agent_decrypted: []u8,
+    user_agent_decrypted: ?[]const u8,
-    master_metadata_track_name: []u8,
+    master_metadata_track_name: ?[]const u8,
-    master_metadata_artist_name: []u8,
+    master_metadata_album_artist_name: ?[]const u8,
-    master_metadata_album_name: []u8,
+    master_metadata_album_album_name: ?[]const u8,
-    spotify_track_uri: []u8,
+    spotify_track_uri: ?[]const u8,
-    episode_name: []u8,
+    episode_name: ?[]const u8,
-    reason_start: []u8,
+    episode_show_name: ?[]const u8,
-    reason_end: []u8,
+    spotify_episode_uri: ?[]const u8,
+    reason_start: []const u8,
+    reason_end: ?[]const u8,
     shuffle: bool,
-    skipped: bool,
+    skipped: ?bool,
     offline: bool,
     offline_timestamp: u64,
-    incognito_mode: bool,
+    incognito_mode: ?bool,
-
-    pub fn scrobblize(self: *SpotifyScrobble) LastFMScrobble {
-        return LastFMScrobble{ .track = self.master_metadata_track_name, .artist = self.master_metadata_artist_name, .album = self.master_metadata_album_name, .date = try zeit.instant(.{ .source = .{ .iso8601 = self.ts } }).unixTimestamp() };
-    }
 };
-
-pub const Spotify = struct { scrobbles: []SpotifyScrobble };
-
-const UploadDataTag = enum { spotify, lastfm };
-
-pub const UploadData = union(UploadDataTag) { spotify: Spotify, lastfm: LastFM };