Use time filters instead of page switching

This change allows us to avoid the bug of only being able to
access up to 10000 tournaments before start.gg's API throws
an error.

Cargo.lock

@@ -626,6 +626,15 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
 
+[[package]]
+name = "itertools"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.9"
@@ -1149,6 +1158,7 @@ dependencies = [
  "cynic",
  "cynic-codegen",
  "dirs",
+ "itertools",
  "reqwest",
  "schema",
  "serde",

Cargo.toml

@@ -26,5 +26,8 @@ serde = "1.0"
 dirs = "5.0"
 sqlite = "0.31"
 
+# Other
+itertools = "0.12.0"
+
 [build-dependencies]
 cynic-codegen = "3.2"

src/queries/scalars.rs

@@ -39,6 +39,11 @@ impl Display for StringOrInt {
 #[repr(transparent)]
 pub struct VideogameId(pub u64);
 
+#[derive(cynic::Scalar, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[cynic(graphql_type = "ID")]
+#[repr(transparent)]
+pub struct TournamentId(pub u64);
+
 #[derive(cynic::Scalar, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[cynic(graphql_type = "ID")]
 #[repr(transparent)]

src/queries/tournament_events.rs

@@ -7,15 +7,12 @@ use schema::schema;
 
 #[derive(cynic::QueryVariables, Debug, Copy, Clone)]
 pub struct TournamentEventsVars<'a> {
-    // HACK: This should really be an optional variable, but there seems to be a
+    pub after_date: Timestamp,
-    // server-side bug that completely breaks everything when this isn't passed.
+    pub before_date: Timestamp,
-    // We can use a dummy value of 1 when we don't want to filter by time.
-    pub last_sync: Timestamp,
 
     pub game_id: VideogameId,
     pub country: Option<&'a str>,
     pub state: Option<&'a str>,
-    pub page: i32,
 }
 
 // Query
@@ -24,12 +21,13 @@ pub struct TournamentEventsVars<'a> {
 #[cynic(graphql_type = "Query", variables = "TournamentEventsVars")]
 pub struct TournamentEvents {
     #[arguments(query: {
-        page: $page,
+        page: 1,
         perPage: 225,
-        sortBy: "endAt asc",
+        sortBy: "startAt asc",
         filter: {
             past: true,
-            afterDate: $last_sync,
+            afterDate: $after_date,
+            beforeDate: $before_date,
             videogameIds: [$game_id],
             countryCode: $country,
             addrState: $state
@@ -40,19 +38,15 @@ pub struct TournamentEvents {
 #[derive(cynic::QueryFragment, Debug)]
 #[cynic(variables = "TournamentEventsVars")]
 struct TournamentConnection {
-    page_info: Option<PageInfo>,
     #[cynic(flatten)]
     nodes: Vec<Tournament>,
 }
 
-#[derive(cynic::QueryFragment, Debug)]
-struct PageInfo {
-    total_pages: Option<i32>,
-}
-
 #[derive(cynic::QueryFragment, Debug)]
 #[cynic(variables = "TournamentEventsVars")]
 struct Tournament {
+    id: Option<TournamentId>,
+    start_at: Option<Timestamp>,
     #[arguments(limit: 99999, filter: { videogameId: [$game_id] })]
     #[cynic(flatten)]
     events: Vec<Event>,
@@ -68,14 +62,10 @@ struct Event {
 
 // Unwrap
 
-#[derive(Debug, Clone)]
-pub struct TournamentEventResponse {
-    pub pages: i32,
-    pub tournaments: Vec<TournamentData>,
-}
-
 #[derive(Debug, Clone)]
 pub struct TournamentData {
+    pub id: TournamentId,
+    pub time: Timestamp,
     pub events: Vec<EventData>,
 }
 
@@ -87,36 +77,33 @@ pub struct EventData {
 }
 
 impl<'a> QueryUnwrap<TournamentEventsVars<'a>> for TournamentEvents {
-    type Unwrapped = TournamentEventResponse;
+    type Unwrapped = Vec<TournamentData>;
 
-    fn unwrap_response(
+    fn unwrap_response(response: GraphQlResponse<TournamentEvents>) -> Option<Vec<TournamentData>> {
-        response: GraphQlResponse<TournamentEvents>,
-    ) -> Option<TournamentEventResponse> {
         let response_tournaments = response.data?.tournaments?;
 
-        let tournaments = response_tournaments
+        Some(
-            .nodes
+            response_tournaments
-            .into_iter()
+                .nodes
-            .filter_map(|tour| {
+                .into_iter()
-                Some(TournamentData {
+                .filter_map(|tour| {
-                    events: tour
+                    Some(TournamentData {
-                        .events
+                        id: tour.id?,
-                        .into_iter()
+                        time: tour.start_at?,
-                        .filter_map(|event| {
+                        events: tour
-                            Some(EventData {
+                            .events
-                                id: event.id?,
+                            .into_iter()
-                                slug: event.slug?,
+                            .filter_map(|event| {
-                                time: event.start_at?,
+                                Some(EventData {
+                                    id: event.id?,
+                                    slug: event.slug?,
+                                    time: event.start_at?,
+                                })
                             })
-                        })
+                            .collect(),
-                        .collect(),
+                    })
                 })
-            })
+                .collect::<Vec<_>>(),
-            .collect::<Vec<_>>();
+        )
-
-        Some(TournamentEventResponse {
-            pages: response_tournaments.page_info?.total_pages?,
-            tournaments,
-        })
     }
 }

src/sync.rs

@@ -5,6 +5,7 @@ use std::time::Duration;
 use crate::database::*;
 use crate::error;
 use crate::queries::*;
+use itertools::Itertools;
 use sqlite::*;
 
 // Glicko-2 system calculation
@@ -118,62 +119,72 @@ fn get_event_sets(event: EventId, auth: &str) -> Option<Vec<SetData>> {
     }
 }
 
-fn get_tournament_events(metadata: &DatasetMetadata, auth: &str) -> Option<Vec<EventData>> {
+fn get_tournament_events(
+    metadata: &DatasetMetadata,
+    current_time: Timestamp,
+    auth: &str,
+) -> Option<Vec<EventData>> {
     println!("Accessing tournaments...");
 
+    let mut after = metadata.last_sync;
+
     let tour_response = run_query::<TournamentEvents, _>(
         TournamentEventsVars {
-            last_sync: metadata.last_sync,
+            after_date: after,
+            before_date: current_time,
             game_id: metadata.game_id,
             country: metadata.country.as_deref(),
             state: metadata.state.as_deref(),
-            page: 1,
         },
         auth,
     )?;
 
-    let pages = tour_response.pages;
+    let mut cont = !tour_response.is_empty();
-    if pages == 0 {
+    after = if tour_response.iter().any(|tour| tour.time != after) {
-        Some(vec![])
+        tour_response.last().unwrap().time
-    } else if pages == 1 {
-        Some(
-            tour_response
-                .tournaments
-                .into_iter()
-                .flat_map(|tour| tour.events)
-                .collect::<Vec<_>>(),
-        )
     } else {
-        let mut tournaments = tour_response
+        Timestamp(after.0 + 1)
-            .tournaments
+    };
-            .into_iter()
-            .flat_map(|tour| tour.events)
-            .collect::<Vec<_>>();
 
-        for page in 2..=pages {
+    let mut tournaments = tour_response;
-            println!("  (Page {})", page);
 
-            let next_response = run_query::<TournamentEvents, _>(
+    let mut page: u64 = 1;
-                TournamentEventsVars {
+    while cont {
-                    last_sync: metadata.last_sync,
+        page += 1;
-                    game_id: metadata.game_id,
+        println!("  (Page {})", page);
-                    country: metadata.country.as_deref(),
-                    state: metadata.state.as_deref(),
-                    page,
-                },
-                auth,
-            )?;
 
-            tournaments.extend(
+        let next_response = run_query::<TournamentEvents, _>(
-                next_response
+            TournamentEventsVars {
-                    .tournaments
+                after_date: after,
-                    .into_iter()
+                before_date: current_time,
-                    .flat_map(|tour| tour.events),
+                game_id: metadata.game_id,
-            );
+                country: metadata.country.as_deref(),
-        }
+                state: metadata.state.as_deref(),
+            },
+            auth,
+        )?;
 
-        Some(tournaments)
+        cont = !next_response.is_empty();
+        after = if next_response.iter().any(|tour| tour.time != after) {
+            next_response.last().unwrap().time
+        } else {
+            Timestamp(after.0 + 1)
+        };
+
+        tournaments.extend(next_response);
     }
+
+    println!("Deduplicating...");
+
+    Some(
+        tournaments
+            .into_iter()
+            .group_by(|tour| tour.time)
+            .into_iter()
+            .flat_map(|(_, group)| group.into_iter().unique_by(|tour| tour.id))
+            .flat_map(|tour| tour.events)
+            .collect::<Vec<_>>(),
+    )
 }
 
 // Dataset syncing
@@ -292,7 +303,7 @@ pub fn sync_dataset(
     current_time: Timestamp,
     auth: &str,
 ) -> sqlite::Result<()> {
-    let events = get_tournament_events(&metadata, auth)
+    let events = get_tournament_events(&metadata, current_time, auth)
         .unwrap_or_else(|| error("Could not access start.gg", 1));
 
     connection.execute("BEGIN;")?;