Use time filters instead of page switching

This change allows us to avoid the bug of only being able to
access up to 10000 tournaments before start.gg's API throws
an error.
This commit is contained in:
Kiana Sheibani 2023-12-01 18:32:55 -05:00
parent 504184e69b
commit b3ff055fd3
Signed by: toki
GPG key ID: 6CB106C25E86A9F7
5 changed files with 102 additions and 86 deletions

10
Cargo.lock generated
View file

@ -626,6 +626,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
[[package]]
name = "itertools"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0"
dependencies = [
"either",
]
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.9" version = "1.0.9"
@ -1149,6 +1158,7 @@ dependencies = [
"cynic", "cynic",
"cynic-codegen", "cynic-codegen",
"dirs", "dirs",
"itertools",
"reqwest", "reqwest",
"schema", "schema",
"serde", "serde",

View file

@ -26,5 +26,8 @@ serde = "1.0"
dirs = "5.0" dirs = "5.0"
sqlite = "0.31" sqlite = "0.31"
# Other
itertools = "0.12.0"
[build-dependencies] [build-dependencies]
cynic-codegen = "3.2" cynic-codegen = "3.2"

View file

@ -39,6 +39,11 @@ impl Display for StringOrInt {
#[repr(transparent)] #[repr(transparent)]
pub struct VideogameId(pub u64); pub struct VideogameId(pub u64);
#[derive(cynic::Scalar, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[cynic(graphql_type = "ID")]
#[repr(transparent)]
pub struct TournamentId(pub u64);
#[derive(cynic::Scalar, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(cynic::Scalar, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[cynic(graphql_type = "ID")] #[cynic(graphql_type = "ID")]
#[repr(transparent)] #[repr(transparent)]

View file

@ -7,15 +7,12 @@ use schema::schema;
#[derive(cynic::QueryVariables, Debug, Copy, Clone)] #[derive(cynic::QueryVariables, Debug, Copy, Clone)]
pub struct TournamentEventsVars<'a> { pub struct TournamentEventsVars<'a> {
// HACK: This should really be an optional variable, but there seems to be a pub after_date: Timestamp,
// server-side bug that completely breaks everything when this isn't passed. pub before_date: Timestamp,
// We can use a dummy value of 1 when we don't want to filter by time.
pub last_sync: Timestamp,
pub game_id: VideogameId, pub game_id: VideogameId,
pub country: Option<&'a str>, pub country: Option<&'a str>,
pub state: Option<&'a str>, pub state: Option<&'a str>,
pub page: i32,
} }
// Query // Query
@ -24,12 +21,13 @@ pub struct TournamentEventsVars<'a> {
#[cynic(graphql_type = "Query", variables = "TournamentEventsVars")] #[cynic(graphql_type = "Query", variables = "TournamentEventsVars")]
pub struct TournamentEvents { pub struct TournamentEvents {
#[arguments(query: { #[arguments(query: {
page: $page, page: 1,
perPage: 225, perPage: 225,
sortBy: "endAt asc", sortBy: "startAt asc",
filter: { filter: {
past: true, past: true,
afterDate: $last_sync, afterDate: $after_date,
beforeDate: $before_date,
videogameIds: [$game_id], videogameIds: [$game_id],
countryCode: $country, countryCode: $country,
addrState: $state addrState: $state
@ -40,19 +38,15 @@ pub struct TournamentEvents {
#[derive(cynic::QueryFragment, Debug)] #[derive(cynic::QueryFragment, Debug)]
#[cynic(variables = "TournamentEventsVars")] #[cynic(variables = "TournamentEventsVars")]
struct TournamentConnection { struct TournamentConnection {
page_info: Option<PageInfo>,
#[cynic(flatten)] #[cynic(flatten)]
nodes: Vec<Tournament>, nodes: Vec<Tournament>,
} }
#[derive(cynic::QueryFragment, Debug)]
struct PageInfo {
total_pages: Option<i32>,
}
#[derive(cynic::QueryFragment, Debug)] #[derive(cynic::QueryFragment, Debug)]
#[cynic(variables = "TournamentEventsVars")] #[cynic(variables = "TournamentEventsVars")]
struct Tournament { struct Tournament {
id: Option<TournamentId>,
start_at: Option<Timestamp>,
#[arguments(limit: 99999, filter: { videogameId: [$game_id] })] #[arguments(limit: 99999, filter: { videogameId: [$game_id] })]
#[cynic(flatten)] #[cynic(flatten)]
events: Vec<Event>, events: Vec<Event>,
@ -68,14 +62,10 @@ struct Event {
// Unwrap // Unwrap
#[derive(Debug, Clone)]
pub struct TournamentEventResponse {
pub pages: i32,
pub tournaments: Vec<TournamentData>,
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct TournamentData { pub struct TournamentData {
pub id: TournamentId,
pub time: Timestamp,
pub events: Vec<EventData>, pub events: Vec<EventData>,
} }
@ -87,36 +77,33 @@ pub struct EventData {
} }
impl<'a> QueryUnwrap<TournamentEventsVars<'a>> for TournamentEvents { impl<'a> QueryUnwrap<TournamentEventsVars<'a>> for TournamentEvents {
type Unwrapped = TournamentEventResponse; type Unwrapped = Vec<TournamentData>;
fn unwrap_response( fn unwrap_response(response: GraphQlResponse<TournamentEvents>) -> Option<Vec<TournamentData>> {
response: GraphQlResponse<TournamentEvents>,
) -> Option<TournamentEventResponse> {
let response_tournaments = response.data?.tournaments?; let response_tournaments = response.data?.tournaments?;
let tournaments = response_tournaments Some(
.nodes response_tournaments
.into_iter() .nodes
.filter_map(|tour| { .into_iter()
Some(TournamentData { .filter_map(|tour| {
events: tour Some(TournamentData {
.events id: tour.id?,
.into_iter() time: tour.start_at?,
.filter_map(|event| { events: tour
Some(EventData { .events
id: event.id?, .into_iter()
slug: event.slug?, .filter_map(|event| {
time: event.start_at?, Some(EventData {
id: event.id?,
slug: event.slug?,
time: event.start_at?,
})
}) })
}) .collect(),
.collect(), })
}) })
}) .collect::<Vec<_>>(),
.collect::<Vec<_>>(); )
Some(TournamentEventResponse {
pages: response_tournaments.page_info?.total_pages?,
tournaments,
})
} }
} }

View file

@ -5,6 +5,7 @@ use std::time::Duration;
use crate::database::*; use crate::database::*;
use crate::error; use crate::error;
use crate::queries::*; use crate::queries::*;
use itertools::Itertools;
use sqlite::*; use sqlite::*;
// Glicko-2 system calculation // Glicko-2 system calculation
@ -118,62 +119,72 @@ fn get_event_sets(event: EventId, auth: &str) -> Option<Vec<SetData>> {
} }
} }
fn get_tournament_events(metadata: &DatasetMetadata, auth: &str) -> Option<Vec<EventData>> { fn get_tournament_events(
metadata: &DatasetMetadata,
current_time: Timestamp,
auth: &str,
) -> Option<Vec<EventData>> {
println!("Accessing tournaments..."); println!("Accessing tournaments...");
let mut after = metadata.last_sync;
let tour_response = run_query::<TournamentEvents, _>( let tour_response = run_query::<TournamentEvents, _>(
TournamentEventsVars { TournamentEventsVars {
last_sync: metadata.last_sync, after_date: after,
before_date: current_time,
game_id: metadata.game_id, game_id: metadata.game_id,
country: metadata.country.as_deref(), country: metadata.country.as_deref(),
state: metadata.state.as_deref(), state: metadata.state.as_deref(),
page: 1,
}, },
auth, auth,
)?; )?;
let pages = tour_response.pages; let mut cont = !tour_response.is_empty();
if pages == 0 { after = if tour_response.iter().any(|tour| tour.time != after) {
Some(vec![]) tour_response.last().unwrap().time
} else if pages == 1 {
Some(
tour_response
.tournaments
.into_iter()
.flat_map(|tour| tour.events)
.collect::<Vec<_>>(),
)
} else { } else {
let mut tournaments = tour_response Timestamp(after.0 + 1)
.tournaments };
.into_iter()
.flat_map(|tour| tour.events)
.collect::<Vec<_>>();
for page in 2..=pages { let mut tournaments = tour_response;
println!(" (Page {})", page);
let next_response = run_query::<TournamentEvents, _>( let mut page: u64 = 1;
TournamentEventsVars { while cont {
last_sync: metadata.last_sync, page += 1;
game_id: metadata.game_id, println!(" (Page {})", page);
country: metadata.country.as_deref(),
state: metadata.state.as_deref(),
page,
},
auth,
)?;
tournaments.extend( let next_response = run_query::<TournamentEvents, _>(
next_response TournamentEventsVars {
.tournaments after_date: after,
.into_iter() before_date: current_time,
.flat_map(|tour| tour.events), game_id: metadata.game_id,
); country: metadata.country.as_deref(),
} state: metadata.state.as_deref(),
},
auth,
)?;
Some(tournaments) cont = !next_response.is_empty();
after = if next_response.iter().any(|tour| tour.time != after) {
next_response.last().unwrap().time
} else {
Timestamp(after.0 + 1)
};
tournaments.extend(next_response);
} }
println!("Deduplicating...");
Some(
tournaments
.into_iter()
.group_by(|tour| tour.time)
.into_iter()
.flat_map(|(_, group)| group.into_iter().unique_by(|tour| tour.id))
.flat_map(|tour| tour.events)
.collect::<Vec<_>>(),
)
} }
// Dataset syncing // Dataset syncing
@ -292,7 +303,7 @@ pub fn sync_dataset(
current_time: Timestamp, current_time: Timestamp,
auth: &str, auth: &str,
) -> sqlite::Result<()> { ) -> sqlite::Result<()> {
let events = get_tournament_events(&metadata, auth) let events = get_tournament_events(&metadata, current_time, auth)
.unwrap_or_else(|| error("Could not access start.gg", 1)); .unwrap_or_else(|| error("Could not access start.gg", 1));
connection.execute("BEGIN;")?; connection.execute("BEGIN;")?;