From 15edd58d3be22b78f109300241167f129aa3eef3 Mon Sep 17 00:00:00 2001 From: PlexSheep Date: Thu, 4 Apr 2024 22:46:25 +0200 Subject: [PATCH] early bench --- src/bench/builtin.rs | 3 +- src/bench/mod.rs | 22 +++++++--- src/bench/report.rs | 96 ++++++++++++++++++++++++++++++++++++++---- src/bin/bench/cli.rs | 23 +++++++--- src/game/mod.rs | 6 +-- src/solve/naive/mod.rs | 4 +- src/wlist/word.rs | 4 +- 7 files changed, 129 insertions(+), 29 deletions(-) diff --git a/src/bench/builtin.rs b/src/bench/builtin.rs index 7c8b313..b556758 100644 --- a/src/bench/builtin.rs +++ b/src/bench/builtin.rs @@ -18,8 +18,7 @@ where SL: Solver<'wl, WL>, SL: 'wl, { - fn build(wordlist: &'wl WL, solver: SL) -> crate::error::WResult { - let builder: GameBuilder<_> = Game::builder(wordlist); + fn build(wordlist: &'wl WL, solver: SL, builder: GameBuilder<'wl, WL>) -> crate::error::WResult { Ok(Self { wordlist, solver, diff --git a/src/bench/mod.rs b/src/bench/mod.rs index d2b1cc3..0cace90 100644 --- a/src/bench/mod.rs +++ b/src/bench/mod.rs @@ -1,4 +1,4 @@ -use std::fmt::Debug; +use std::fmt::{Debug, Display}; use libpt::log::debug; @@ -14,6 +14,9 @@ use report::*; #[cfg(feature = "builtin")] pub mod builtin; +/// Default amount of games to play for a [Benchmark] +pub const DEFAULT_N: usize = 50; + pub trait Benchmark<'wl, WL, SL>: Clone + Sized + Debug where WL: WordList, @@ -21,7 +24,11 @@ where SL: Solver<'wl, WL>, SL: 'wl, { - fn build(wordlist: &'wl WL, solver: SL) -> WResult; + fn build( + wordlist: &'wl WL, + solver: SL, + builder: GameBuilder<'wl, WL>, + ) -> crate::error::WResult; fn builder(&'wl self) -> &'wl GameBuilder<'wl, WL>; fn make_game(&'wl self) -> WResult> { Ok(self.builder().build()?) @@ -30,16 +37,21 @@ where fn play(&'wl self) -> WResult { self.solver().play(&mut self.make_game()?) } + // TODO: add some interface to get reports while the benchmark runs + // TODO: make the benchmark optionally multithreaded fn bench(&'wl self, n: usize) -> WResult { // PERF: it would be better to make this multithreaded - let part = n / 20; + let part = match n / 20 { + 0 => 19, + other => other, + }; let mut report = Report::new(); for i in 0..n { - // TODO: limit execution time for the following, perhaps async report.add(self.play()?); if i % part == part - 1 { - debug!("{}", report); + // TODO: add the report to the struct so that users can poll it to print the status + // TODO: update the report in the struct } } diff --git a/src/bench/report.rs b/src/bench/report.rs index dbc3a20..c38963d 100644 --- a/src/bench/report.rs +++ b/src/bench/report.rs @@ -1,11 +1,18 @@ -use chrono::{self, NaiveDateTime, NaiveTime, TimeDelta}; +use chrono::{self, Duration, NaiveDateTime, NaiveTime, TimeDelta}; +use libpt::log::debug; +use core::panic; use std::fmt::Display; +use std::ops::Div; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use crate::game::response::GuessResponse; +pub const WEIGHTING_SCORE: f64 = 0.9; +pub const WEIGHTING_TIME: f64 = 0.1; +pub const WEIGHTING_WIN: f64 = 0.0; + #[derive(Debug, Clone, PartialEq)] pub struct Report { data: Vec, @@ -34,12 +41,68 @@ impl Report { self.data.len() } - pub fn win_ratio(&self) -> usize { - todo!() + pub fn total_wins(&self) -> usize { + let mut wins: usize = 0; + self.data.iter().for_each(|d| { + if d.won() { + wins += 1; + } + }); + wins } - pub fn avg_score(&self) -> usize { - todo!() + pub fn avg_win(&self) -> f64 { + self.total_wins() as f64 / self.n() as f64 + } + + pub fn total_score(&self) -> usize { + let mut score: usize = 0; + self.data.iter().for_each(|d| score += d.step()); + score + } + + pub fn avg_score(&self) -> f64 { + self.total_score() as f64 / self.n() as f64 + } + + pub fn avg_time(&self) -> Option { + let av = self.benchtime()? / self.n() as i32; + Some(av) + } + + pub fn rating(&self) -> Option { + assert_eq!(WEIGHTING_WIN + WEIGHTING_SCORE + WEIGHTING_TIME, 1.0); + debug!( + "partial rating - score: {}", + WEIGHTING_SCORE * self.avg_score() + ); + debug!( + "partial rating - win: {}", + WEIGHTING_WIN * (1.0 - self.avg_win()) + ); + // FIXME: this is not normalized, which can lead to negative score + debug!( + "partial rating - time: {}", + WEIGHTING_TIME + * (1.0 + - (1_000_000_000.0 + / self + .avg_time()? + .num_nanoseconds() + .expect("took so many ns per game that an integer overflow occured") + as f64)) + ); + let r = WEIGHTING_SCORE * self.avg_score() + + WEIGHTING_WIN * (1.0 - self.avg_win()) + + WEIGHTING_TIME + * (1.0 + - (1_000_000_000.0 + / self + .avg_time()? + .num_nanoseconds() + .expect("took so many ns per game that an integer overflow occured") + as f64)); + Some(r) } /// finalize the record @@ -58,16 +121,33 @@ impl Report { pub fn finished(&self) -> bool { self.finished } + + pub fn benchtime(&self) -> Option { + self.benchtime + } } impl Display for Report { + /// Implement the [Display] trait + /// + /// # Panics + /// + /// This will panic if the [Report] is not finished fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !self.finished { + panic!("can only display finished reports"); + } write!( f, - "n: {}, win_ratio: {}, avg_score: {}", + "n: {}, win_ratio: {:.2}%, avg_score: {:.4} steps until finish, avgerage time per game: {}μs, \n\ + rating: {:.4}, full time until completion: {}ms + ", self.n(), - self.win_ratio(), - self.avg_score() + self.avg_win() * 100.0, + self.avg_score(), + self.avg_time().unwrap().num_microseconds().expect("overflow when converting to micrseconds"), + self.rating().unwrap(), + self.benchtime().unwrap().num_milliseconds() ) } } diff --git a/src/bin/bench/cli.rs b/src/bin/bench/cli.rs index c64f022..0eb4aed 100644 --- a/src/bin/bench/cli.rs +++ b/src/bin/bench/cli.rs @@ -2,10 +2,14 @@ // #![warn(missing_docs)] #![warn(missing_debug_implementations)] +use std::thread::sleep_ms; + use clap::Parser; use libpt::log::*; -use wordle_analyzer::solve::BuiltinSolverNames; +use wordle_analyzer::bench::builtin::BuiltinBenchmark; +use wordle_analyzer::bench::{Benchmark, DEFAULT_N}; +use wordle_analyzer::solve::{BuiltinSolverNames, Solver}; use wordle_analyzer::wlist::builtin::BuiltinWList; use wordle_analyzer::{self, game}; @@ -28,26 +32,31 @@ struct Cli { /// which solver to use #[arg(short, long, default_value_t = BuiltinSolverNames::default())] solver: BuiltinSolverNames, + /// how many games to play for the benchmark + #[arg(short, long, default_value_t = DEFAULT_N)] + n: usize, } fn main() -> anyhow::Result<()> { let cli = Cli::parse(); if cli.verbose { - Logger::build_mini(Some(Level::TRACE))?; + Logger::build_mini(Some(Level::DEBUG))?; } else { Logger::build_mini(Some(Level::INFO))?; } - debug!("dumping CLI: {:#?}", cli); + trace!("dumping CLI: {:#?}", cli); let wl = BuiltinWList::default(); - let _builder = game::Game::builder(&wl) + let builder = game::Game::builder(&wl) .length(cli.length) .max_steps(cli.max_steps) .precompute(cli.precompute); - let _solver = cli.solver.to_solver(&wl); - let _bench = cli.solver.to_solver(&wl); + let solver = cli.solver.to_solver(&wl); + let bench = BuiltinBenchmark::build(&wl, solver, builder)?; + trace!("{bench:#?}"); + let report = bench.bench(cli.n)?; - todo!(); + println!("{report}"); Ok(()) } diff --git a/src/game/mod.rs b/src/game/mod.rs index ea26110..592714c 100644 --- a/src/game/mod.rs +++ b/src/game/mod.rs @@ -2,7 +2,7 @@ use crate::error::*; use crate::wlist::word::{ManyWordDatas, ManyWordsRef, Word, WordData}; use crate::wlist::WordList; -use libpt::log::debug; +use libpt::log::{debug, trace}; pub mod response; use response::GuessResponse; @@ -56,7 +56,7 @@ impl<'wl, WL: WordList> Game<'wl, WL> { length, precompute, max_steps, - step: 1, + step: 0, solution, wordlist: wlist, finished: false, @@ -204,7 +204,7 @@ impl<'wl, WL: WordList> GameBuilder<'wl, WL> { /// build a [`Game`] with the stored configuration pub fn build(&'wl self) -> GameResult> { - debug!("{:#?}", self); + trace!("{:#?}", self); let game: Game = Game::build(self.length, self.precompute, self.max_steps, self.wordlist)?; Ok(game) diff --git a/src/solve/naive/mod.rs b/src/solve/naive/mod.rs index 655efcf..4f5c69f 100644 --- a/src/solve/naive/mod.rs +++ b/src/solve/naive/mod.rs @@ -1,4 +1,4 @@ -use libpt::log::{debug, info}; +use libpt::log::{debug, info, trace}; use crate::wlist::word::{ManyWordDatas, Word}; use crate::wlist::WordList; @@ -29,7 +29,7 @@ impl<'wl, WL: WordList> Solver<'wl, WL> for NaiveSolver<'wl, WL> { } } } - debug!("other chars: {:?}", other_chars); + trace!("other chars: {:?}", other_chars); let matches: ManyWordDatas = game .wordlist() .get_words_matching(pattern) diff --git a/src/wlist/word.rs b/src/wlist/word.rs index abb848e..9f1dc20 100644 --- a/src/wlist/word.rs +++ b/src/wlist/word.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::fmt::write; use std::hash::Hash; -use libpt::log::debug; +use libpt::log::{debug, trace}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -61,7 +61,7 @@ impl WordMap { let len = self.len(); let mut c: f64 = l_under_sigmoid * (0.5 + self.n_common() as f64 / len as f64); c *= 1e-7; - debug!(threshold = c); + trace!(threshold = c); c } pub fn inner(&self) -> &HashMap {