early bench

src/bench/builtin.rs

@@ -18,8 +18,7 @@ where
     SL: Solver<'wl, WL>,
     SL: 'wl,
 {
-    fn build(wordlist: &'wl WL, solver: SL) -> crate::error::WResult<Self> {
+    fn build(wordlist: &'wl WL, solver: SL, builder: GameBuilder<'wl, WL>) -> crate::error::WResult<Self> {
-        let builder: GameBuilder<_> = Game::builder(wordlist);
         Ok(Self {
             wordlist,
             solver,

src/bench/mod.rs

@@ -1,4 +1,4 @@
-use std::fmt::Debug;
+use std::fmt::{Debug, Display};
 
 use libpt::log::debug;
 
@@ -14,6 +14,9 @@ use report::*;
 #[cfg(feature = "builtin")]
 pub mod builtin;
 
+/// Default amount of games to play for a [Benchmark]
+pub const DEFAULT_N: usize = 50;
+
 pub trait Benchmark<'wl, WL, SL>: Clone + Sized + Debug
 where
     WL: WordList,
@@ -21,7 +24,11 @@ where
     SL: Solver<'wl, WL>,
     SL: 'wl,
 {
-    fn build(wordlist: &'wl WL, solver: SL) -> WResult<Self>;
+    fn build(
+        wordlist: &'wl WL,
+        solver: SL,
+        builder: GameBuilder<'wl, WL>,
+    ) -> crate::error::WResult<Self>;
     fn builder(&'wl self) -> &'wl GameBuilder<'wl, WL>;
     fn make_game(&'wl self) -> WResult<Game<'wl, WL>> {
         Ok(self.builder().build()?)
@@ -30,16 +37,21 @@ where
     fn play(&'wl self) -> WResult<GuessResponse> {
         self.solver().play(&mut self.make_game()?)
     }
+    // TODO: add some interface to get reports while the benchmark runs
+    // TODO: make the benchmark optionally multithreaded
     fn bench(&'wl self, n: usize) -> WResult<Report> {
         // PERF: it would be better to make this multithreaded
-        let part = n / 20;
+        let part = match n / 20 {
+            0 => 19,
+            other => other,
+        };
         let mut report = Report::new();
 
         for i in 0..n {
-            // TODO: limit execution time for the following, perhaps async
             report.add(self.play()?);
             if i % part == part - 1 {
-                debug!("{}", report);
+                // TODO: add the report to the struct so that users can poll it to print the status
+                // TODO: update the report in the struct
             }
         }
 

src/bench/report.rs

@@ -1,11 +1,18 @@
-use chrono::{self, NaiveDateTime, NaiveTime, TimeDelta};
+use chrono::{self, Duration, NaiveDateTime, NaiveTime, TimeDelta};
+use libpt::log::debug;
+use core::panic;
 use std::fmt::Display;
+use std::ops::Div;
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
 use crate::game::response::GuessResponse;
 
+pub const WEIGHTING_SCORE: f64 = 0.9;
+pub const WEIGHTING_TIME: f64 = 0.1;
+pub const WEIGHTING_WIN: f64 = 0.0;
+
 #[derive(Debug, Clone, PartialEq)]
 pub struct Report {
     data: Vec<GuessResponse>,
@@ -34,12 +41,68 @@ impl Report {
         self.data.len()
     }
 
-    pub fn win_ratio(&self) -> usize {
+    pub fn total_wins(&self) -> usize {
-        todo!()
+        let mut wins: usize = 0;
+        self.data.iter().for_each(|d| {
+            if d.won() {
+                wins += 1;
+            }
+        });
+        wins
     }
 
-    pub fn avg_score(&self) -> usize {
+    pub fn avg_win(&self) -> f64 {
-        todo!()
+        self.total_wins() as f64 / self.n() as f64
+    }
+
+    pub fn total_score(&self) -> usize {
+        let mut score: usize = 0;
+        self.data.iter().for_each(|d| score += d.step());
+        score
+    }
+
+    pub fn avg_score(&self) -> f64 {
+        self.total_score() as f64 / self.n() as f64
+    }
+
+    pub fn avg_time(&self) -> Option<TimeDelta> {
+        let av = self.benchtime()? / self.n() as i32;
+        Some(av)
+    }
+
+    pub fn rating(&self) -> Option<f64> {
+        assert_eq!(WEIGHTING_WIN + WEIGHTING_SCORE + WEIGHTING_TIME, 1.0);
+        debug!(
+            "partial rating - score: {}",
+            WEIGHTING_SCORE * self.avg_score()
+        );
+        debug!(
+            "partial rating - win: {}",
+            WEIGHTING_WIN * (1.0 - self.avg_win())
+        );
+        // FIXME: this is not normalized, which can lead to negative score
+        debug!(
+            "partial rating - time: {}",
+            WEIGHTING_TIME
+                * (1.0
+                    - (1_000_000_000.0
+                        / self
+                            .avg_time()?
+                            .num_nanoseconds()
+                            .expect("took so many ns per game that an integer overflow occured")
+                            as f64))
+        );
+        let r = WEIGHTING_SCORE * self.avg_score()
+            + WEIGHTING_WIN * (1.0 - self.avg_win())
+            + WEIGHTING_TIME
+                * (1.0
+                    - (1_000_000_000.0
+                        / self
+                            .avg_time()?
+                            .num_nanoseconds()
+                            .expect("took so many ns per game that an integer overflow occured")
+                            as f64));
+        Some(r)
     }
 
     /// finalize the record
@@ -58,16 +121,33 @@ impl Report {
     pub fn finished(&self) -> bool {
         self.finished
     }
+
+    pub fn benchtime(&self) -> Option<TimeDelta> {
+        self.benchtime
+    }
 }
 
 impl Display for Report {
+    /// Implement the [Display] trait
+    ///
+    /// # Panics
+    ///
+    /// This will panic if the [Report] is not finished
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if !self.finished {
+            panic!("can only display finished reports");
+        }
         write!(
             f,
-            "n: {}, win_ratio: {}, avg_score: {}",
+            "n: {}, win_ratio: {:.2}%, avg_score: {:.4} steps until finish, avgerage time per game: {}μs, \n\
+            rating: {:.4}, full time until completion: {}ms
+            ",
             self.n(),
-            self.win_ratio(),
+            self.avg_win() * 100.0,
-            self.avg_score()
+            self.avg_score(),
+            self.avg_time().unwrap().num_microseconds().expect("overflow when converting to micrseconds"),
+            self.rating().unwrap(),
+            self.benchtime().unwrap().num_milliseconds()
         )
     }
 }

src/bin/bench/cli.rs

@@ -2,10 +2,14 @@
 // #![warn(missing_docs)]
 #![warn(missing_debug_implementations)]
 
+use std::thread::sleep_ms;
+
 use clap::Parser;
 use libpt::log::*;
 
-use wordle_analyzer::solve::BuiltinSolverNames;
+use wordle_analyzer::bench::builtin::BuiltinBenchmark;
+use wordle_analyzer::bench::{Benchmark, DEFAULT_N};
+use wordle_analyzer::solve::{BuiltinSolverNames, Solver};
 use wordle_analyzer::wlist::builtin::BuiltinWList;
 
 use wordle_analyzer::{self, game};
@@ -28,26 +32,31 @@ struct Cli {
     /// which solver to use
     #[arg(short, long, default_value_t = BuiltinSolverNames::default())]
     solver: BuiltinSolverNames,
+    /// how many games to play for the benchmark
+    #[arg(short, long, default_value_t = DEFAULT_N)]
+    n: usize,
 }
 
 fn main() -> anyhow::Result<()> {
     let cli = Cli::parse();
     if cli.verbose {
-        Logger::build_mini(Some(Level::TRACE))?;
+        Logger::build_mini(Some(Level::DEBUG))?;
     } else {
         Logger::build_mini(Some(Level::INFO))?;
     }
-    debug!("dumping CLI: {:#?}", cli);
+    trace!("dumping CLI: {:#?}", cli);
 
     let wl = BuiltinWList::default();
-    let _builder = game::Game::builder(&wl)
+    let builder = game::Game::builder(&wl)
         .length(cli.length)
         .max_steps(cli.max_steps)
         .precompute(cli.precompute);
-    let _solver = cli.solver.to_solver(&wl);
+    let solver = cli.solver.to_solver(&wl);
-    let _bench = cli.solver.to_solver(&wl);
+    let bench = BuiltinBenchmark::build(&wl, solver, builder)?;
+    trace!("{bench:#?}");
+    let report = bench.bench(cli.n)?;
 
-    todo!();
+    println!("{report}");
 
     Ok(())
 }

src/game/mod.rs

@@ -2,7 +2,7 @@ use crate::error::*;
 use crate::wlist::word::{ManyWordDatas, ManyWordsRef, Word, WordData};
 use crate::wlist::WordList;
 
-use libpt::log::debug;
+use libpt::log::{debug, trace};
 
 pub mod response;
 use response::GuessResponse;
@@ -56,7 +56,7 @@ impl<'wl, WL: WordList> Game<'wl, WL> {
             length,
             precompute,
             max_steps,
-            step: 1,
+            step: 0,
             solution,
             wordlist: wlist,
             finished: false,
@@ -204,7 +204,7 @@ impl<'wl, WL: WordList> GameBuilder<'wl, WL> {
 
     /// build a [`Game`] with the stored configuration
     pub fn build(&'wl self) -> GameResult<Game<'wl, WL>> {
-        debug!("{:#?}", self);
+        trace!("{:#?}", self);
         let game: Game<WL> =
             Game::build(self.length, self.precompute, self.max_steps, self.wordlist)?;
         Ok(game)

src/solve/naive/mod.rs

@@ -1,4 +1,4 @@
-use libpt::log::{debug, info};
+use libpt::log::{debug, info, trace};
 
 use crate::wlist::word::{ManyWordDatas, Word};
 use crate::wlist::WordList;
@@ -29,7 +29,7 @@ impl<'wl, WL: WordList> Solver<'wl, WL> for NaiveSolver<'wl, WL> {
                 }
             }
         }
-        debug!("other chars: {:?}", other_chars);
+        trace!("other chars: {:?}", other_chars);
         let matches: ManyWordDatas = game
             .wordlist()
             .get_words_matching(pattern)

src/wlist/word.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 use std::fmt::write;
 use std::hash::Hash;
 
-use libpt::log::debug;
+use libpt::log::{debug, trace};
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
@@ -61,7 +61,7 @@ impl WordMap {
         let len = self.len();
         let mut c: f64 = l_under_sigmoid * (0.5 + self.n_common() as f64 / len as f64);
         c *= 1e-7;
-        debug!(threshold = c);
+        trace!(threshold = c);
         c
     }
     pub fn inner(&self) -> &HashMap<Word, Frequency> {