diff --git a/VSharp.ML.AIAgent/config.py b/VSharp.ML.AIAgent/config.py index 7c7bd5aaa..7c18a410a 100644 --- a/VSharp.ML.AIAgent/config.py +++ b/VSharp.ML.AIAgent/config.py @@ -34,6 +34,18 @@ def create_save_path_if_not_exists(self): self.save_path.mkdir() +@dataclass(slots=True, frozen=True) +class SaveEpochsCoveragesFeature: + enabled: bool + save_path: Path + + def create_save_path_if_not_exists(self): + if self.enabled: + if self.save_path.exists(): + rmtree(self.save_path) + self.save_path.mkdir() + + class FeatureConfig: VERBOSE_TABLES = True SHOW_SUCCESSORS = True @@ -43,3 +55,6 @@ class FeatureConfig: DUMP_BY_TIMEOUT = DumpByTimeoutFeature( enabled=True, timeout_seconds=1200, save_path=Path("./report/timeouted_agents/") ) + SAVE_EPOCHS_COVERAGES = SaveEpochsCoveragesFeature( + enabled=True, save_path=Path("./report/epochs_tables/") + ) diff --git a/VSharp.ML.AIAgent/epochs_statistics/tables.py b/VSharp.ML.AIAgent/epochs_statistics/tables.py index 08e97f61b..b47d1b852 100644 --- a/VSharp.ML.AIAgent/epochs_statistics/tables.py +++ b/VSharp.ML.AIAgent/epochs_statistics/tables.py @@ -1,4 +1,5 @@ from collections import defaultdict +from pathlib import Path import pandas as pd @@ -11,7 +12,7 @@ from config import FeatureConfig from epochs_statistics.common import Interval, Name2ResultViewModel from epochs_statistics.gen_stats import compute_euc_dist_to_full_coverage -from selection.classes import AgentResultsOnGameMaps, Agent2Result +from selection.classes import Agent2Result, AgentResultsOnGameMaps from selection.utils import invert_mapping_mrgm_gmmr @@ -43,20 +44,27 @@ def create_stats( def create_pivot_table( model_map_results_mapping: AgentResultsOnGameMaps, sort: bool = True -) -> pd.DataFrame: +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: map_results_with_models = invert_mapping_mrgm_gmmr(model_map_results_mapping) euc_dists2full_cov, avs, medians, intervals = create_stats( model_map_results_mapping ) name_results_dict: defaultdict[int, list[Name2ResultViewModel]] = defaultdict(list) + epoch_percents_dict: defaultdict[int, list[Name2ResultViewModel]] = defaultdict( + list + ) for map_obj, mutable2result_list in map_results_with_models.items(): for mutable2result in mutable2result_list: name_results_dict[map_obj.Id].append(convert_to_view_model(mutable2result)) + epoch_percents_dict[map_obj.Id].append( + mutable2result.game_result.actual_coverage_percent + ) mutable_names = get_model_names_in_order(name_results_dict) df = pd.DataFrame(name_results_dict, index=mutable_names) + epochs_percent_df = pd.DataFrame(epoch_percents_dict, index=mutable_names) for col in df: df[col] = df[col].map(lambda name2result_vm: name2result_vm.pretty_result) @@ -65,6 +73,7 @@ def create_pivot_table( ) df.rename(columns=lambda map_id: maps_indexes[map_id], inplace=True) + epochs_percent_df.rename(columns=lambda map_id: maps_indexes[map_id], inplace=True) df[EUC_DIST2FULL_COV_COL_NAME] = euc_dists2full_cov df[AV_COVERAGE_COL_NAME] = avs df[MEDIAN_COVERAGE_COL_NAME] = medians @@ -84,13 +93,17 @@ def create_pivot_table( df.drop([AV_COVERAGE_COL_NAME], axis=1, inplace=True) df.drop([MEDIAN_COVERAGE_COL_NAME], axis=1, inplace=True) df.drop([COV_DEVIATION_COL_NAME], axis=1, inplace=True) - return df, stats_df + return df, stats_df, epochs_percent_df def table_to_string(table: pd.DataFrame): return table.to_markdown(tablefmt="psql") +def table_to_csv(table: pd.DataFrame, path: Path): + table.to_csv(path_or_buf=path) + + def convert_to_view_model( m2r_mapping: Agent2Result, ) -> Name2ResultViewModel: diff --git a/VSharp.ML.AIAgent/learning/genetic_alorithm.py b/VSharp.ML.AIAgent/learning/genetic_alorithm.py index 3565602f8..7c4aac99b 100644 --- a/VSharp.ML.AIAgent/learning/genetic_alorithm.py +++ b/VSharp.ML.AIAgent/learning/genetic_alorithm.py @@ -60,6 +60,7 @@ def run( init_log_file() init_epochs_best_dir() FeatureConfig.DUMP_BY_TIMEOUT.create_save_path_if_not_exists() + FeatureConfig.SAVE_EPOCHS_COVERAGES.create_save_path_if_not_exists() ga_instance = pygad.GA( num_generations=num_generations, diff --git a/VSharp.ML.AIAgent/learning/r_learn.py b/VSharp.ML.AIAgent/learning/r_learn.py index f06193493..64a9a579c 100644 --- a/VSharp.ML.AIAgent/learning/r_learn.py +++ b/VSharp.ML.AIAgent/learning/r_learn.py @@ -20,7 +20,7 @@ from conn.classes import Agent2ResultsOnMaps from conn.requests import recv_game_result_list, send_game_results from conn.socket_manager import game_server_socket_manager -from epochs_statistics.tables import create_pivot_table, table_to_string +from epochs_statistics.tables import create_pivot_table, table_to_csv, table_to_string from epochs_statistics.utils import ( append_to_tables_file, create_epoch_subdir, @@ -190,10 +190,16 @@ def on_generation(ga_instance): else: leader_table[best_solution_nnwrapper] = best_solution_results - _, stats = create_pivot_table(leader_table, sort=False) + _, stats, _ = create_pivot_table(leader_table, sort=False) rewrite_best_tables_file(table_to_string(stats) + "\n") - pivot, stats = create_pivot_table(info_for_tables_filtered) + pivot, stats, epoch_table = create_pivot_table(info_for_tables_filtered) + if FeatureConfig.SAVE_EPOCHS_COVERAGES.enabled: + path_to_save_to = ( + FeatureConfig.SAVE_EPOCHS_COVERAGES.save_path + / f"{ga_instance.generations_completed}.csv" + ) + table_to_csv(epoch_table, path=path_to_save_to) append_to_tables_file(table_to_string(pivot) + "\n") append_to_tables_file(table_to_string(stats) + "\n") mean, std = compute_statistics(load_times_array())