From 007e307cc555c627fd78546a6ebfdae2c1247411 Mon Sep 17 00:00:00 2001 From: Max Nigmatulin Date: Thu, 20 Jul 2023 19:28:37 +0300 Subject: [PATCH 1/2] Add WeightedPredictor ABC, rework learning module --- .../{genetic_alorithm.py => entry_point.py} | 2 +- .../learning/genetic_learning.py | 146 ++++++++++ VSharp.ML.AIAgent/learning/play_game.py | 141 ++++++++++ VSharp.ML.AIAgent/learning/r_learn.py | 265 ------------------ VSharp.ML.AIAgent/main.py | 2 +- .../ml/model_wrappers/last_layer_learner.py | 8 +- .../ml/model_wrappers/nnwrapper.py | 21 +- .../ml/model_wrappers/protocols.py | 13 +- VSharp.ML.AIAgent/selection/scorer.py | 2 +- 9 files changed, 315 insertions(+), 285 deletions(-) rename VSharp.ML.AIAgent/learning/{genetic_alorithm.py => entry_point.py} (97%) create mode 100644 VSharp.ML.AIAgent/learning/genetic_learning.py create mode 100644 VSharp.ML.AIAgent/learning/play_game.py delete mode 100644 VSharp.ML.AIAgent/learning/r_learn.py diff --git a/VSharp.ML.AIAgent/learning/genetic_alorithm.py b/VSharp.ML.AIAgent/learning/entry_point.py similarity index 97% rename from VSharp.ML.AIAgent/learning/genetic_alorithm.py rename to VSharp.ML.AIAgent/learning/entry_point.py index 7c4aac99b..1d511d63f 100644 --- a/VSharp.ML.AIAgent/learning/genetic_alorithm.py +++ b/VSharp.ML.AIAgent/learning/entry_point.py @@ -29,7 +29,7 @@ os.environ["NUMEXPR_NUM_THREADS"] = "1" -from .r_learn import fitness_function, on_generation +from .genetic_learning import fitness_function, on_generation @contextmanager diff --git a/VSharp.ML.AIAgent/learning/genetic_learning.py b/VSharp.ML.AIAgent/learning/genetic_learning.py new file mode 100644 index 000000000..d233c22a6 --- /dev/null +++ b/VSharp.ML.AIAgent/learning/genetic_learning.py @@ -0,0 +1,146 @@ +import copy +import json +import random +from collections import defaultdict +from os import getpid + +import pygad.torchga + +import ml +from agent.utils import MapsType +from common.constants import DEVICE +from config import FeatureConfig, GeneralConfig +from conn.classes import Agent2ResultsOnMaps +from conn.requests import recv_game_result_list, send_game_results +from epochs_statistics.tables import create_pivot_table, table_to_csv, table_to_string +from epochs_statistics.utils import ( + append_to_tables_file, + create_epoch_subdir, + rewrite_best_tables_file, +) +from ml.fileop import save_model +from ml.model_wrappers.nnwrapper import NNWrapper +from selection.classes import AgentResultsOnGameMaps +from selection.scorer import straight_scorer +from timer.stats import compute_statistics +from timer.utils import ( + create_temp_epoch_inference_dir, + dump_and_reset_epoch_times, + load_times_array, +) + +from .play_game import play_game + +info_for_tables: AgentResultsOnGameMaps = defaultdict(list) +leader_table: AgentResultsOnGameMaps = defaultdict(list) + + +def get_n_best_weights_in_last_generation(ga_instance, n: int): + population = ga_instance.population + population_fitnesses = ga_instance.last_generation_fitness + + assert n <= len( + population + ), f"asked for {n} best when population size is {len(population)}" + + sorted_population = sorted( + zip(population, population_fitnesses), key=lambda x: x[1], reverse=True + ) + + return list(map(lambda x: x[0], sorted_population))[:n] + + +def on_generation(ga_instance): + game_results_raw = json.loads(recv_game_result_list()) + game_results_decoded = [ + Agent2ResultsOnMaps.from_json(item) for item in game_results_raw + ] + + for full_game_result in game_results_decoded: + info_for_tables[full_game_result.agent] = full_game_result.results + + print(f"Generation = {ga_instance.generations_completed};") + epoch_subdir = create_epoch_subdir(ga_instance.generations_completed) + + for weights in get_n_best_weights_in_last_generation( + ga_instance, FeatureConfig.N_BEST_SAVED_EACH_GEN + ): + save_model( + GeneralConfig.MODEL_INIT(), + to=epoch_subdir / f"{sum(weights)}.pth", + weights=weights, + ) + + ga_pop_inner_hashes = [ + tuple(individual).__hash__() for individual in ga_instance.population + ] + info_for_tables_filtered = { + nnwrapper: res + for nnwrapper, res in info_for_tables.items() + if nnwrapper.weights_hash in ga_pop_inner_hashes + } + + best_solution_hash = tuple( + ga_instance.best_solution(pop_fitness=ga_instance.last_generation_fitness)[0] + ).__hash__() + best_solution_nnwrapper, best_solution_results = next( + filter( + lambda item: item[0].weights_hash == best_solution_hash, + info_for_tables_filtered.items(), + ) + ) + + append_to_tables_file( + f"Generations completed: {ga_instance.generations_completed}" + "\n" + ) + + if best_solution_nnwrapper in leader_table.keys(): + best_wrapper_copy = copy.copy(best_solution_nnwrapper) + best_wrapper_copy.weights_hash += random.randint(0, 10**3) + leader_table[best_wrapper_copy] = best_solution_results + else: + leader_table[best_solution_nnwrapper] = best_solution_results + + _, stats, _ = create_pivot_table(leader_table, sort=False) + rewrite_best_tables_file(table_to_string(stats) + "\n") + + pivot, stats, epoch_table = create_pivot_table(info_for_tables_filtered) + if FeatureConfig.SAVE_EPOCHS_COVERAGES.enabled: + path_to_save_to = ( + FeatureConfig.SAVE_EPOCHS_COVERAGES.save_path + / f"{ga_instance.generations_completed}.csv" + ) + table_to_csv(epoch_table, path=path_to_save_to) + append_to_tables_file(table_to_string(pivot) + "\n") + append_to_tables_file(table_to_string(stats) + "\n") + mean, std = compute_statistics(load_times_array()) + print(f"Gen#{ga_instance.generations_completed} inference statistics:") + print(f"{mean=}ms") + print(f"{std=}ms") + create_temp_epoch_inference_dir() + + +def fitness_function(ga_inst, solution, solution_idx) -> float: + model = GeneralConfig.MODEL_INIT() + model.forward(*ml.onnx.onnx_import.create_torch_dummy_input()) + model_weights_dict = pygad.torchga.model_weights_as_dict( + model=model, weights_vector=solution + ) + + model.load_state_dict(model_weights_dict) + model.to(DEVICE) + model.eval() + nnwrapper = NNWrapper(model, weights_flat=solution) + + list_of_map2result = play_game( + weighted_predictor=nnwrapper, + max_steps=GeneralConfig.MAX_STEPS, + maps_type=MapsType.TRAIN, + ) + send_game_results(Agent2ResultsOnMaps(nnwrapper, list_of_map2result)) + + dump_and_reset_epoch_times( + f"{nnwrapper.name()}_epoch{ga_inst.generations_completed}_pid{getpid()}" + ) + rst = map(lambda map2res: map2res.game_result, list_of_map2result) + return straight_scorer(rst) diff --git a/VSharp.ML.AIAgent/learning/play_game.py b/VSharp.ML.AIAgent/learning/play_game.py new file mode 100644 index 000000000..31074133b --- /dev/null +++ b/VSharp.ML.AIAgent/learning/play_game.py @@ -0,0 +1,141 @@ +import logging +from statistics import StatisticsError +from time import perf_counter +from typing import TypeAlias + +import tqdm + +from agent.n_agent import NAgent +from agent.utils import MapsType, get_maps +from common.constants import TQDM_FORMAT_DICT +from common.utils import get_states +from config import FeatureConfig, GeneralConfig +from conn.socket_manager import game_server_socket_manager +from ml.fileop import save_model +from ml.model_wrappers.protocols import WeightedPredictor +from selection.classes import GameResult, Map2Result +from timer.resources_manager import manage_map_inference_times_array +from timer.stats import compute_statistics +from timer.utils import get_map_inference_times + +TimeDuration: TypeAlias = float + + +def play_map( + with_agent: NAgent, with_weighted_model: WeightedPredictor +) -> tuple[GameResult, TimeDuration]: + steps_count = 0 + game_state = None + actual_coverage = None + steps = with_agent.steps + + start_time = perf_counter() + + try: + for _ in range(steps): + game_state = with_agent.recv_state_or_throw_gameover() + predicted_state_id = with_weighted_model.predict(game_state) + logging.debug( + f"<{with_weighted_model.name()}> step: {steps_count}, available states: {get_states(game_state)}, predicted: {predicted_state_id}" + ) + + with_agent.send_step( + next_state_id=predicted_state_id, + predicted_usefullness=42.0, # left it a constant for now + ) + + _ = with_agent.recv_reward_or_throw_gameover() + steps_count += 1 + + _ = with_agent.recv_state_or_throw_gameover() # wait for gameover + steps_count += 1 + except NAgent.GameOver as gameover: + if game_state is None: + logging.error( + f"<{with_weighted_model.name()}>: immediate GameOver on {with_agent.map.MapName}" + ) + return GameResult( + steps_count=steps, + tests_count=0, + errors_count=0, + actual_coverage_percent=0, + ) + if gameover.actual_coverage is not None: + actual_coverage = gameover.actual_coverage + + tests_count = gameover.tests_count + errors_count = gameover.errors_count + + end_time = perf_counter() + + if ( + FeatureConfig.DUMP_BY_TIMEOUT.enabled + and end_time - start_time > FeatureConfig.DUMP_BY_TIMEOUT.timeout_seconds + ): + save_model( + GeneralConfig.MODEL_INIT(), + to=FeatureConfig.DUMP_BY_TIMEOUT.save_path + / f"{with_weighted_model.name()}.pth", + weights=with_weighted_model.weights(), + ) + + if actual_coverage != 100 and steps_count != steps: + logging.error( + f"<{with_weighted_model.name()}>: not all steps exshausted on {with_agent.map.MapName} with non-100% coverage" + f"steps taken: {steps_count}, actual coverage: {actual_coverage:.2f}" + ) + steps_count = steps + + model_result = GameResult( + steps_count=steps_count, + tests_count=tests_count, + errors_count=errors_count, + actual_coverage_percent=actual_coverage, + ) + + with manage_map_inference_times_array(): + try: + map_inference_times = get_map_inference_times() + mean, std = compute_statistics(map_inference_times) + logging.info( + f"Inference stats for <{with_weighted_model.name()}> on {with_agent.map.MapName}: {mean=}ms, {std=}ms" + ) + except StatisticsError: + logging.info( + f"<{with_weighted_model.name()}> on {with_agent.map.MapName}: too few samples for stats count" + ) + + return model_result, end_time - start_time + + +def play_game( + weighted_predictor: WeightedPredictor, max_steps: int, maps_type: MapsType +): + with game_server_socket_manager() as ws: + maps = get_maps(websocket=ws, type=maps_type) + with tqdm.tqdm( + total=len(maps), + desc=f"{weighted_predictor.name():20}: {maps_type.value}", + **TQDM_FORMAT_DICT, + ) as pbar: + rst: list[GameResult] = [] + list_of_map2result: list[Map2Result] = [] + for game_map in maps: + logging.info( + f"<{weighted_predictor.name()}> is playing {game_map.MapName}" + ) + + game_result, time = play_map( + with_agent=NAgent(ws, game_map, max_steps), + with_weighted_model=weighted_predictor, + ) + rst.append(game_result) + list_of_map2result.append(Map2Result(game_map, game_result)) + + logging.info( + f"<{weighted_predictor.name()}> finished map {game_map.MapName} " + f"in {game_result.steps_count} steps, {time} seconds, " + f"actual coverage: {game_result.actual_coverage_percent:.2f}" + ) + pbar.update(1) + return list_of_map2result diff --git a/VSharp.ML.AIAgent/learning/r_learn.py b/VSharp.ML.AIAgent/learning/r_learn.py deleted file mode 100644 index 1e8b16566..000000000 --- a/VSharp.ML.AIAgent/learning/r_learn.py +++ /dev/null @@ -1,265 +0,0 @@ -import copy -import json -import logging -import random -from collections import defaultdict -from os import getpid -from statistics import StatisticsError -from time import perf_counter -from typing import TypeAlias - -import pygad.torchga -import tqdm - -import ml -from agent.n_agent import NAgent -from agent.utils import MapsType, get_maps -from common.constants import DEVICE, TQDM_FORMAT_DICT -from common.utils import get_states -from config import FeatureConfig, GeneralConfig -from conn.classes import Agent2ResultsOnMaps -from conn.requests import recv_game_result_list, send_game_results -from conn.socket_manager import game_server_socket_manager -from epochs_statistics.tables import create_pivot_table, table_to_csv, table_to_string -from epochs_statistics.utils import ( - append_to_tables_file, - create_epoch_subdir, - rewrite_best_tables_file, -) -from ml.fileop import save_model -from ml.model_wrappers.nnwrapper import NNWrapper -from selection.classes import AgentResultsOnGameMaps, GameResult, Map2Result -from selection.scorer import straight_scorer -from timer.resources_manager import manage_map_inference_times_array -from timer.stats import compute_statistics -from timer.utils import ( - create_temp_epoch_inference_dir, - dump_and_reset_epoch_times, - get_map_inference_times, - load_times_array, -) - -TimeDuration: TypeAlias = float - - -def play_map( - with_agent: NAgent, with_model: NNWrapper -) -> tuple[GameResult, TimeDuration]: - steps_count = 0 - game_state = None - actual_coverage = None - steps = with_agent.steps - - start_time = perf_counter() - - try: - for _ in range(steps): - game_state = with_agent.recv_state_or_throw_gameover() - predicted_state_id = with_model.predict(game_state) - logging.debug( - f"<{with_model.name()}> step: {steps_count}, available states: {get_states(game_state)}, predicted: {predicted_state_id}" - ) - - with_agent.send_step( - next_state_id=predicted_state_id, - predicted_usefullness=42.0, # left it a constant for now - ) - - _ = with_agent.recv_reward_or_throw_gameover() - steps_count += 1 - - _ = with_agent.recv_state_or_throw_gameover() # wait for gameover - steps_count += 1 - except NAgent.GameOver as gameover: - if game_state is None: - logging.error( - f"<{with_model.name()}>: immediate GameOver on {with_agent.map.MapName}" - ) - return GameResult( - steps_count=steps, - tests_count=0, - errors_count=0, - actual_coverage_percent=0, - ) - if gameover.actual_coverage is not None: - actual_coverage = gameover.actual_coverage - - tests_count = gameover.tests_count - errors_count = gameover.errors_count - - end_time = perf_counter() - - if ( - FeatureConfig.DUMP_BY_TIMEOUT.enabled - and end_time - start_time > FeatureConfig.DUMP_BY_TIMEOUT.timeout_seconds - ): - save_model( - GeneralConfig.MODEL_INIT(), - to=FeatureConfig.DUMP_BY_TIMEOUT.save_path - / f"{sum(with_model.weights)}.pth", - weights=with_model.weights, - ) - - if actual_coverage != 100 and steps_count != steps: - logging.error( - f"<{with_model.name()}>: not all steps exshausted on {with_agent.map.MapName} with non-100% coverage" - f"steps taken: {steps_count}, actual coverage: {actual_coverage:.2f}" - ) - steps_count = steps - - model_result = GameResult( - steps_count=steps_count, - tests_count=tests_count, - errors_count=errors_count, - actual_coverage_percent=actual_coverage, - ) - - with manage_map_inference_times_array(): - try: - map_inference_times = get_map_inference_times() - mean, std = compute_statistics(map_inference_times) - logging.info( - f"Inference stats for <{with_model.name()}> on {with_agent.map.MapName}: {mean=}ms, {std=}ms" - ) - except StatisticsError: - logging.info( - f"<{with_model.name()}> on {with_agent.map.MapName}: too few samples for stats count" - ) - - return model_result, end_time - start_time - - -info_for_tables: AgentResultsOnGameMaps = defaultdict(list) -leader_table: AgentResultsOnGameMaps = defaultdict(list) - - -def get_n_best_weights_in_last_generation(ga_instance, n: int): - population = ga_instance.population - population_fitnesses = ga_instance.last_generation_fitness - - assert n <= len( - population - ), f"asked for {n} best when population size is {len(population)}" - - sorted_population = sorted( - zip(population, population_fitnesses), key=lambda x: x[1], reverse=True - ) - - return list(map(lambda x: x[0], sorted_population))[:n] - - -def on_generation(ga_instance): - game_results_raw = json.loads(recv_game_result_list()) - game_results_decoded = [ - Agent2ResultsOnMaps.from_json(item) for item in game_results_raw - ] - - for full_game_result in game_results_decoded: - info_for_tables[full_game_result.agent] = full_game_result.results - - print(f"Generation = {ga_instance.generations_completed};") - epoch_subdir = create_epoch_subdir(ga_instance.generations_completed) - - for weights in get_n_best_weights_in_last_generation( - ga_instance, FeatureConfig.N_BEST_SAVED_EACH_GEN - ): - save_model( - GeneralConfig.MODEL_INIT(), - to=epoch_subdir / f"{sum(weights)}.pth", - weights=weights, - ) - - ga_pop_inner_hashes = [ - tuple(individual).__hash__() for individual in ga_instance.population - ] - info_for_tables_filtered = { - nnwrapper: res - for nnwrapper, res in info_for_tables.items() - if nnwrapper._hash in ga_pop_inner_hashes - } - - best_solution_hash = tuple( - ga_instance.best_solution(pop_fitness=ga_instance.last_generation_fitness)[0] - ).__hash__() - best_solution_nnwrapper, best_solution_results = next( - filter( - lambda item: item[0]._hash == best_solution_hash, - info_for_tables_filtered.items(), - ) - ) - - append_to_tables_file( - f"Generations completed: {ga_instance.generations_completed}" + "\n" - ) - - if best_solution_nnwrapper in leader_table.keys(): - best_wrapper_copy = copy.copy(best_solution_nnwrapper) - best_wrapper_copy._hash += random.randint(0, 10**3) - leader_table[best_wrapper_copy] = best_solution_results - else: - leader_table[best_solution_nnwrapper] = best_solution_results - - _, stats, _ = create_pivot_table(leader_table, sort=False) - rewrite_best_tables_file(table_to_string(stats) + "\n") - - pivot, stats, epoch_table = create_pivot_table(info_for_tables_filtered) - if FeatureConfig.SAVE_EPOCHS_COVERAGES.enabled: - path_to_save_to = ( - FeatureConfig.SAVE_EPOCHS_COVERAGES.save_path - / f"{ga_instance.generations_completed}.csv" - ) - table_to_csv(epoch_table, path=path_to_save_to) - append_to_tables_file(table_to_string(pivot) + "\n") - append_to_tables_file(table_to_string(stats) + "\n") - mean, std = compute_statistics(load_times_array()) - print(f"Gen#{ga_instance.generations_completed} inference statistics:") - print(f"{mean=}ms") - print(f"{std=}ms") - create_temp_epoch_inference_dir() - - -def fitness_function(ga_inst, solution, solution_idx) -> float: - maps_type = MapsType.TRAIN - max_steps = GeneralConfig.MAX_STEPS - - model = GeneralConfig.MODEL_INIT() - model.forward(*ml.onnx.onnx_import.create_torch_dummy_input()) - model_weights_dict = pygad.torchga.model_weights_as_dict( - model=model, weights_vector=solution - ) - - model.load_state_dict(model_weights_dict) - model.to(DEVICE) - model.eval() - nnwrapper = NNWrapper(model, weights_flat=solution) - - with game_server_socket_manager() as ws: - maps = get_maps(websocket=ws, type=maps_type) - with tqdm.tqdm( - total=len(maps), - desc=f"{nnwrapper.name():20}: {maps_type.value}", - **TQDM_FORMAT_DICT, - ) as pbar: - rst: list[GameResult] = [] - list_of_map2result: list[Map2Result] = [] - for game_map in maps: - logging.info(f"<{nnwrapper.name()}> is playing {game_map.MapName}") - - game_result, time = play_map( - with_agent=NAgent(ws, game_map, max_steps), with_model=nnwrapper - ) - rst.append(game_result) - list_of_map2result.append(Map2Result(game_map, game_result)) - - logging.info( - f"<{nnwrapper.name()}> finished map {game_map.MapName} " - f"in {game_result.steps_count} steps, {time} seconds, " - f"actual coverage: {game_result.actual_coverage_percent:.2f}" - ) - pbar.update(1) - send_game_results(Agent2ResultsOnMaps(nnwrapper, list_of_map2result)) - - dump_and_reset_epoch_times( - f"{nnwrapper.name()}_epoch{ga_inst.generations_completed}_pid{getpid()}" - ) - return straight_scorer(rst) diff --git a/VSharp.ML.AIAgent/main.py b/VSharp.ML.AIAgent/main.py index 6705bb27a..1b0f722ed 100644 --- a/VSharp.ML.AIAgent/main.py +++ b/VSharp.ML.AIAgent/main.py @@ -1,7 +1,7 @@ import pygad import pygad.torchga -import learning.genetic_alorithm as ga +import learning.entry_point as ga import ml.onnx.onnx_import from config import GeneralConfig from selection.crossover_type import CrossoverType diff --git a/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py b/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py index f495625e1..ccbe5199c 100644 --- a/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py +++ b/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py @@ -8,12 +8,12 @@ from ml.predict_state_vector_hetero import PredictStateVectorHetGNN from ml.utils import load_model_with_last_layer -from .protocols import ModelWrapper +from .protocols import Predictor MAX_W, MIN_W = 1, -1 -class LastLayerLearner(ModelWrapper): +class LastLayerLearner(Predictor): def name(self) -> str: return self._name @@ -55,13 +55,13 @@ def train_single_val(self): return super().train_single_val() def copy(self, new_name: str) -> "LastLayerLearner": - assert new_name != self.name() + assert new_name != self._name copy = LastLayerLearner(self.weights.copy()) copy.rename(new_name) return copy def __str__(self) -> str: - return f"{self.name()}: {self.weights}" + return f"{self._name}: {self.weights}" def __hash__(self) -> int: return self.__str__().__hash__() diff --git a/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py b/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py index a4f569be3..1f8c2a252 100644 --- a/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py +++ b/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py @@ -4,20 +4,23 @@ from common.game import GameState from ml.data_loader_compact import ServerDataloaderHeteroVector -from ml.model_wrappers.protocols import Predictor +from ml.model_wrappers.protocols import WeightedPredictor from ml.predict_state_vector_hetero import PredictStateVectorHetGNN -class NNWrapper(Predictor): +class NNWrapper(WeightedPredictor): def __init__(self, model: torch.nn.Module, weights_flat: list[float]) -> None: self.model = model - self.weights = weights_flat self._name = str(sum(weights_flat)) - self._hash = tuple(weights_flat).__hash__() + self._weights = weights_flat + self.weights_hash = tuple(weights_flat).__hash__() def name(self) -> str: return self._name + def weights(self) -> list: + return self._weights + def predict(self, input: GameState): hetero_input, state_map = ServerDataloaderHeteroVector.convert_input_to_tensor( input @@ -36,20 +39,20 @@ def __eq__(self, __value: object) -> bool: return self.__hash__() == __value.__hash__() def __hash__(self) -> int: - return self._name.__hash__() + self._hash + return self.name().__hash__() + self.weights_hash def encode(obj): if isinstance(obj, NNWrapper): - return {"_name": obj._name, "_hash": obj._hash} + return {"name": obj._name, "weights_hash": obj.weights_hash} return json.dumps(obj) def decode(obj): - if "_name" not in obj or "_hash" not in obj: + if "name" not in obj or "weights_hash" not in obj: return obj fake_nnwrapper = NNWrapper(None, []) - fake_nnwrapper._name = obj["_name"] - fake_nnwrapper._hash = obj["_hash"] + fake_nnwrapper._name = obj["name"] + fake_nnwrapper.weights_hash = obj["weights_hash"] return fake_nnwrapper diff --git a/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py b/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py index d27c76483..0984a5bb5 100644 --- a/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py +++ b/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py @@ -1,16 +1,21 @@ -from abc import abstractmethod -from typing import Protocol +from abc import ABC, abstractmethod from common.game import GameState -class Named(Protocol): +class Named(ABC): @abstractmethod def name(self) -> str: raise NotImplementedError -class Predictor(Named, Protocol): +class Predictor(Named, ABC): @abstractmethod def predict(self, input: GameState): raise NotImplementedError + + +class WeightedPredictor(Predictor, ABC): + @abstractmethod + def weights(self) -> list: + raise NotImplementedError diff --git a/VSharp.ML.AIAgent/selection/scorer.py b/VSharp.ML.AIAgent/selection/scorer.py index 641c115d1..087a035c7 100644 --- a/VSharp.ML.AIAgent/selection/scorer.py +++ b/VSharp.ML.AIAgent/selection/scorer.py @@ -28,7 +28,7 @@ def euc_dist(data: Iterable, bound): return sum([abs(bound - item) ** 2 for item in data]) -def straight_scorer(model_results: list[GameResult]) -> tuple: +def straight_scorer(model_results: Iterable[GameResult]) -> tuple: # less is better coverage_score = -1 * euc_dist( data=[game_res.actual_coverage_percent for game_res in model_results], bound=100 From dc549dd65f028dab21ae1ca05466fe5d6ce3954d Mon Sep 17 00:00:00 2001 From: Max Nigmatulin Date: Thu, 20 Jul 2023 20:19:52 +0300 Subject: [PATCH 2/2] Big refactor --- .../{selection => common}/classes.py | 0 VSharp.ML.AIAgent/common/utils.py | 20 +++- .../broker_conn}/classes.py | 4 +- .../broker_conn}/requests.py | 0 .../broker_conn}/socket_manager.py | 1 - .../game_server_conn/connector.py} | 10 +- .../game_server_conn}/messages.py | 5 +- .../game_server_conn}/unsafe_json.py | 3 +- .../game_server_conn}/utils.py | 0 .../epochs_statistics/gen_stats.py | 2 +- VSharp.ML.AIAgent/epochs_statistics/tables.py | 4 +- VSharp.ML.AIAgent/epochs_statistics/utils.py | 6 +- VSharp.ML.AIAgent/learning/entry_point.py | 8 +- .../learning/genetic_learning.py | 20 ++-- VSharp.ML.AIAgent/learning/play_game.py | 104 +++++++++--------- .../selection/crossover_type.py | 0 .../{ => learning}/selection/mutation_type.py | 0 .../selection/parent_selection_type.py | 0 .../{ => learning}/selection/scorer.py | 2 +- .../{ => learning}/timer/resources_manager.py | 0 .../{ => learning}/timer/stats.py | 0 .../{ => learning}/timer/utils.py | 0 .../{ => learning}/timer/wrapper.py | 0 VSharp.ML.AIAgent/main.py | 6 +- .../ml/model_wrappers/last_layer_learner.py | 4 +- .../ml/model_wrappers/nnwrapper.py | 15 ++- .../ml/model_wrappers/protocols.py | 6 +- VSharp.ML.AIAgent/ml/models.py | 6 +- .../ml/predict_state_vector_hetero.py | 7 +- VSharp.ML.AIAgent/selection/utils.py | 17 --- 30 files changed, 127 insertions(+), 123 deletions(-) rename VSharp.ML.AIAgent/{selection => common}/classes.py (100%) rename VSharp.ML.AIAgent/{conn => connection/broker_conn}/classes.py (86%) rename VSharp.ML.AIAgent/{conn => connection/broker_conn}/requests.py (100%) rename VSharp.ML.AIAgent/{conn => connection/broker_conn}/socket_manager.py (99%) rename VSharp.ML.AIAgent/{agent/n_agent.py => connection/game_server_conn/connector.py} (95%) rename VSharp.ML.AIAgent/{agent => connection/game_server_conn}/messages.py (99%) rename VSharp.ML.AIAgent/{agent => connection/game_server_conn}/unsafe_json.py (92%) rename VSharp.ML.AIAgent/{agent => connection/game_server_conn}/utils.py (100%) rename VSharp.ML.AIAgent/{ => learning}/selection/crossover_type.py (100%) rename VSharp.ML.AIAgent/{ => learning}/selection/mutation_type.py (100%) rename VSharp.ML.AIAgent/{ => learning}/selection/parent_selection_type.py (100%) rename VSharp.ML.AIAgent/{ => learning}/selection/scorer.py (97%) rename VSharp.ML.AIAgent/{ => learning}/timer/resources_manager.py (100%) rename VSharp.ML.AIAgent/{ => learning}/timer/stats.py (100%) rename VSharp.ML.AIAgent/{ => learning}/timer/utils.py (100%) rename VSharp.ML.AIAgent/{ => learning}/timer/wrapper.py (100%) delete mode 100644 VSharp.ML.AIAgent/selection/utils.py diff --git a/VSharp.ML.AIAgent/selection/classes.py b/VSharp.ML.AIAgent/common/classes.py similarity index 100% rename from VSharp.ML.AIAgent/selection/classes.py rename to VSharp.ML.AIAgent/common/classes.py diff --git a/VSharp.ML.AIAgent/common/utils.py b/VSharp.ML.AIAgent/common/utils.py index f85e38dfd..e7b0a9ac2 100644 --- a/VSharp.ML.AIAgent/common/utils.py +++ b/VSharp.ML.AIAgent/common/utils.py @@ -1,5 +1,23 @@ -from .game import GameState +from collections import defaultdict + +from common.classes import Agent2Result, AgentResultsOnGameMaps, GameMapsModelResults + +from common.game import GameState def get_states(game_state: GameState) -> set[int]: return {s.Id for s in game_state.States} + + +def invert_mapping_mrgm_gmmr( + model_results_on_map: AgentResultsOnGameMaps, +) -> GameMapsModelResults: + inverse_mapping: GameMapsModelResults = defaultdict(list) + + for named_agent, list_of_map_result_mappings in model_results_on_map.items(): + for map_result_mapping in list_of_map_result_mappings: + map, result = (map_result_mapping.map, map_result_mapping.game_result) + + inverse_mapping[map].append(Agent2Result(named_agent, result)) + + return inverse_mapping diff --git a/VSharp.ML.AIAgent/conn/classes.py b/VSharp.ML.AIAgent/connection/broker_conn/classes.py similarity index 86% rename from VSharp.ML.AIAgent/conn/classes.py rename to VSharp.ML.AIAgent/connection/broker_conn/classes.py index 02701fa08..668e5cf5d 100644 --- a/VSharp.ML.AIAgent/conn/classes.py +++ b/VSharp.ML.AIAgent/connection/broker_conn/classes.py @@ -3,10 +3,10 @@ from dataclasses_json import config, dataclass_json -from agent.unsafe_json import asdict +from common.classes import Map2Result from config import FeatureConfig +from connection.game_server_conn.unsafe_json import asdict from ml.model_wrappers.nnwrapper import NNWrapper, decode, encode -from selection.classes import Map2Result def custom_encoder_if_disable_message_checks() -> Callable | None: diff --git a/VSharp.ML.AIAgent/conn/requests.py b/VSharp.ML.AIAgent/connection/broker_conn/requests.py similarity index 100% rename from VSharp.ML.AIAgent/conn/requests.py rename to VSharp.ML.AIAgent/connection/broker_conn/requests.py diff --git a/VSharp.ML.AIAgent/conn/socket_manager.py b/VSharp.ML.AIAgent/connection/broker_conn/socket_manager.py similarity index 99% rename from VSharp.ML.AIAgent/conn/socket_manager.py rename to VSharp.ML.AIAgent/connection/broker_conn/socket_manager.py index 538b9ce15..4a349df02 100644 --- a/VSharp.ML.AIAgent/conn/socket_manager.py +++ b/VSharp.ML.AIAgent/connection/broker_conn/socket_manager.py @@ -1,7 +1,6 @@ from contextlib import contextmanager import websocket - from .requests import aquire_ws, return_ws diff --git a/VSharp.ML.AIAgent/agent/n_agent.py b/VSharp.ML.AIAgent/connection/game_server_conn/connector.py similarity index 95% rename from VSharp.ML.AIAgent/agent/n_agent.py rename to VSharp.ML.AIAgent/connection/game_server_conn/connector.py index fed66d710..f91409d67 100644 --- a/VSharp.ML.AIAgent/agent/n_agent.py +++ b/VSharp.ML.AIAgent/connection/game_server_conn/connector.py @@ -19,8 +19,8 @@ ) -class NAgent: - class WrongAgentStateError(Exception): +class Connector: + class WrongConnectorStateError(Exception): def __init__( self, source: str, received: str, expected: str, at_step: int ) -> None: @@ -63,7 +63,7 @@ def __init__( def _raise_if_gameover(self, msg) -> GameOverServerMessage | str: if self.game_is_over: - raise NAgent.GameOver + raise Connector.GameOver matching_message_type = ServerMessage.from_json_handle( msg, expected=ServerMessage @@ -75,7 +75,7 @@ def _raise_if_gameover(self, msg) -> GameOverServerMessage | str: ) self.game_is_over = True logging.debug(f"--> {matching_message_type}") - raise NAgent.GameOver( + raise Connector.GameOver( actual_coverage=deser_msg.MessageBody.ActualCoverage, tests_count=deser_msg.MessageBody.TestsCount, errors_count=deser_msg.MessageBody.ErrorsCount, @@ -114,7 +114,7 @@ def recv_reward_or_throw_gameover(self) -> Reward: def _process_reward_server_message(self, msg): match msg.MessageType: case ServerMessageType.INCORRECT_PREDICTED_STATEID: - raise NAgent.IncorrectSentStateError( + raise Connector.IncorrectSentStateError( f"Sending state_id={self._sent_state_id} \ at step #{self._current_step} resulted in {msg.MessageType}" ) diff --git a/VSharp.ML.AIAgent/agent/messages.py b/VSharp.ML.AIAgent/connection/game_server_conn/messages.py similarity index 99% rename from VSharp.ML.AIAgent/agent/messages.py rename to VSharp.ML.AIAgent/connection/game_server_conn/messages.py index 76cb10556..f0df8c603 100644 --- a/VSharp.ML.AIAgent/agent/messages.py +++ b/VSharp.ML.AIAgent/connection/game_server_conn/messages.py @@ -4,12 +4,13 @@ from enum import Enum from typing import Optional -from .unsafe_json import obj_from_dict +from dataclasses_json import config, dataclass_json from common.game import GameMap, GameState, Reward -from dataclasses_json import config, dataclass_json from config import FeatureConfig +from .unsafe_json import obj_from_dict + class ClientMessageType(str, Enum): START = "start" diff --git a/VSharp.ML.AIAgent/agent/unsafe_json.py b/VSharp.ML.AIAgent/connection/game_server_conn/unsafe_json.py similarity index 92% rename from VSharp.ML.AIAgent/agent/unsafe_json.py rename to VSharp.ML.AIAgent/connection/game_server_conn/unsafe_json.py index 708ee813b..f40f19053 100644 --- a/VSharp.ML.AIAgent/agent/unsafe_json.py +++ b/VSharp.ML.AIAgent/connection/game_server_conn/unsafe_json.py @@ -1,4 +1,5 @@ -from dataclasses import asdict as dataclasses_asdict, is_dataclass +from dataclasses import asdict as dataclasses_asdict +from dataclasses import is_dataclass from typing import ClassVar, Protocol diff --git a/VSharp.ML.AIAgent/agent/utils.py b/VSharp.ML.AIAgent/connection/game_server_conn/utils.py similarity index 100% rename from VSharp.ML.AIAgent/agent/utils.py rename to VSharp.ML.AIAgent/connection/game_server_conn/utils.py diff --git a/VSharp.ML.AIAgent/epochs_statistics/gen_stats.py b/VSharp.ML.AIAgent/epochs_statistics/gen_stats.py index 65830db9e..079e21723 100644 --- a/VSharp.ML.AIAgent/epochs_statistics/gen_stats.py +++ b/VSharp.ML.AIAgent/epochs_statistics/gen_stats.py @@ -1,7 +1,7 @@ from statistics import mean, median from epochs_statistics.common import CoverageStats, Interval -from selection.classes import Map2Result +from common.classes import Map2Result def euc_dist2full_coverage(data: list[float]) -> float: diff --git a/VSharp.ML.AIAgent/epochs_statistics/tables.py b/VSharp.ML.AIAgent/epochs_statistics/tables.py index b47d1b852..0f764ad33 100644 --- a/VSharp.ML.AIAgent/epochs_statistics/tables.py +++ b/VSharp.ML.AIAgent/epochs_statistics/tables.py @@ -3,17 +3,17 @@ import pandas as pd +from common.classes import Agent2Result, AgentResultsOnGameMaps from common.strings import ( AV_COVERAGE_COL_NAME, COV_DEVIATION_COL_NAME, EUC_DIST2FULL_COV_COL_NAME, MEDIAN_COVERAGE_COL_NAME, ) +from common.utils import invert_mapping_mrgm_gmmr from config import FeatureConfig from epochs_statistics.common import Interval, Name2ResultViewModel from epochs_statistics.gen_stats import compute_euc_dist_to_full_coverage -from selection.classes import Agent2Result, AgentResultsOnGameMaps -from selection.utils import invert_mapping_mrgm_gmmr def get_sample_val(d: dict): diff --git a/VSharp.ML.AIAgent/epochs_statistics/utils.py b/VSharp.ML.AIAgent/epochs_statistics/utils.py index f0dc65ad8..0bf0b96e5 100644 --- a/VSharp.ML.AIAgent/epochs_statistics/utils.py +++ b/VSharp.ML.AIAgent/epochs_statistics/utils.py @@ -3,11 +3,11 @@ from shutil import rmtree from common.constants import ( - LEADERS_TABLES_LOG_FILE, - TABLES_LOG_FILE, APP_LOG_FILE, - EPOCH_BEST_DIR, BASE_REPORT_DIR, + EPOCH_BEST_DIR, + LEADERS_TABLES_LOG_FILE, + TABLES_LOG_FILE, ) diff --git a/VSharp.ML.AIAgent/learning/entry_point.py b/VSharp.ML.AIAgent/learning/entry_point.py index 1d511d63f..80d1a7339 100644 --- a/VSharp.ML.AIAgent/learning/entry_point.py +++ b/VSharp.ML.AIAgent/learning/entry_point.py @@ -15,10 +15,10 @@ init_log_file, init_tables_file, ) -from selection.crossover_type import CrossoverType -from selection.mutation_type import MutationType -from selection.parent_selection_type import ParentSelectionType -from timer.resources_manager import manage_inference_stats +from learning.selection.crossover_type import CrossoverType +from learning.selection.mutation_type import MutationType +from learning.selection.parent_selection_type import ParentSelectionType +from learning.timer.resources_manager import manage_inference_stats logging.basicConfig( level=GeneralConfig.LOGGER_LEVEL, diff --git a/VSharp.ML.AIAgent/learning/genetic_learning.py b/VSharp.ML.AIAgent/learning/genetic_learning.py index d233c22a6..8df2aaabc 100644 --- a/VSharp.ML.AIAgent/learning/genetic_learning.py +++ b/VSharp.ML.AIAgent/learning/genetic_learning.py @@ -7,27 +7,27 @@ import pygad.torchga import ml -from agent.utils import MapsType +from common.classes import AgentResultsOnGameMaps from common.constants import DEVICE from config import FeatureConfig, GeneralConfig -from conn.classes import Agent2ResultsOnMaps -from conn.requests import recv_game_result_list, send_game_results +from connection.broker_conn.classes import Agent2ResultsOnMaps +from connection.broker_conn.requests import recv_game_result_list, send_game_results +from connection.game_server_conn.utils import MapsType from epochs_statistics.tables import create_pivot_table, table_to_csv, table_to_string from epochs_statistics.utils import ( append_to_tables_file, create_epoch_subdir, rewrite_best_tables_file, ) -from ml.fileop import save_model -from ml.model_wrappers.nnwrapper import NNWrapper -from selection.classes import AgentResultsOnGameMaps -from selection.scorer import straight_scorer -from timer.stats import compute_statistics -from timer.utils import ( +from learning.selection.scorer import straight_scorer +from learning.timer.stats import compute_statistics +from learning.timer.utils import ( create_temp_epoch_inference_dir, dump_and_reset_epoch_times, load_times_array, ) +from ml.fileop import save_model +from ml.model_wrappers.nnwrapper import NNWrapper from .play_game import play_game @@ -133,7 +133,7 @@ def fitness_function(ga_inst, solution, solution_idx) -> float: nnwrapper = NNWrapper(model, weights_flat=solution) list_of_map2result = play_game( - weighted_predictor=nnwrapper, + with_predictor=nnwrapper, max_steps=GeneralConfig.MAX_STEPS, maps_type=MapsType.TRAIN, ) diff --git a/VSharp.ML.AIAgent/learning/play_game.py b/VSharp.ML.AIAgent/learning/play_game.py index 31074133b..06791f8dc 100644 --- a/VSharp.ML.AIAgent/learning/play_game.py +++ b/VSharp.ML.AIAgent/learning/play_game.py @@ -5,54 +5,54 @@ import tqdm -from agent.n_agent import NAgent -from agent.utils import MapsType, get_maps +from common.classes import GameResult, Map2Result from common.constants import TQDM_FORMAT_DICT from common.utils import get_states -from config import FeatureConfig, GeneralConfig -from conn.socket_manager import game_server_socket_manager +from config import FeatureConfig +from connection.broker_conn.socket_manager import game_server_socket_manager +from connection.game_server_conn.connector import Connector +from connection.game_server_conn.utils import MapsType, get_maps +from learning.timer.resources_manager import manage_map_inference_times_array +from learning.timer.stats import compute_statistics +from learning.timer.utils import get_map_inference_times from ml.fileop import save_model -from ml.model_wrappers.protocols import WeightedPredictor -from selection.classes import GameResult, Map2Result -from timer.resources_manager import manage_map_inference_times_array -from timer.stats import compute_statistics -from timer.utils import get_map_inference_times +from ml.model_wrappers.protocols import Predictor TimeDuration: TypeAlias = float def play_map( - with_agent: NAgent, with_weighted_model: WeightedPredictor + with_connector: Connector, with_predictor: Predictor ) -> tuple[GameResult, TimeDuration]: steps_count = 0 game_state = None actual_coverage = None - steps = with_agent.steps + steps = with_connector.steps start_time = perf_counter() try: for _ in range(steps): - game_state = with_agent.recv_state_or_throw_gameover() - predicted_state_id = with_weighted_model.predict(game_state) + game_state = with_connector.recv_state_or_throw_gameover() + predicted_state_id = with_predictor.predict(game_state) logging.debug( - f"<{with_weighted_model.name()}> step: {steps_count}, available states: {get_states(game_state)}, predicted: {predicted_state_id}" + f"<{with_predictor.name()}> step: {steps_count}, available states: {get_states(game_state)}, predicted: {predicted_state_id}" ) - with_agent.send_step( + with_connector.send_step( next_state_id=predicted_state_id, predicted_usefullness=42.0, # left it a constant for now ) - _ = with_agent.recv_reward_or_throw_gameover() + _ = with_connector.recv_reward_or_throw_gameover() steps_count += 1 - _ = with_agent.recv_state_or_throw_gameover() # wait for gameover + _ = with_connector.recv_state_or_throw_gameover() # wait for gameover steps_count += 1 - except NAgent.GameOver as gameover: + except Connector.GameOver as gameover: if game_state is None: - logging.error( - f"<{with_weighted_model.name()}>: immediate GameOver on {with_agent.map.MapName}" + logging.warning( + f"<{with_predictor.name()}>: immediate GameOver on {with_connector.map.MapName}" ) return GameResult( steps_count=steps, @@ -68,20 +68,9 @@ def play_map( end_time = perf_counter() - if ( - FeatureConfig.DUMP_BY_TIMEOUT.enabled - and end_time - start_time > FeatureConfig.DUMP_BY_TIMEOUT.timeout_seconds - ): - save_model( - GeneralConfig.MODEL_INIT(), - to=FeatureConfig.DUMP_BY_TIMEOUT.save_path - / f"{with_weighted_model.name()}.pth", - weights=with_weighted_model.weights(), - ) - if actual_coverage != 100 and steps_count != steps: - logging.error( - f"<{with_weighted_model.name()}>: not all steps exshausted on {with_agent.map.MapName} with non-100% coverage" + logging.warning( + f"<{with_predictor.name()}>: not all steps exshausted on {with_connector.map.MapName} with non-100% coverage" f"steps taken: {steps_count}, actual coverage: {actual_coverage:.2f}" ) steps_count = steps @@ -93,49 +82,64 @@ def play_map( actual_coverage_percent=actual_coverage, ) + return model_result, end_time - start_time + + +def play_map_with_stats( + with_connector: Connector, with_predictor: Predictor +) -> tuple[GameResult, TimeDuration]: + model_result, time_duration = play_map(with_connector, with_predictor) + with manage_map_inference_times_array(): try: map_inference_times = get_map_inference_times() mean, std = compute_statistics(map_inference_times) logging.info( - f"Inference stats for <{with_weighted_model.name()}> on {with_agent.map.MapName}: {mean=}ms, {std=}ms" + f"Inference stats for <{with_predictor.name()}> on {with_connector.map.MapName}: {mean=}ms, {std=}ms" ) except StatisticsError: logging.info( - f"<{with_weighted_model.name()}> on {with_agent.map.MapName}: too few samples for stats count" + f"<{with_predictor.name()}> on {with_connector.map.MapName}: too few samples for stats count" ) - return model_result, end_time - start_time + return model_result, time_duration -def play_game( - weighted_predictor: WeightedPredictor, max_steps: int, maps_type: MapsType -): +def play_game(with_predictor: Predictor, max_steps: int, maps_type: MapsType): with game_server_socket_manager() as ws: maps = get_maps(websocket=ws, type=maps_type) with tqdm.tqdm( total=len(maps), - desc=f"{weighted_predictor.name():20}: {maps_type.value}", + desc=f"{with_predictor.name():20}: {maps_type.value}", **TQDM_FORMAT_DICT, ) as pbar: - rst: list[GameResult] = [] list_of_map2result: list[Map2Result] = [] for game_map in maps: - logging.info( - f"<{weighted_predictor.name()}> is playing {game_map.MapName}" - ) + logging.info(f"<{with_predictor.name()}> is playing {game_map.MapName}") - game_result, time = play_map( - with_agent=NAgent(ws, game_map, max_steps), - with_weighted_model=weighted_predictor, + game_result, time = play_map_with_stats( + with_connector=Connector(ws, game_map, max_steps), + with_predictor=with_predictor, ) - rst.append(game_result) list_of_map2result.append(Map2Result(game_map, game_result)) - logging.info( - f"<{weighted_predictor.name()}> finished map {game_map.MapName} " + message = ( + f"<{with_predictor.name()}> finished map {game_map.MapName} " f"in {game_result.steps_count} steps, {time} seconds, " f"actual coverage: {game_result.actual_coverage_percent:.2f}" ) + logging_func = logging.info + if ( + FeatureConfig.DUMP_BY_TIMEOUT.enabled + and time > FeatureConfig.DUMP_BY_TIMEOUT.timeout_seconds + ): + logging_func = logging.warning + message = "OVERTIME: " + message + save_model( + with_predictor.model(), + to=FeatureConfig.DUMP_BY_TIMEOUT.save_path + / f"{with_predictor.name()}.pth", + ) + logging_func(message) pbar.update(1) return list_of_map2result diff --git a/VSharp.ML.AIAgent/selection/crossover_type.py b/VSharp.ML.AIAgent/learning/selection/crossover_type.py similarity index 100% rename from VSharp.ML.AIAgent/selection/crossover_type.py rename to VSharp.ML.AIAgent/learning/selection/crossover_type.py diff --git a/VSharp.ML.AIAgent/selection/mutation_type.py b/VSharp.ML.AIAgent/learning/selection/mutation_type.py similarity index 100% rename from VSharp.ML.AIAgent/selection/mutation_type.py rename to VSharp.ML.AIAgent/learning/selection/mutation_type.py diff --git a/VSharp.ML.AIAgent/selection/parent_selection_type.py b/VSharp.ML.AIAgent/learning/selection/parent_selection_type.py similarity index 100% rename from VSharp.ML.AIAgent/selection/parent_selection_type.py rename to VSharp.ML.AIAgent/learning/selection/parent_selection_type.py diff --git a/VSharp.ML.AIAgent/selection/scorer.py b/VSharp.ML.AIAgent/learning/selection/scorer.py similarity index 97% rename from VSharp.ML.AIAgent/selection/scorer.py rename to VSharp.ML.AIAgent/learning/selection/scorer.py index 087a035c7..e53cbd995 100644 --- a/VSharp.ML.AIAgent/selection/scorer.py +++ b/VSharp.ML.AIAgent/learning/selection/scorer.py @@ -5,7 +5,7 @@ from bit_coder.bit_coder import convert_to_big_int -from .classes import GameResult +from common.classes import GameResult def minkowski_superscorer(model_results: list[GameResult], k) -> float: diff --git a/VSharp.ML.AIAgent/timer/resources_manager.py b/VSharp.ML.AIAgent/learning/timer/resources_manager.py similarity index 100% rename from VSharp.ML.AIAgent/timer/resources_manager.py rename to VSharp.ML.AIAgent/learning/timer/resources_manager.py diff --git a/VSharp.ML.AIAgent/timer/stats.py b/VSharp.ML.AIAgent/learning/timer/stats.py similarity index 100% rename from VSharp.ML.AIAgent/timer/stats.py rename to VSharp.ML.AIAgent/learning/timer/stats.py diff --git a/VSharp.ML.AIAgent/timer/utils.py b/VSharp.ML.AIAgent/learning/timer/utils.py similarity index 100% rename from VSharp.ML.AIAgent/timer/utils.py rename to VSharp.ML.AIAgent/learning/timer/utils.py diff --git a/VSharp.ML.AIAgent/timer/wrapper.py b/VSharp.ML.AIAgent/learning/timer/wrapper.py similarity index 100% rename from VSharp.ML.AIAgent/timer/wrapper.py rename to VSharp.ML.AIAgent/learning/timer/wrapper.py diff --git a/VSharp.ML.AIAgent/main.py b/VSharp.ML.AIAgent/main.py index 1b0f722ed..d488c5be5 100644 --- a/VSharp.ML.AIAgent/main.py +++ b/VSharp.ML.AIAgent/main.py @@ -4,9 +4,9 @@ import learning.entry_point as ga import ml.onnx.onnx_import from config import GeneralConfig -from selection.crossover_type import CrossoverType -from selection.mutation_type import MutationType -from selection.parent_selection_type import ParentSelectionType +from learning.selection.crossover_type import CrossoverType +from learning.selection.mutation_type import MutationType +from learning.selection.parent_selection_type import ParentSelectionType def main(): diff --git a/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py b/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py index ccbe5199c..a4e881c4c 100644 --- a/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py +++ b/VSharp.ML.AIAgent/ml/model_wrappers/last_layer_learner.py @@ -1,5 +1,7 @@ import random +from protocols import Predictor + from common.constants import BASE_NN_OUT_FEATURES_NUM, IMPORTED_DICT_MODEL_PATH from common.game import GameState from config import FeatureConfig @@ -8,8 +10,6 @@ from ml.predict_state_vector_hetero import PredictStateVectorHetGNN from ml.utils import load_model_with_last_layer -from .protocols import Predictor - MAX_W, MIN_W = 1, -1 diff --git a/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py b/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py index 1f8c2a252..f100e4539 100644 --- a/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py +++ b/VSharp.ML.AIAgent/ml/model_wrappers/nnwrapper.py @@ -4,31 +4,30 @@ from common.game import GameState from ml.data_loader_compact import ServerDataloaderHeteroVector -from ml.model_wrappers.protocols import WeightedPredictor +from ml.model_wrappers.protocols import Predictor from ml.predict_state_vector_hetero import PredictStateVectorHetGNN -class NNWrapper(WeightedPredictor): +class NNWrapper(Predictor): def __init__(self, model: torch.nn.Module, weights_flat: list[float]) -> None: - self.model = model self._name = str(sum(weights_flat)) - self._weights = weights_flat + self._model = model self.weights_hash = tuple(weights_flat).__hash__() def name(self) -> str: return self._name - def weights(self) -> list: - return self._weights + def model(self) -> list: + return self._model def predict(self, input: GameState): hetero_input, state_map = ServerDataloaderHeteroVector.convert_input_to_tensor( input ) - assert self.model is not None + assert self._model is not None next_step_id = PredictStateVectorHetGNN.predict_state_single_out( - self.model, hetero_input, state_map + self._model, hetero_input, state_map ) del hetero_input return next_step_id diff --git a/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py b/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py index 0984a5bb5..5039497b3 100644 --- a/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py +++ b/VSharp.ML.AIAgent/ml/model_wrappers/protocols.py @@ -1,5 +1,7 @@ from abc import ABC, abstractmethod +import torch + from common.game import GameState @@ -14,8 +16,6 @@ class Predictor(Named, ABC): def predict(self, input: GameState): raise NotImplementedError - -class WeightedPredictor(Predictor, ABC): @abstractmethod - def weights(self) -> list: + def model(self) -> torch.nn.Module: raise NotImplementedError diff --git a/VSharp.ML.AIAgent/ml/models.py b/VSharp.ML.AIAgent/ml/models.py index 3bb8ce26f..f5d938eb1 100644 --- a/VSharp.ML.AIAgent/ml/models.py +++ b/VSharp.ML.AIAgent/ml/models.py @@ -1,12 +1,12 @@ import torch import torch.nn.functional as F +from .data_loader_compact import NUM_NODE_FEATURES from torch import nn from torch.nn import Linear from torch_geometric.nn import ( ARMAConv, FeaStConv, GATConv, - GatedGraphConv, GCNConv, HeteroConv, Linear, @@ -19,9 +19,7 @@ ) from torchvision.ops import MLP -from timer.wrapper import timeit - -from .data_loader_compact import NUM_NODE_FEATURES +from learning.timer.wrapper import timeit NUM_PREDICTED_VALUES = 4 diff --git a/VSharp.ML.AIAgent/ml/predict_state_vector_hetero.py b/VSharp.ML.AIAgent/ml/predict_state_vector_hetero.py index a00579f9d..7ba6baf12 100644 --- a/VSharp.ML.AIAgent/ml/predict_state_vector_hetero.py +++ b/VSharp.ML.AIAgent/ml/predict_state_vector_hetero.py @@ -3,13 +3,14 @@ import torch import torch.nn.functional as F -from common.constants import DEVICE -from ml import data_loader_compact -from ml.models import GNN_Het from torch_geometric.data import HeteroData from torch_geometric.loader import DataLoader from torch_geometric.nn import to_hetero +from common.constants import DEVICE +from ml import data_loader_compact +from ml.models import GNN_Het + StateVectorMapping = namedtuple("StateVectorMapping", ["state", "vector"]) diff --git a/VSharp.ML.AIAgent/selection/utils.py b/VSharp.ML.AIAgent/selection/utils.py deleted file mode 100644 index 467d392d4..000000000 --- a/VSharp.ML.AIAgent/selection/utils.py +++ /dev/null @@ -1,17 +0,0 @@ -from collections import defaultdict - -from .classes import Agent2Result, AgentResultsOnGameMaps, GameMapsModelResults - - -def invert_mapping_mrgm_gmmr( - model_results_on_map: AgentResultsOnGameMaps, -) -> GameMapsModelResults: - inverse_mapping: GameMapsModelResults = defaultdict(list) - - for named_agent, list_of_map_result_mappings in model_results_on_map.items(): - for map_result_mapping in list_of_map_result_mappings: - map, result = (map_result_mapping.map, map_result_mapping.game_result) - - inverse_mapping[map].append(Agent2Result(named_agent, result)) - - return inverse_mapping