From e8275f655a053015a00f2b33bfeea94d61e8ef5d Mon Sep 17 00:00:00 2001 From: Vinay Sagar Gonabavi Date: Fri, 23 Aug 2024 17:11:55 -0700 Subject: [PATCH 1/3] Add separate fileprefix handlers for vitess crs --- paasta_tools/cli/cmds/status.py | 16 +- paasta_tools/cli/utils.py | 20 +- paasta_tools/instance/kubernetes.py | 6 + paasta_tools/setup_kubernetes_cr.py | 10 +- paasta_tools/utils.py | 4 + paasta_tools/vitesscell_tools.py | 360 ++++++++++++++++ paasta_tools/vitesscluster_tools.py | 449 +------------------- paasta_tools/vitesskeyspace_tools.py | 612 +++++++++++++++++++++++++++ 8 files changed, 1038 insertions(+), 439 deletions(-) create mode 100644 paasta_tools/vitesscell_tools.py create mode 100644 paasta_tools/vitesskeyspace_tools.py diff --git a/paasta_tools/cli/cmds/status.py b/paasta_tools/cli/cmds/status.py index 32f1324619..b23db83d84 100644 --- a/paasta_tools/cli/cmds/status.py +++ b/paasta_tools/cli/cmds/status.py @@ -95,14 +95,18 @@ from paasta_tools.utils import PaastaColors from paasta_tools.utils import remove_ansi_escape_sequences from paasta_tools.utils import SystemPaastaConfig -from paasta_tools.vitesscluster_tools import VitessDeploymentConfig +from paasta_tools.vitesscell_tools import VitessCellConfig +from paasta_tools.vitesscluster_tools import VitessClusterConfig +from paasta_tools.vitesskeyspace_tools import VitessKeyspaceConfig FLINK_STATUS_MAX_THREAD_POOL_WORKERS = 50 ALLOWED_INSTANCE_CONFIG: Sequence[Type[InstanceConfig]] = [ FlinkDeploymentConfig, FlinkEksDeploymentConfig, CassandraClusterDeploymentConfig, - VitessDeploymentConfig, + VitessClusterConfig, + VitessCellConfig, + VitessKeyspaceConfig, KafkaClusterDeploymentConfig, KubernetesDeploymentConfig, EksDeploymentConfig, @@ -115,7 +119,9 @@ FlinkDeploymentConfig, FlinkEksDeploymentConfig, CassandraClusterDeploymentConfig, - VitessDeploymentConfig, + VitessClusterConfig, + VitessCellConfig, + VitessKeyspaceConfig, KafkaClusterDeploymentConfig, KubernetesDeploymentConfig, EksDeploymentConfig, @@ -137,7 +143,9 @@ EKS_DEPLOYMENT_CONFIGS = [ EksDeploymentConfig, FlinkEksDeploymentConfig, - VitessDeploymentConfig, + VitessClusterConfig, + VitessCellConfig, + VitessKeyspaceConfig, ] FLINK_DEPLOYMENT_CONFIGS = [FlinkDeploymentConfig, FlinkEksDeploymentConfig] diff --git a/paasta_tools/cli/utils.py b/paasta_tools/cli/utils.py index 979fbff996..9861de18d9 100644 --- a/paasta_tools/cli/utils.py +++ b/paasta_tools/cli/utils.py @@ -76,7 +76,9 @@ from paasta_tools.utils import PaastaColors from paasta_tools.utils import SystemPaastaConfig from paasta_tools.utils import validate_service_instance -from paasta_tools.vitesscluster_tools import load_vitess_instance_config +from paasta_tools.vitesscell_tools import load_vitess_cell_instance_config +from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_config +from paasta_tools.vitesskeyspace_tools import load_vitess_keyspace_instance_config try: from vault_tools.paasta_secret import get_client as get_vault_client @@ -680,7 +682,13 @@ class LongRunningInstanceTypeHandler(NamedTuple): get_service_instance_list, load_kafkacluster_instance_config ), vitesscluster=InstanceTypeHandler( - get_service_instance_list, load_vitess_instance_config + get_service_instance_list, load_vitess_cluster_instance_config + ), + vitesscell=InstanceTypeHandler( + get_service_instance_list, load_vitess_cell_instance_config + ), + vitesskeyspace=InstanceTypeHandler( + get_service_instance_list, load_vitess_keyspace_instance_config ), nrtsearchservice=InstanceTypeHandler( get_service_instance_list, load_nrtsearchservice_instance_config @@ -713,7 +721,13 @@ class LongRunningInstanceTypeHandler(NamedTuple): get_service_instance_list, load_kafkacluster_instance_config ), vitesscluster=LongRunningInstanceTypeHandler( - get_service_instance_list, load_vitess_instance_config + get_service_instance_list, load_vitess_cluster_instance_config + ), + vitesscell=LongRunningInstanceTypeHandler( + get_service_instance_list, load_vitess_cell_instance_config + ), + vitesskeyspace=LongRunningInstanceTypeHandler( + get_service_instance_list, load_vitess_keyspace_instance_config ), nrtsearchservice=LongRunningInstanceTypeHandler( get_service_instance_list, load_nrtsearchservice_instance_config diff --git a/paasta_tools/instance/kubernetes.py b/paasta_tools/instance/kubernetes.py index 0a90db4bd9..9964fc59b8 100644 --- a/paasta_tools/instance/kubernetes.py +++ b/paasta_tools/instance/kubernetes.py @@ -33,7 +33,9 @@ from paasta_tools import monkrelaycluster_tools from paasta_tools import nrtsearchservice_tools from paasta_tools import smartstack_tools +from paasta_tools import vitesscell_tools from paasta_tools import vitesscluster_tools +from paasta_tools import vitesskeyspace_tools from paasta_tools.cli.utils import LONG_RUNNING_INSTANCE_TYPE_HANDLERS from paasta_tools.instance.hpa_metrics_parser import HPAMetricsDict from paasta_tools.instance.hpa_metrics_parser import HPAMetricsParser @@ -57,6 +59,8 @@ "cassandracluster", "kafkacluster", "vitesscluster", + "vitesscell", + "vitesskeyspace", } INSTANCE_TYPES_K8S = { "cassandracluster", @@ -72,6 +76,8 @@ cassandracluster=cassandracluster_tools.cr_id, kafkacluster=kafkacluster_tools.cr_id, vitesscluster=vitesscluster_tools.cr_id, + vitesscell=vitesscell_tools.cr_id, + vitesskeyspace=vitesskeyspace_tools.cr_id, nrtsearchservice=nrtsearchservice_tools.cr_id, nrtsearchserviceeks=nrtsearchservice_tools.cr_id, monkrelaycluster=monkrelaycluster_tools.cr_id, diff --git a/paasta_tools/setup_kubernetes_cr.py b/paasta_tools/setup_kubernetes_cr.py index 5972b4da72..50647e1899 100644 --- a/paasta_tools/setup_kubernetes_cr.py +++ b/paasta_tools/setup_kubernetes_cr.py @@ -49,13 +49,19 @@ from paasta_tools.utils import get_git_sha_from_dockerurl from paasta_tools.utils import load_all_configs from paasta_tools.utils import load_system_paasta_config -from paasta_tools.vitesscluster_tools import load_vitess_service_instance_configs +from paasta_tools.vitesscell_tools import load_vitess_cell_instance_configs +from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_configs +from paasta_tools.vitesskeyspace_tools import load_vitess_keyspace_instance_configs log = logging.getLogger(__name__) -INSTANCE_TYPE_TO_CONFIG_LOADER = {"vitesscluster": load_vitess_service_instance_configs} +INSTANCE_TYPE_TO_CONFIG_LOADER = { + "vitesscluster": load_vitess_cluster_instance_configs, + "vitesscell": load_vitess_cell_instance_configs, + "vitesskeyspace": load_vitess_keyspace_instance_configs, +} class StdoutKubeClient: diff --git a/paasta_tools/utils.py b/paasta_tools/utils.py index a5d585c8ca..12c40dd03b 100644 --- a/paasta_tools/utils.py +++ b/paasta_tools/utils.py @@ -137,6 +137,8 @@ "cassandracluster", "kafkacluster", "vitesscluster", + "vitesscell", + "vitesskeyspace", "monkrelays", "nrtsearchservice", "nrtsearchserviceeks", @@ -156,6 +158,8 @@ "cassandracluster": "paasta-cassandraclusters", "kafkacluster": "paasta-kafkaclusters", "vitesscluster": "paasta-vitessclusters", + "vitesscell": "paasta-vitessclusters", + "vitesskeyspace": "paasta-vitessclusters", "nrtsearchservice": "paasta-nrtsearchservices", "nrtsearchserviceeks": "paasta-nrtsearchservices", } diff --git a/paasta_tools/vitesscell_tools.py b/paasta_tools/vitesscell_tools.py new file mode 100644 index 0000000000..25ba088236 --- /dev/null +++ b/paasta_tools/vitesscell_tools.py @@ -0,0 +1,360 @@ +import json +import logging +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import TypedDict +from typing import Union + +import service_configuration_lib +from kubernetes.client import ApiClient + +from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig +from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict +from paasta_tools.kubernetes_tools import sanitised_cr_name +from paasta_tools.long_running_service_tools import load_service_namespace_config +from paasta_tools.utils import BranchDictV2 +from paasta_tools.utils import deep_merge_dictionaries +from paasta_tools.utils import DEFAULT_SOA_DIR +from paasta_tools.utils import get_git_sha_from_dockerurl +from paasta_tools.utils import load_service_instance_config +from paasta_tools.utils import load_system_paasta_config +from paasta_tools.utils import load_v2_deployments_json +from paasta_tools.vitesscluster_tools import get_formatted_environment_variables +from paasta_tools.vitesscluster_tools import KVEnvVar +from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom +from paasta_tools.vitesscluster_tools import RequestsDict +from paasta_tools.vitesscluster_tools import ResourceConfigDict + + +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) + + +KUBERNETES_NAMESPACE = "paasta-vitessclusters" + + +# Global variables +TOPO_IMPLEMENTATION = "zk2" +TOPO_GLOBAL_ROOT = "/vitess-paasta/global" + +VTGATE_EXTRA_ENV = { + "VAULT_ROLEID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-roleid", + "key": "vault-vtgate-approle-roleid", + } + }, + "VAULT_SECRETID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-secretid", + "key": "vault-vtgate-approle-secretid", + } + }, +} + + +class GatewayConfigDict(TypedDict, total=False): + affinity: Dict[str, Any] + extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] + extraFlags: Dict[str, str] + extraLabels: Dict[str, str] + replicas: int + resources: Dict[str, Any] + annotations: Mapping[str, Any] + + +class VitessCellConfigDict(KubernetesDeploymentConfigDict, total=False): + name: str + images: Dict[str, str] + allCells: List[str] + globalLockserver: Dict[str, str] + gateway: GatewayConfigDict + + +def get_cell_config( + cell: str, + images: Dict[str, str], + allCells: List[str], + global_lock_server: Dict[str, str], + region: str, + vtgate_resources: ResourceConfigDict, + env: List[Union[KVEnvVar, KVEnvVarValueFrom]], + labels: Dict[str, str], + node_affinity: dict, + annotations: Mapping[str, Any], +) -> VitessCellConfigDict: + """ + get vtgate config + """ + replicas = vtgate_resources.get("replicas") + requests = vtgate_resources.get( + "requests", RequestsDict(cpu="100m", memory="256Mi") + ) + environment_overrides: Dict[str, Any] = { + "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", + "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + } + environment_overrides.update(VTGATE_EXTRA_ENV) + updated_vtgate_extra_env = ( + get_formatted_environment_variables(environment_overrides) + env + ) + + config = VitessCellConfigDict( + name=cell, + images=images, + allCells=allCells, + globalLockserver=global_lock_server, + gateway=GatewayConfigDict( + affinity={"nodeAffinity": node_affinity}, + extraEnv=updated_vtgate_extra_env, + extraFlags={ + "mysql_auth_server_impl": "vault", + "mysql_auth_vault_addr": f"https://vault-dre.{region}.yelpcorp.com:8200", + "mysql_auth_vault_path": "secrets/vitess/vt-gate/vttablet_credentials.json", + "mysql_auth_vault_tls_ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + "mysql_auth_vault_ttl": "60s", + }, + extraLabels=labels, + replicas=replicas, + resources={ + "requests": requests, + "limits": requests, + }, + annotations=annotations, + ), + ) + return config + + +class VitessCellInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): + cell: str + cells: List[str] + zk_address: str + vtgate_resources: ResourceConfigDict + images: Dict[str, str] + + +class VitessCellConfig(KubernetesDeploymentConfig): + config_dict: VitessCellInstanceConfigDict + + config_filename_prefix = "vitesscell" + + def __init__( + self, + service: str, + cluster: str, + instance: str, + config_dict: VitessCellConfigDict, + branch_dict: Optional[BranchDictV2], + soa_dir: str = DEFAULT_SOA_DIR, + ) -> None: + super().__init__( + cluster=cluster, # superregion + instance=instance, # host-1 + service=service, # vitess + soa_dir=soa_dir, + config_dict=config_dict, + branch_dict=branch_dict, + ) + + def get_namespace(self) -> str: + return KUBERNETES_NAMESPACE + + def get_images(self) -> Dict[str, str]: + vitess_images = self.config_dict.get( + "images", load_system_paasta_config().get_vitess_images() + ) + return { + "vtctld": vitess_images["vtctld_image"], + "vtadmin": vitess_images["vtadmin_image"], + "vtgate": vitess_images["vtgate_image"], + "vttablet": vitess_images["vttablet_image"], + } + + def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: + # get all K8s container env vars and format their keys to camel case + + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + env = [ + api_client.sanitize_for_serialization(env) + for env in self.get_container_env() + ] + return env + + def get_labels(self) -> Dict[str, str]: + # get default labels from parent class to adhere to paasta contract + docker_url = self.get_docker_url( + system_paasta_config=load_system_paasta_config() + ) + git_sha = get_git_sha_from_dockerurl(docker_url) + labels = self.get_kubernetes_metadata(git_sha=git_sha).labels + if "yelp.com/owner" in labels.keys(): + labels["yelp.com/owner"] = "dre_mysql" + return labels + + def get_annotations(self) -> Mapping[str, Any]: + # get required annotations to be added to the formatted resource before creating or updating custom resource + service_namespace_config = load_service_namespace_config( + service=self.service, namespace=self.get_nerve_namespace() + ) + system_paasta_config = load_system_paasta_config() + has_routable_ip = self.has_routable_ip( + service_namespace_config, system_paasta_config + ) + annotations: Mapping[str, Any] = { + "smartstack_registrations": json.dumps(self.get_registrations()), + "paasta.yelp.com/routable_ip": has_routable_ip, + } + + return annotations + + def get_vitess_node_affinity(self) -> dict: + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) + return node_affinity + + def get_region(self) -> str: + superregion = self.get_cluster() + superregion_to_region_map = ( + load_system_paasta_config().get_superregion_to_region_mapping() + ) + region = None + for superregion_prefix in superregion_to_region_map: + if superregion.startswith(superregion_prefix): + region = superregion.replace( + superregion_prefix, superregion_to_region_map[superregion_prefix] + ) + if region is None: + log.error( + f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" + ) + # Exiting early here since region is needed to fetch secrets from vault + sys.exit(1) + return region + + def get_global_lock_server(self) -> Dict[str, str]: + zk_address = self.config_dict.get("zk_address") + return { + "implementation": TOPO_IMPLEMENTATION, + "address": zk_address, + "rootPath": TOPO_GLOBAL_ROOT, + } + + def get_vitess_cell_config(self) -> VitessCellConfigDict: + cell = self.config_dict.get("cell") + all_cells = self.config_dict.get("cells") + images = self.get_images() + global_lock_server = self.get_global_lock_server() + region = self.get_region() + vtgate_resources = self.config_dict.get("vtgate_resources") + + formatted_env = self.get_env_variables() + labels = self.get_labels() + node_affinity = self.get_vitess_node_affinity() + annotations = self.get_annotations() + + return get_cell_config( + cell, + images, + all_cells, + global_lock_server, + region, + vtgate_resources, + formatted_env, + labels, + node_affinity, + annotations, + ) + + def validate( + self, + params: List[str] = [ + "cpus", + "security", + "dependencies_reference", + "deploy_group", + ], + ) -> List[str]: + # Use InstanceConfig to validate shared config keys like cpus and mem + # TODO: add mem back to this list once we fix PAASTA-15582 and + # move to using the same units as flink/marathon etc. + error_msgs = super().validate(params=params) + + if error_msgs: + name = self.get_instance() + return [f"{name}: {msg}" for msg in error_msgs] + else: + return [] + + +def load_vitess_cell_instance_config( + service: str, + instance: str, + cluster: str, + load_deployments: bool = True, + soa_dir: str = DEFAULT_SOA_DIR, +) -> VitessCellConfig: + general_config = service_configuration_lib.read_service_configuration( + service, soa_dir=soa_dir + ) + instance_config = load_service_instance_config( + service, instance, "vitesscell", cluster, soa_dir=soa_dir + ) + general_config = deep_merge_dictionaries( + overrides=instance_config, defaults=general_config + ) + + branch_dict: Optional[BranchDictV2] = None + if load_deployments: + deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) + temp_instance_config = VitessCellConfig( + service=service, + cluster=cluster, + instance=instance, + config_dict=general_config, + branch_dict=None, + soa_dir=soa_dir, + ) + branch = temp_instance_config.get_branch() + deploy_group = temp_instance_config.get_deploy_group() + branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) + + vitess_cell_config = VitessCellConfig( + service=service, + cluster=cluster, + instance=instance, + config_dict=general_config, + branch_dict=branch_dict, + soa_dir=soa_dir, + ) + + return vitess_cell_config + + +def load_vitess_cell_instance_configs( + service: str, + instance: str, + cluster: str, + soa_dir: str = DEFAULT_SOA_DIR, +) -> VitessCellConfigDict: + vitess_cell_instance_configs = load_vitess_cell_instance_config( + service, instance, cluster, soa_dir=soa_dir + ).get_vitess_cell_config() + return vitess_cell_instance_configs + + +# TODO: read this from CRD in service configs +def cr_id(service: str, instance: str) -> Mapping[str, str]: + return dict( + group="planetscale.com", + version="v2", + namespace=KUBERNETES_NAMESPACE, + plural="vitesscells", + name=sanitised_cr_name(service, instance), + ) diff --git a/paasta_tools/vitesscluster_tools.py b/paasta_tools/vitesscluster_tools.py index 35212a3840..cd1543996d 100644 --- a/paasta_tools/vitesscluster_tools.py +++ b/paasta_tools/vitesscluster_tools.py @@ -14,7 +14,6 @@ from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict -from paasta_tools.kubernetes_tools import limit_size_with_hash from paasta_tools.kubernetes_tools import sanitised_cr_name from paasta_tools.long_running_service_tools import load_service_namespace_config from paasta_tools.utils import BranchDictV2 @@ -36,7 +35,6 @@ # Global variables TOPO_IMPLEMENTATION = "zk2" TOPO_GLOBAL_ROOT = "/vitess-paasta/global" -SOURCE_DB_HOST = "169.254.255.254" WEB_PORT = "15000" GRPC_PORT = "15999" @@ -47,41 +45,6 @@ "GRPC_PORT": GRPC_PORT, } -VTTABLET_EXTRA_ENV = { - "WEB_PORT": WEB_PORT, - "GRPC_PORT": GRPC_PORT, - "SHARD": "0", - "EXTERNAL_DB": "1", - "ROLE": "replica", - "VAULT_ROLEID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", - "key": "vault-vttablet-approle-roleid", - } - }, - "VAULT_SECRETID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", - "key": "vault-vttablet-approle-secretid", - } - }, -} - -VTGATE_EXTRA_ENV = { - "VAULT_ROLEID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-roleid", - "key": "vault-vtgate-approle-roleid", - } - }, - "VAULT_SECRETID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-secretid", - "key": "vault-vtgate-approle-secretid", - } - }, -} - # Extra Flags VTADMIN_EXTRA_FLAGS = {"grpc-allow-reflection": "true"} @@ -91,21 +54,6 @@ "security_policy": "read-only", } -VTTABLET_EXTRA_FLAGS = { - "log_err_stacks": "true", - "grpc_max_message_size": "134217728", - "init_tablet_type": "replica", - "queryserver-config-schema-reload-time": "1800", - "dba_pool_size": "4", - "vreplication_heartbeat_update_interval": "60", - "vreplication_tablet_type": "REPLICA", - "keep_logs": "72h", - "enable-lag-throttler": "true", - "throttle_check_as_check_self": "true", - "db_charset": "utf8mb4", - "disable_active_reparents": "true", -} - class KVEnvVar(TypedDict, total=False): name: str @@ -130,13 +78,7 @@ class ResourceConfigDict(TypedDict, total=False): class GatewayConfigDict(TypedDict, total=False): - affinity: Dict[str, Any] - extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] - extraFlags: Dict[str, str] - extraLabels: Dict[str, str] replicas: int - resources: Dict[str, Any] - annotations: Mapping[str, Any] class CellConfigDict(TypedDict, total=False): @@ -169,44 +111,6 @@ class VtAdminConfigDict(TypedDict, total=False): annotations: Mapping[str, Any] -class VtTabletDict(TypedDict, total=False): - extraFlags: Dict[str, str] - resources: Dict[str, Any] - - -class TabletPoolDict(TypedDict, total=False): - cell: str - name: str - type: str - affinity: Dict[str, Any] - extraLabels: Dict[str, str] - extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] - extraVolumeMounts: List[Dict[str, Any]] - extraVolumes: List[Dict[str, Any]] - replicas: int - vttablet: VtTabletDict - externalDatastore: Dict[str, Any] - dataVolumeClaimTemplate: Dict[str, Any] - annotations: Mapping[str, Any] - - -class ShardTemplateDict(TypedDict, total=False): - databaseInitScriptSecret: Dict[str, str] - tabletPools: List[TabletPoolDict] - - -class PartitioningValueDict(TypedDict, total=False): - parts: int - shardTemplate: ShardTemplateDict - - -class KeyspaceConfigDict(TypedDict, total=False): - durabilityPolicy: str - turndownPolicy: str - partitionings: List[Dict[str, PartitioningValueDict]] - name: str - - def get_formatted_environment_variables( env_vars: Dict[str, Any] ) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: @@ -230,48 +134,14 @@ def get_formatted_environment_variables( def get_cell_config( cell: str, - region: str, - vtgate_resources: ResourceConfigDict, - env: List[Union[KVEnvVar, KVEnvVarValueFrom]], - labels: Dict[str, str], - node_affinity: dict, - annotations: Mapping[str, Any], ) -> CellConfigDict: """ get vtgate config """ - replicas = vtgate_resources.get("replicas") - requests = vtgate_resources.get( - "requests", RequestsDict(cpu="100m", memory="256Mi") - ) - environment_overrides: Dict[str, Any] = { - "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", - "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - } - environment_overrides.update(VTGATE_EXTRA_ENV) - updated_vtgate_extra_env = ( - get_formatted_environment_variables(environment_overrides) + env - ) - config = CellConfigDict( name=cell, gateway=GatewayConfigDict( - affinity={"nodeAffinity": node_affinity}, - extraEnv=updated_vtgate_extra_env, - extraFlags={ - "mysql_auth_server_impl": "vault", - "mysql_auth_vault_addr": f"https://vault-dre.{region}.yelpcorp.com:8200", - "mysql_auth_vault_path": "secrets/vitess/vt-gate/vttablet_credentials.json", - "mysql_auth_vault_tls_ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - "mysql_auth_vault_ttl": "60s", - }, - extraLabels=labels, - replicas=replicas, - resources={ - "requests": requests, - "limits": requests, - }, - annotations=annotations, + replicas=0, ), ) return config @@ -354,254 +224,16 @@ def get_vt_admin_config( return config -def get_tablet_pool_config( - cell: str, - db_name: str, - keyspace: str, - host: str, - zk_address: str, - throttle_query_table: str, - throttle_metrics_threshold: str, - tablet_type: str, - region: str, - vttablet_resources: ResourceConfigDict, - env: List[Union[KVEnvVar, KVEnvVarValueFrom]], - labels: Dict[str, str], - node_affinity: dict, - annotations: Mapping[str, Any], -) -> TabletPoolDict: - """ - get vttablet config - """ - vttablet_extra_flags = VTTABLET_EXTRA_FLAGS.copy() - flag_overrides = { - "throttle_metrics_query": f"select max_replication_delay from max_mysql_replication_delay.{throttle_query_table};", - "throttle_metrics_threshold": throttle_metrics_threshold, - "enforce-tableacl-config": "true", - "table-acl-config": f"/nail/srv/configs/vitess_keyspace_acls/acls_for_{db_name}.json", - "table-acl-config-reload-interval": "60s", - "queryserver-config-strict-table-acl": "true", - "db-credentials-server": "vault", - "db-credentials-vault-addr": f"https://vault-dre.{region}.yelpcorp.com:8200", - "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", - "db-credentials-vault-tls-ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - "db-credentials-vault-ttl": "60s", - } - vttablet_extra_flags.update(flag_overrides) - - environment_overrides: Dict[str, Any] = { - "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", - "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - "TOPOLOGY_FLAGS": f"--topo_implementation {TOPO_IMPLEMENTATION} --topo_global_server_address ${zk_address} --topo_global_root {TOPO_GLOBAL_ROOT}", - "CELL_TOPOLOGY_SERVERS": zk_address, - "DB": db_name, - "KEYSPACE": keyspace, - } - environment_overrides.update(VTTABLET_EXTRA_ENV) - updated_vttablet_extra_env = ( - get_formatted_environment_variables(environment_overrides) + env - ) - - # Add extra pod label to filter - tablet_type_label = limit_size_with_hash(name=f"{db_name}_{tablet_type}", limit=63) - labels.update({"tablet_type": tablet_type_label}) - - try: - type = load_system_paasta_config().get_vitess_tablet_pool_type_mapping()[ - tablet_type - ] - except KeyError: - log.error( - f"Tablet type {tablet_type} not found in system paasta config vitess_tablet_pool_type_mapping" - ) - type = "externalmaster" - - replicas = vttablet_resources.get("replicas") - requests = vttablet_resources.get( - "requests", RequestsDict(cpu="100m", memory="256Mi") - ) - - config = TabletPoolDict( - cell=cell, - name=f"{db_name}_{tablet_type}", - type=type, - affinity={"nodeAffinity": node_affinity}, - extraLabels=labels, - extraEnv=updated_vttablet_extra_env, - extraVolumeMounts=[ - { - "mountPath": "/etc/vault/all_cas", - "name": "vault-secrets", - "readOnly": True, - }, - { - "mountPath": "/nail/srv", - "name": "srv-configs", - "readOnly": True, - }, - { - "mountPath": "/nail/etc/srv-configs", - "name": "etc-srv-configs", - "readOnly": True, - }, - { - "mountPath": "etc/credentials.yaml", - "name": "vttablet-fake-credentials", - "readOnly": True, - }, - { - "mountPath": "/etc/init_db.sql", - "name": "keyspace-fake-init-script", - "readOnly": True, - }, - ], - extraVolumes=[ - {"name": "vault-secrets", "hostPath": {"path": "/nail/etc/vault/all_cas"}}, - { - "name": "srv-configs", - "hostPath": {"path": "/nail/srv"}, - }, - { - "name": "etc-srv-configs", - "hostPath": {"path": "/nail/etc/srv-configs"}, - }, - {"name": "vttablet-fake-credentials", "hostPath": {"path": "/dev/null"}}, - {"name": "keyspace-fake-init-script", "hostPath": {"path": "/dev/null"}}, - ], - replicas=replicas, - vttablet={ - "extraFlags": vttablet_extra_flags, - "resources": { - "requests": requests, - "limits": requests, - }, - }, - externalDatastore={ - "database": db_name, - "host": host, - "port": 3306, - "user": "vt_app", - "credentialsSecret": { - "key": "/etc/credentials.yaml", - "volumeName": "vttablet-fake-credentials", - }, - }, - dataVolumeClaimTemplate={ - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - "storageClassName": "ebs-csi-gp3", - }, - annotations=annotations, - ) - return config - - -def get_keyspaces_config( - cells: List[str], - keyspaces: List[Dict[str, Any]], - zk_address: str, - region: str, - env: List[Union[KVEnvVar, KVEnvVarValueFrom]], - labels: Dict[str, str], - node_affinity: dict, - annotations: Mapping[str, Any], -) -> List[KeyspaceConfigDict]: - """ - get vitess keyspace config - """ - config = [] - - for keyspace_config in keyspaces: - keyspace = keyspace_config["keyspace"] - db_name = keyspace_config["keyspace"] - cluster = keyspace_config["cluster"] - vttablet_resources = keyspace_config.get("vttablet_resources") - - tablet_pools = [] - - # get vttablets - tablet_types = load_system_paasta_config().get_vitess_tablet_types() - for tablet_type in tablet_types: - ecosystem = region.split("-")[-1] - host = f"mysql-{cluster}-{tablet_type}.dre-{ecosystem}" - - # We use migration_replication delay for migration tablets and read_replication_delay for everything else - # Also throttling threshold for refresh and sanitized primaries is set at 30 seconds and everything else at 3 seconds - try: - throttling_configs = ( - load_system_paasta_config().get_vitess_throttling_config() - ) - throttle_query_table = throttling_configs[tablet_type][ - "throttle_query_table" - ] - throttle_metrics_threshold = throttling_configs[tablet_type][ - "throttle_metrics_threshold" - ] - except KeyError: - log.error( - f"Throttling configs for tablet type {tablet_type} not found in system paasta config vitess_throttling_configs" - ) - - if cluster.startswith("refresh") or cluster.startswith("sanitized"): - throttle_metrics_threshold = "30" - else: - throttle_metrics_threshold = "3" - - tablet_pools.extend( - [ - get_tablet_pool_config( - cell, - db_name, - keyspace, - host, - zk_address, - throttle_query_table, - throttle_metrics_threshold, - tablet_type, - region, - vttablet_resources, - env, - labels, - node_affinity, - annotations, - ) - for cell in cells - ] - ) - keyspace_config_value = KeyspaceConfigDict( - name=keyspace, - durabilityPolicy="none", - turndownPolicy="Immediate", - partitionings=[ - { - "equal": PartitioningValueDict( - parts=1, - shardTemplate=ShardTemplateDict( - databaseInitScriptSecret={ - "volumeName": "keyspace-fake-init-script", - "key": "/etc/init_db.sql", - }, - tabletPools=tablet_pools, - ), - ) - } - ], - ) - config.append(keyspace_config_value) - return config - - -class VitessDeploymentConfigDict(KubernetesDeploymentConfigDict, total=False): +class VitessClusterConfigDict(KubernetesDeploymentConfigDict, total=False): images: Dict[str, str] cells: List[CellConfigDict] vitessDashboard: VitessDashboardConfigDict vtadmin: VtAdminConfigDict - keyspaces: List[KeyspaceConfigDict] updateStrategy: Dict[str, str] globalLockserver: Dict[str, Dict[str, str]] -class VitessInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): +class VitessClusterInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): cells: List[str] zk_address: str vtctld_resources: ResourceConfigDict @@ -609,11 +241,10 @@ class VitessInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): vttablet_resources: ResourceConfigDict vtadmin_resources: ResourceConfigDict images: Dict[str, str] - keyspaces: List[Dict[str, Any]] -class VitessDeploymentConfig(KubernetesDeploymentConfig): - config_dict: VitessInstanceConfigDict +class VitessClusterConfig(KubernetesDeploymentConfig): + config_dict: VitessClusterInstanceConfigDict config_filename_prefix = "vitesscluster" @@ -622,7 +253,7 @@ def __init__( service: str, cluster: str, instance: str, - config_dict: VitessDeploymentConfigDict, + config_dict: VitessClusterConfigDict, branch_dict: Optional[BranchDictV2], soa_dir: str = DEFAULT_SOA_DIR, ) -> None: @@ -722,29 +353,6 @@ def get_global_lock_server(self) -> Dict[str, Dict[str, str]]: } } - def get_cells(self) -> List[CellConfigDict]: - cells = self.config_dict.get("cells") - region = self.get_region() - vtgate_resources = self.config_dict.get("vtgate_resources") - - formatted_env = self.get_env_variables() - labels = self.get_labels() - node_affinity = self.get_vitess_node_affinity() - annotations = self.get_annotations() - - return [ - get_cell_config( - cell, - region, - vtgate_resources, - formatted_env, - labels, - node_affinity, - annotations, - ) - for cell in cells - ] - def get_vitess_dashboard(self) -> VitessDashboardConfigDict: cells = self.config_dict.get("cells") zk_address = self.config_dict.get("zk_address") @@ -778,40 +386,21 @@ def get_vtadmin(self) -> VtAdminConfigDict: cells, vtadmin_resources, formatted_env, labels, node_affinity, annotations ) - def get_keyspaces(self) -> List[KeyspaceConfigDict]: + def get_cells(self) -> List[CellConfigDict]: cells = self.config_dict.get("cells") - zk_address = self.config_dict.get("zk_address") - region = self.get_region() - keyspaces = self.config_dict.get("keyspaces") - - formatted_env = self.get_env_variables() - labels = self.get_labels() - node_affinity = self.get_vitess_node_affinity() - annotations = self.get_annotations() - - return get_keyspaces_config( - cells, - keyspaces, - zk_address, - region, - formatted_env, - labels, - node_affinity, - annotations, - ) + return [get_cell_config(cell) for cell in cells] def get_update_strategy(self) -> Dict[str, str]: return {"type": "Immediate"} - def get_vitess_config(self) -> VitessDeploymentConfigDict: - vitess_config = VitessDeploymentConfigDict( + def get_vitess_config(self) -> VitessClusterConfigDict: + vitess_config = VitessClusterConfigDict( namespace=self.get_namespace(), images=self.get_images(), globalLockserver=self.get_global_lock_server(), cells=self.get_cells(), vitessDashboard=self.get_vitess_dashboard(), vtadmin=self.get_vtadmin(), - keyspaces=self.get_keyspaces(), updateStrategy=self.get_update_strategy(), ) return vitess_config @@ -837,13 +426,13 @@ def validate( return [] -def load_vitess_instance_config( +def load_vitess_cluster_instance_config( service: str, instance: str, cluster: str, load_deployments: bool = True, soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessDeploymentConfig: +) -> VitessClusterConfig: general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir ) @@ -857,7 +446,7 @@ def load_vitess_instance_config( branch_dict: Optional[BranchDictV2] = None if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) - temp_instance_config = VitessDeploymentConfig( + temp_instance_config = VitessClusterConfig( service=service, cluster=cluster, instance=instance, @@ -869,7 +458,7 @@ def load_vitess_instance_config( deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) - vitess_deployment_config = VitessDeploymentConfig( + vitess_cluster_config = VitessClusterConfig( service=service, cluster=cluster, instance=instance, @@ -878,19 +467,19 @@ def load_vitess_instance_config( soa_dir=soa_dir, ) - return vitess_deployment_config + return vitess_cluster_config -def load_vitess_service_instance_configs( +def load_vitess_cluster_instance_configs( service: str, instance: str, cluster: str, soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessDeploymentConfigDict: - vitess_service_instance_configs = load_vitess_instance_config( +) -> VitessClusterConfigDict: + vitess_cluster_instance_configs = load_vitess_cluster_instance_config( service, instance, cluster, soa_dir=soa_dir ).get_vitess_config() - return vitess_service_instance_configs + return vitess_cluster_instance_configs # TODO: read this from CRD in service configs diff --git a/paasta_tools/vitesskeyspace_tools.py b/paasta_tools/vitesskeyspace_tools.py new file mode 100644 index 0000000000..d16161d47b --- /dev/null +++ b/paasta_tools/vitesskeyspace_tools.py @@ -0,0 +1,612 @@ +import json +import logging +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import TypedDict +from typing import Union + +import service_configuration_lib +from kubernetes.client import ApiClient + +from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig +from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict +from paasta_tools.kubernetes_tools import limit_size_with_hash +from paasta_tools.kubernetes_tools import sanitised_cr_name +from paasta_tools.long_running_service_tools import load_service_namespace_config +from paasta_tools.utils import BranchDictV2 +from paasta_tools.utils import deep_merge_dictionaries +from paasta_tools.utils import DEFAULT_SOA_DIR +from paasta_tools.utils import get_git_sha_from_dockerurl +from paasta_tools.utils import load_service_instance_config +from paasta_tools.utils import load_system_paasta_config +from paasta_tools.utils import load_v2_deployments_json +from paasta_tools.vitesscluster_tools import get_formatted_environment_variables +from paasta_tools.vitesscluster_tools import KVEnvVar +from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom +from paasta_tools.vitesscluster_tools import RequestsDict +from paasta_tools.vitesscluster_tools import ResourceConfigDict + + +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) + + +KUBERNETES_NAMESPACE = "paasta-vitessclusters" + + +# Global variables +TOPO_IMPLEMENTATION = "zk2" +TOPO_GLOBAL_ROOT = "/vitess-paasta/global" +WEB_PORT = "15000" +GRPC_PORT = "15999" + + +# Environment variables +VTTABLET_EXTRA_ENV = { + "WEB_PORT": WEB_PORT, + "GRPC_PORT": GRPC_PORT, + "SHARD": "0", + "EXTERNAL_DB": "1", + "ROLE": "replica", + "VAULT_ROLEID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", + "key": "vault-vttablet-approle-roleid", + } + }, + "VAULT_SECRETID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", + "key": "vault-vttablet-approle-secretid", + } + }, +} + +# Extra Flags +VTTABLET_EXTRA_FLAGS = { + "log_err_stacks": "true", + "grpc_max_message_size": "134217728", + "init_tablet_type": "replica", + "queryserver-config-schema-reload-time": "1800", + "dba_pool_size": "4", + "vreplication_heartbeat_update_interval": "60", + "vreplication_tablet_type": "REPLICA", + "keep_logs": "72h", + "enable-lag-throttler": "true", + "throttle_check_as_check_self": "true", + "db_charset": "utf8mb4", + "disable_active_reparents": "true", +} + + +class VtTabletDict(TypedDict, total=False): + extraFlags: Dict[str, str] + resources: Dict[str, Any] + + +class TabletPoolDict(TypedDict, total=False): + cell: str + name: str + type: str + affinity: Dict[str, Any] + extraLabels: Dict[str, str] + extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] + extraVolumeMounts: List[Dict[str, Any]] + extraVolumes: List[Dict[str, Any]] + replicas: int + vttablet: VtTabletDict + externalDatastore: Dict[str, Any] + dataVolumeClaimTemplate: Dict[str, Any] + annotations: Mapping[str, Any] + + +class ShardTemplateDict(TypedDict, total=False): + databaseInitScriptSecret: Dict[str, str] + tabletPools: List[TabletPoolDict] + + +class PartitioningValueDict(TypedDict, total=False): + parts: int + shardTemplate: ShardTemplateDict + + +class KeyspaceConfigDict(TypedDict, total=False): + durabilityPolicy: str + turndownPolicy: str + partitionings: List[Dict[str, PartitioningValueDict]] + name: str + + +def get_tablet_pool_config( + cell: str, + db_name: str, + keyspace: str, + host: str, + zk_address: str, + throttle_query_table: str, + throttle_metrics_threshold: str, + tablet_type: str, + region: str, + vttablet_resources: ResourceConfigDict, + env: List[Union[KVEnvVar, KVEnvVarValueFrom]], + labels: Dict[str, str], + node_affinity: dict, + annotations: Mapping[str, Any], +) -> TabletPoolDict: + """ + get vttablet config + """ + vttablet_extra_flags = VTTABLET_EXTRA_FLAGS.copy() + flag_overrides = { + "throttle_metrics_query": f"select max_replication_delay from max_mysql_replication_delay.{throttle_query_table};", + "throttle_metrics_threshold": throttle_metrics_threshold, + "enforce-tableacl-config": "true", + "table-acl-config": f"/nail/srv/configs/vitess_keyspace_acls/acls_for_{db_name}.json", + "table-acl-config-reload-interval": "60s", + "queryserver-config-strict-table-acl": "true", + "db-credentials-server": "vault", + "db-credentials-vault-addr": f"https://vault-dre.{region}.yelpcorp.com:8200", + "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", + "db-credentials-vault-tls-ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + "db-credentials-vault-ttl": "60s", + } + vttablet_extra_flags.update(flag_overrides) + + environment_overrides: Dict[str, Any] = { + "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", + "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + "TOPOLOGY_FLAGS": f"--topo_implementation {TOPO_IMPLEMENTATION} --topo_global_server_address ${zk_address} --topo_global_root {TOPO_GLOBAL_ROOT}", + "CELL_TOPOLOGY_SERVERS": zk_address, + "DB": db_name, + "KEYSPACE": keyspace, + } + environment_overrides.update(VTTABLET_EXTRA_ENV) + updated_vttablet_extra_env = ( + get_formatted_environment_variables(environment_overrides) + env + ) + + # Add extra pod label to filter + tablet_type_label = limit_size_with_hash(name=f"{db_name}_{tablet_type}", limit=63) + labels.update({"tablet_type": tablet_type_label}) + + try: + type = load_system_paasta_config().get_vitess_tablet_pool_type_mapping()[ + tablet_type + ] + except KeyError: + log.error( + f"Tablet type {tablet_type} not found in system paasta config vitess_tablet_pool_type_mapping" + ) + type = "externalmaster" + + replicas = vttablet_resources.get("replicas") + requests = vttablet_resources.get( + "requests", RequestsDict(cpu="100m", memory="256Mi") + ) + + config = TabletPoolDict( + cell=cell, + name=f"{db_name}_{tablet_type}", + type=type, + affinity={"nodeAffinity": node_affinity}, + extraLabels=labels, + extraEnv=updated_vttablet_extra_env, + extraVolumeMounts=[ + { + "mountPath": "/etc/vault/all_cas", + "name": "vault-secrets", + "readOnly": True, + }, + { + "mountPath": "/nail/srv", + "name": "srv-configs", + "readOnly": True, + }, + { + "mountPath": "/nail/etc/srv-configs", + "name": "etc-srv-configs", + "readOnly": True, + }, + { + "mountPath": "etc/credentials.yaml", + "name": "vttablet-fake-credentials", + "readOnly": True, + }, + { + "mountPath": "/etc/init_db.sql", + "name": "keyspace-fake-init-script", + "readOnly": True, + }, + ], + extraVolumes=[ + {"name": "vault-secrets", "hostPath": {"path": "/nail/etc/vault/all_cas"}}, + { + "name": "srv-configs", + "hostPath": {"path": "/nail/srv"}, + }, + { + "name": "etc-srv-configs", + "hostPath": {"path": "/nail/etc/srv-configs"}, + }, + {"name": "vttablet-fake-credentials", "hostPath": {"path": "/dev/null"}}, + {"name": "keyspace-fake-init-script", "hostPath": {"path": "/dev/null"}}, + ], + replicas=replicas, + vttablet={ + "extraFlags": vttablet_extra_flags, + "resources": { + "requests": requests, + "limits": requests, + }, + }, + externalDatastore={ + "database": db_name, + "host": host, + "port": 3306, + "user": "vt_app", + "credentialsSecret": { + "key": "/etc/credentials.yaml", + "volumeName": "vttablet-fake-credentials", + }, + }, + dataVolumeClaimTemplate={ + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "10Gi"}}, + "storageClassName": "ebs-csi-gp3", + }, + annotations=annotations, + ) + return config + + +class VitessKeyspaceConfigDict(KubernetesDeploymentConfigDict, total=False): + name: str + images: Dict[str, str] + databaseName: str + durabilityPolicy: str + turndownPolicy: str + partitionings: List[Dict[str, PartitioningValueDict]] + updateStrategy: Dict[str, str] + globalLockserver: Dict[str, str] + zoneMap: Dict[str, Any] + + +def get_keyspace_config( + cells: List[str], + keyspace: str, + cluster: str, + vttablet_resources: ResourceConfigDict, + images: Dict[str, str], + update_strategy: Dict[str, str], + global_lockserver: Dict[str, str], + zk_address: str, + region: str, + env: List[Union[KVEnvVar, KVEnvVarValueFrom]], + labels: Dict[str, str], + node_affinity: dict, + annotations: Mapping[str, Any], +) -> VitessKeyspaceConfigDict: + """ + get vitess keyspace config + """ + db_name = keyspace + + tablet_pools = [] + + # get vttablets + tablet_types = load_system_paasta_config().get_vitess_tablet_types() + for tablet_type in tablet_types: + ecosystem = region.split("-")[-1] + host = f"mysql-{cluster}-{tablet_type}.dre-{ecosystem}" + + # We use migration_replication delay for migration tablets and read_replication_delay for everything else + # Also throttling threshold for refresh and sanitized primaries is set at 30 seconds and everything else at 3 seconds + try: + throttling_configs = ( + load_system_paasta_config().get_vitess_throttling_config() + ) + throttle_query_table = throttling_configs[tablet_type][ + "throttle_query_table" + ] + throttle_metrics_threshold = throttling_configs[tablet_type][ + "throttle_metrics_threshold" + ] + except KeyError: + log.error( + f"Throttling configs for tablet type {tablet_type} not found in system paasta config vitess_throttling_configs" + ) + + if cluster.startswith("refresh") or cluster.startswith("sanitized"): + throttle_metrics_threshold = "30" + else: + throttle_metrics_threshold = "3" + + tablet_pools.extend( + [ + get_tablet_pool_config( + cell, + db_name, + keyspace, + host, + zk_address, + throttle_query_table, + throttle_metrics_threshold, + tablet_type, + region, + vttablet_resources, + env, + labels, + node_affinity, + annotations, + ) + for cell in cells + ] + ) + vitess_keyspace_config = VitessKeyspaceConfigDict( + name=keyspace, + durabilityPolicy="none", + turndownPolicy="Immediate", + images=images, + databaseName=db_name, + updateStrategy=update_strategy, + globalLockserver=global_lockserver, + partitionings=[ + { + "equal": PartitioningValueDict( + parts=1, + shardTemplate=ShardTemplateDict( + databaseInitScriptSecret={ + "volumeName": "keyspace-fake-init-script", + "key": "/etc/init_db.sql", + }, + tabletPools=tablet_pools, + ), + ) + } + ], + zoneMap={}, + ) + return vitess_keyspace_config + + +class VitessKeyspaceInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): + cells: List[str] + zk_address: str + images: Dict[str, str] + keyspace: str + cluster: str + vttablet_resources: ResourceConfigDict + + +class VitessKeyspaceConfig(KubernetesDeploymentConfig): + config_dict: VitessKeyspaceInstanceConfigDict + + config_filename_prefix = "vitesskeyspace" + + def __init__( + self, + service: str, + cluster: str, + instance: str, + config_dict: VitessKeyspaceConfigDict, + branch_dict: Optional[BranchDictV2], + soa_dir: str = DEFAULT_SOA_DIR, + ) -> None: + super().__init__( + cluster=cluster, # superregion + instance=instance, # host-1 + service=service, # vitess + soa_dir=soa_dir, + config_dict=config_dict, + branch_dict=branch_dict, + ) + + def get_namespace(self) -> str: + return KUBERNETES_NAMESPACE + + def get_images(self) -> Dict[str, str]: + vitess_images = self.config_dict.get( + "images", load_system_paasta_config().get_vitess_images() + ) + return { + "vtctld": vitess_images["vtctld_image"], + "vtadmin": vitess_images["vtadmin_image"], + "vtgate": vitess_images["vtgate_image"], + "vttablet": vitess_images["vttablet_image"], + } + + def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: + # get all K8s container env vars and format their keys to camel case + + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + env = [ + api_client.sanitize_for_serialization(env) + for env in self.get_container_env() + ] + return env + + def get_labels(self) -> Dict[str, str]: + # get default labels from parent class to adhere to paasta contract + docker_url = self.get_docker_url( + system_paasta_config=load_system_paasta_config() + ) + git_sha = get_git_sha_from_dockerurl(docker_url) + labels = self.get_kubernetes_metadata(git_sha=git_sha).labels + if "yelp.com/owner" in labels.keys(): + labels["yelp.com/owner"] = "dre_mysql" + return labels + + def get_annotations(self) -> Mapping[str, Any]: + # get required annotations to be added to the formatted resource before creating or updating custom resource + service_namespace_config = load_service_namespace_config( + service=self.service, namespace=self.get_nerve_namespace() + ) + system_paasta_config = load_system_paasta_config() + has_routable_ip = self.has_routable_ip( + service_namespace_config, system_paasta_config + ) + annotations: Mapping[str, Any] = { + "smartstack_registrations": json.dumps(self.get_registrations()), + "paasta.yelp.com/routable_ip": has_routable_ip, + } + return annotations + + def get_vitess_node_affinity(self) -> dict: + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) + return node_affinity + + def get_region(self) -> str: + superregion = self.get_cluster() + superregion_to_region_map = ( + load_system_paasta_config().get_superregion_to_region_mapping() + ) + region = None + for superregion_prefix in superregion_to_region_map: + if superregion.startswith(superregion_prefix): + region = superregion.replace( + superregion_prefix, superregion_to_region_map[superregion_prefix] + ) + if region is None: + log.error( + f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" + ) + # Exiting early here since region is needed to fetch secrets from vault + sys.exit(1) + return region + + def get_global_lock_server(self) -> Dict[str, str]: + zk_address = self.config_dict.get("zk_address") + return { + "implementation": TOPO_IMPLEMENTATION, + "address": zk_address, + "rootPath": TOPO_GLOBAL_ROOT, + } + + def get_update_strategy(self) -> Dict[str, str]: + return {"type": "Immediate"} + + def get_vitess_config(self) -> VitessKeyspaceConfigDict: + cells = self.config_dict.get("cells") + zk_address = self.config_dict.get("zk_address") + region = self.get_region() + keyspace = self.config_dict.get("keyspace") + cluster = self.config_dict.get("cluster") + vttablet_resources = self.config_dict.get("vttablet_resources") + global_lockserver = self.get_global_lock_server() + images = self.get_images() + update_strategy = self.get_update_strategy() + + formatted_env = self.get_env_variables() + labels = self.get_labels() + node_affinity = self.get_vitess_node_affinity() + annotations = self.get_annotations() + + return get_keyspace_config( + cells, + keyspace, + cluster, + vttablet_resources, + images, + update_strategy, + global_lockserver, + zk_address, + region, + formatted_env, + labels, + node_affinity, + annotations, + ) + + def validate( + self, + params: List[str] = [ + "cpus", + "security", + "dependencies_reference", + "deploy_group", + ], + ) -> List[str]: + # Use InstanceConfig to validate shared config keys like cpus and mem + # TODO: add mem back to this list once we fix PAASTA-15582 and + # move to using the same units as flink/marathon etc. + error_msgs = super().validate(params=params) + + if error_msgs: + name = self.get_instance() + return [f"{name}: {msg}" for msg in error_msgs] + else: + return [] + + +def load_vitess_keyspace_instance_config( + service: str, + instance: str, + cluster: str, + load_deployments: bool = True, + soa_dir: str = DEFAULT_SOA_DIR, +) -> VitessKeyspaceConfig: + general_config = service_configuration_lib.read_service_configuration( + service, soa_dir=soa_dir + ) + instance_config = load_service_instance_config( + service, instance, "vitesskeyspace", cluster, soa_dir=soa_dir + ) + general_config = deep_merge_dictionaries( + overrides=instance_config, defaults=general_config + ) + + branch_dict: Optional[BranchDictV2] = None + if load_deployments: + deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) + temp_instance_config = VitessKeyspaceConfig( + service=service, + cluster=cluster, + instance=instance, + config_dict=general_config, + branch_dict=None, + soa_dir=soa_dir, + ) + branch = temp_instance_config.get_branch() + deploy_group = temp_instance_config.get_deploy_group() + branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) + + vitess_keyspace_config = VitessKeyspaceConfig( + service=service, + cluster=cluster, + instance=instance, + config_dict=general_config, + branch_dict=branch_dict, + soa_dir=soa_dir, + ) + + return vitess_keyspace_config + + +def load_vitess_keyspace_instance_configs( + service: str, + instance: str, + cluster: str, + soa_dir: str = DEFAULT_SOA_DIR, +) -> VitessKeyspaceConfigDict: + vitess_keyspace_instance_configs = load_vitess_keyspace_instance_config( + service, instance, cluster, soa_dir=soa_dir + ).get_vitess_config() + return vitess_keyspace_instance_configs + + +# TODO: read this from CRD in service configs +def cr_id(service: str, instance: str) -> Mapping[str, str]: + return dict( + group="planetscale.com", + version="v2", + namespace=KUBERNETES_NAMESPACE, + plural="vitesskeyspaces", + name=sanitised_cr_name(service, instance), + ) From b45e1b75862cbc3cfa714718429380e077f98bb8 Mon Sep 17 00:00:00 2001 From: Vinay Sagar Gonabavi Date: Fri, 23 Aug 2024 21:24:03 -0700 Subject: [PATCH 2/3] Fix tests and reduce code redundancy --- paasta_tools/vitesscell_tools.py | 79 +--- paasta_tools/vitesscluster_tools.py | 82 +++-- paasta_tools/vitesskeyspace_tools.py | 81 +--- tests/test_vitesscluster_tools.py | 529 ++------------------------- 4 files changed, 78 insertions(+), 693 deletions(-) diff --git a/paasta_tools/vitesscell_tools.py b/paasta_tools/vitesscell_tools.py index 25ba088236..0bdb9918f7 100644 --- a/paasta_tools/vitesscell_tools.py +++ b/paasta_tools/vitesscell_tools.py @@ -1,6 +1,4 @@ -import json import logging -import sys from typing import Any from typing import Dict from typing import List @@ -10,16 +8,12 @@ from typing import Union import service_configuration_lib -from kubernetes.client import ApiClient -from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict from paasta_tools.kubernetes_tools import sanitised_cr_name -from paasta_tools.long_running_service_tools import load_service_namespace_config from paasta_tools.utils import BranchDictV2 from paasta_tools.utils import deep_merge_dictionaries from paasta_tools.utils import DEFAULT_SOA_DIR -from paasta_tools.utils import get_git_sha_from_dockerurl from paasta_tools.utils import load_service_instance_config from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import load_v2_deployments_json @@ -28,6 +22,8 @@ from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom from paasta_tools.vitesscluster_tools import RequestsDict from paasta_tools.vitesscluster_tools import ResourceConfigDict +from paasta_tools.vitesscluster_tools import VitessDeploymentConfig +from paasta_tools.vitesscluster_tools import VitessDeploymentConfigDict log = logging.getLogger(__name__) @@ -67,9 +63,8 @@ class GatewayConfigDict(TypedDict, total=False): annotations: Mapping[str, Any] -class VitessCellConfigDict(KubernetesDeploymentConfigDict, total=False): +class VitessCellConfigDict(VitessDeploymentConfigDict, total=False): name: str - images: Dict[str, str] allCells: List[str] globalLockserver: Dict[str, str] gateway: GatewayConfigDict @@ -138,7 +133,7 @@ class VitessCellInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): images: Dict[str, str] -class VitessCellConfig(KubernetesDeploymentConfig): +class VitessCellConfig(VitessDeploymentConfig): config_dict: VitessCellInstanceConfigDict config_filename_prefix = "vitesscell" @@ -161,9 +156,6 @@ def __init__( branch_dict=branch_dict, ) - def get_namespace(self) -> str: - return KUBERNETES_NAMESPACE - def get_images(self) -> Dict[str, str]: vitess_images = self.config_dict.get( "images", load_system_paasta_config().get_vitess_images() @@ -175,69 +167,6 @@ def get_images(self) -> Dict[str, str]: "vttablet": vitess_images["vttablet_image"], } - def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: - # get all K8s container env vars and format their keys to camel case - - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - env = [ - api_client.sanitize_for_serialization(env) - for env in self.get_container_env() - ] - return env - - def get_labels(self) -> Dict[str, str]: - # get default labels from parent class to adhere to paasta contract - docker_url = self.get_docker_url( - system_paasta_config=load_system_paasta_config() - ) - git_sha = get_git_sha_from_dockerurl(docker_url) - labels = self.get_kubernetes_metadata(git_sha=git_sha).labels - if "yelp.com/owner" in labels.keys(): - labels["yelp.com/owner"] = "dre_mysql" - return labels - - def get_annotations(self) -> Mapping[str, Any]: - # get required annotations to be added to the formatted resource before creating or updating custom resource - service_namespace_config = load_service_namespace_config( - service=self.service, namespace=self.get_nerve_namespace() - ) - system_paasta_config = load_system_paasta_config() - has_routable_ip = self.has_routable_ip( - service_namespace_config, system_paasta_config - ) - annotations: Mapping[str, Any] = { - "smartstack_registrations": json.dumps(self.get_registrations()), - "paasta.yelp.com/routable_ip": has_routable_ip, - } - - return annotations - - def get_vitess_node_affinity(self) -> dict: - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) - return node_affinity - - def get_region(self) -> str: - superregion = self.get_cluster() - superregion_to_region_map = ( - load_system_paasta_config().get_superregion_to_region_mapping() - ) - region = None - for superregion_prefix in superregion_to_region_map: - if superregion.startswith(superregion_prefix): - region = superregion.replace( - superregion_prefix, superregion_to_region_map[superregion_prefix] - ) - if region is None: - log.error( - f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" - ) - # Exiting early here since region is needed to fetch secrets from vault - sys.exit(1) - return region - def get_global_lock_server(self) -> Dict[str, str]: zk_address = self.config_dict.get("zk_address") return { diff --git a/paasta_tools/vitesscluster_tools.py b/paasta_tools/vitesscluster_tools.py index cd1543996d..c64797aff7 100644 --- a/paasta_tools/vitesscluster_tools.py +++ b/paasta_tools/vitesscluster_tools.py @@ -224,8 +224,12 @@ def get_vt_admin_config( return config -class VitessClusterConfigDict(KubernetesDeploymentConfigDict, total=False): +class VitessDeploymentConfigDict(KubernetesDeploymentConfigDict, total=False): images: Dict[str, str] + zk_address: str + + +class VitessClusterConfigDict(VitessDeploymentConfigDict, total=False): cells: List[CellConfigDict] vitessDashboard: VitessDashboardConfigDict vtadmin: VtAdminConfigDict @@ -243,43 +247,10 @@ class VitessClusterInstanceConfigDict(KubernetesDeploymentConfigDict, total=Fals images: Dict[str, str] -class VitessClusterConfig(KubernetesDeploymentConfig): - config_dict: VitessClusterInstanceConfigDict - - config_filename_prefix = "vitesscluster" - - def __init__( - self, - service: str, - cluster: str, - instance: str, - config_dict: VitessClusterConfigDict, - branch_dict: Optional[BranchDictV2], - soa_dir: str = DEFAULT_SOA_DIR, - ) -> None: - super().__init__( - cluster=cluster, # superregion - instance=instance, # host-1 - service=service, # vitess - soa_dir=soa_dir, - config_dict=config_dict, - branch_dict=branch_dict, - ) - +class VitessDeploymentConfig(KubernetesDeploymentConfig): def get_namespace(self) -> str: return KUBERNETES_NAMESPACE - def get_images(self) -> Dict[str, str]: - vitess_images = self.config_dict.get( - "images", load_system_paasta_config().get_vitess_images() - ) - return { - "vtctld": vitess_images["vtctld_image"], - "vtadmin": vitess_images["vtadmin_image"], - "vtgate": vitess_images["vtgate_image"], - "vttablet": vitess_images["vttablet_image"], - } - def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: # get all K8s container env vars and format their keys to camel case @@ -343,6 +314,44 @@ def get_region(self) -> str: sys.exit(1) return region + def get_update_strategy(self) -> Dict[str, str]: + return {"type": "Immediate"} + + +class VitessClusterConfig(VitessDeploymentConfig): + config_dict: VitessClusterInstanceConfigDict + + config_filename_prefix = "vitesscluster" + + def __init__( + self, + service: str, + cluster: str, + instance: str, + config_dict: VitessClusterConfigDict, + branch_dict: Optional[BranchDictV2], + soa_dir: str = DEFAULT_SOA_DIR, + ) -> None: + super().__init__( + cluster=cluster, # superregion + instance=instance, # host-1 + service=service, # vitess + soa_dir=soa_dir, + config_dict=config_dict, + branch_dict=branch_dict, + ) + + def get_images(self) -> Dict[str, str]: + vitess_images = self.config_dict.get( + "images", load_system_paasta_config().get_vitess_images() + ) + return { + "vtctld": vitess_images["vtctld_image"], + "vtadmin": vitess_images["vtadmin_image"], + "vtgate": vitess_images["vtgate_image"], + "vttablet": vitess_images["vttablet_image"], + } + def get_global_lock_server(self) -> Dict[str, Dict[str, str]]: zk_address = self.config_dict.get("zk_address") return { @@ -390,9 +399,6 @@ def get_cells(self) -> List[CellConfigDict]: cells = self.config_dict.get("cells") return [get_cell_config(cell) for cell in cells] - def get_update_strategy(self) -> Dict[str, str]: - return {"type": "Immediate"} - def get_vitess_config(self) -> VitessClusterConfigDict: vitess_config = VitessClusterConfigDict( namespace=self.get_namespace(), diff --git a/paasta_tools/vitesskeyspace_tools.py b/paasta_tools/vitesskeyspace_tools.py index d16161d47b..90e3067369 100644 --- a/paasta_tools/vitesskeyspace_tools.py +++ b/paasta_tools/vitesskeyspace_tools.py @@ -1,6 +1,4 @@ -import json import logging -import sys from typing import Any from typing import Dict from typing import List @@ -10,17 +8,13 @@ from typing import Union import service_configuration_lib -from kubernetes.client import ApiClient -from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict from paasta_tools.kubernetes_tools import limit_size_with_hash from paasta_tools.kubernetes_tools import sanitised_cr_name -from paasta_tools.long_running_service_tools import load_service_namespace_config from paasta_tools.utils import BranchDictV2 from paasta_tools.utils import deep_merge_dictionaries from paasta_tools.utils import DEFAULT_SOA_DIR -from paasta_tools.utils import get_git_sha_from_dockerurl from paasta_tools.utils import load_service_instance_config from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import load_v2_deployments_json @@ -29,6 +23,8 @@ from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom from paasta_tools.vitesscluster_tools import RequestsDict from paasta_tools.vitesscluster_tools import ResourceConfigDict +from paasta_tools.vitesscluster_tools import VitessDeploymentConfig +from paasta_tools.vitesscluster_tools import VitessDeploymentConfigDict log = logging.getLogger(__name__) @@ -263,9 +259,8 @@ def get_tablet_pool_config( return config -class VitessKeyspaceConfigDict(KubernetesDeploymentConfigDict, total=False): +class VitessKeyspaceConfigDict(VitessDeploymentConfigDict, total=False): name: str - images: Dict[str, str] databaseName: str durabilityPolicy: str turndownPolicy: str @@ -382,7 +377,7 @@ class VitessKeyspaceInstanceConfigDict(KubernetesDeploymentConfigDict, total=Fal vttablet_resources: ResourceConfigDict -class VitessKeyspaceConfig(KubernetesDeploymentConfig): +class VitessKeyspaceConfig(VitessDeploymentConfig): config_dict: VitessKeyspaceInstanceConfigDict config_filename_prefix = "vitesskeyspace" @@ -405,9 +400,6 @@ def __init__( branch_dict=branch_dict, ) - def get_namespace(self) -> str: - return KUBERNETES_NAMESPACE - def get_images(self) -> Dict[str, str]: vitess_images = self.config_dict.get( "images", load_system_paasta_config().get_vitess_images() @@ -419,68 +411,6 @@ def get_images(self) -> Dict[str, str]: "vttablet": vitess_images["vttablet_image"], } - def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: - # get all K8s container env vars and format their keys to camel case - - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - env = [ - api_client.sanitize_for_serialization(env) - for env in self.get_container_env() - ] - return env - - def get_labels(self) -> Dict[str, str]: - # get default labels from parent class to adhere to paasta contract - docker_url = self.get_docker_url( - system_paasta_config=load_system_paasta_config() - ) - git_sha = get_git_sha_from_dockerurl(docker_url) - labels = self.get_kubernetes_metadata(git_sha=git_sha).labels - if "yelp.com/owner" in labels.keys(): - labels["yelp.com/owner"] = "dre_mysql" - return labels - - def get_annotations(self) -> Mapping[str, Any]: - # get required annotations to be added to the formatted resource before creating or updating custom resource - service_namespace_config = load_service_namespace_config( - service=self.service, namespace=self.get_nerve_namespace() - ) - system_paasta_config = load_system_paasta_config() - has_routable_ip = self.has_routable_ip( - service_namespace_config, system_paasta_config - ) - annotations: Mapping[str, Any] = { - "smartstack_registrations": json.dumps(self.get_registrations()), - "paasta.yelp.com/routable_ip": has_routable_ip, - } - return annotations - - def get_vitess_node_affinity(self) -> dict: - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) - return node_affinity - - def get_region(self) -> str: - superregion = self.get_cluster() - superregion_to_region_map = ( - load_system_paasta_config().get_superregion_to_region_mapping() - ) - region = None - for superregion_prefix in superregion_to_region_map: - if superregion.startswith(superregion_prefix): - region = superregion.replace( - superregion_prefix, superregion_to_region_map[superregion_prefix] - ) - if region is None: - log.error( - f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" - ) - # Exiting early here since region is needed to fetch secrets from vault - sys.exit(1) - return region - def get_global_lock_server(self) -> Dict[str, str]: zk_address = self.config_dict.get("zk_address") return { @@ -489,9 +419,6 @@ def get_global_lock_server(self) -> Dict[str, str]: "rootPath": TOPO_GLOBAL_ROOT, } - def get_update_strategy(self) -> Dict[str, str]: - return {"type": "Immediate"} - def get_vitess_config(self) -> VitessKeyspaceConfigDict: cells = self.config_dict.get("cells") zk_address = self.config_dict.get("zk_address") diff --git a/tests/test_vitesscluster_tools.py b/tests/test_vitesscluster_tools.py index ad483f8913..7ddd99ec52 100644 --- a/tests/test_vitesscluster_tools.py +++ b/tests/test_vitesscluster_tools.py @@ -3,9 +3,9 @@ from kubernetes.client import V1ObjectMeta from paasta_tools.utils import SystemPaastaConfig -from paasta_tools.vitesscluster_tools import load_vitess_instance_config -from paasta_tools.vitesscluster_tools import load_vitess_service_instance_configs -from paasta_tools.vitesscluster_tools import VitessDeploymentConfig +from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_config +from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_configs +from paasta_tools.vitesscluster_tools import VitessClusterConfig CONFIG_DICT = { @@ -20,16 +20,6 @@ "healthcheck_cmd": "fake_cmd", "healthcheck_grace_period_seconds": 60, "healthcheck_mode": "cmd", - "keyspaces": [ - { - "cluster": "fake_cluster", - "keyspace": "fake_keyspaces", - "vttablet_resources": { - "replicas": 1, - "requests": {"cpu": "100m", "memory": "256Mi"}, - }, - } - ], "monitoring": {}, "node_selectors": {"fake_pool": ["fake_pool_value"]}, "paasta_pool": "fake_pool_value", @@ -40,7 +30,6 @@ "requests": {"cpu": "100m", "memory": "256Mi"}, }, "vtctld_resources": {"replicas": 1, "requests": {"cpu": "100m", "memory": "256Mi"}}, - "vtgate_resources": {"replicas": 1, "requests": {"cpu": "100m", "memory": "256Mi"}}, "zk_address": "fake_zk_address", } @@ -57,75 +46,7 @@ VITESS_CONFIG = { "namespace": "paasta-vitessclusters", - "cells": [ - { - "gateway": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "fake_pool", - "operator": "In", - "values": ["fake_pool_value"], - } - ] - } - ] - } - } - }, - "annotations": { - "paasta.yelp.com/routable_ip": "false", - "smartstack_registrations": '["fake_service.fake_instance"]', - }, - "extraEnv": [ - { - "name": "VAULT_ADDR", - "value": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", - }, - { - "name": "VAULT_CACERT", - "value": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", - }, - { - "name": "VAULT_ROLEID", - "valueFrom": { - "secretKeyRef": { - "key": "vault-vtgate-approle-roleid", - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-roleid", - } - }, - }, - { - "name": "VAULT_SECRETID", - "valueFrom": { - "secretKeyRef": { - "key": "vault-vtgate-approle-secretid", - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-secretid", - } - }, - }, - ], - "extraFlags": { - "mysql_auth_server_impl": "vault", - "mysql_auth_vault_addr": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", - "mysql_auth_vault_path": "secrets/vitess/vt-gate/vttablet_credentials.json", - "mysql_auth_vault_tls_ca": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", - "mysql_auth_vault_ttl": "60s", - }, - "extraLabels": {"tablet_type": "fake_keyspaces_migration"}, - "replicas": 1, - "resources": { - "limits": {"cpu": "100m", "memory": "256Mi"}, - "requests": {"cpu": "100m", "memory": "256Mi"}, - }, - }, - "name": "fake_cell", - } - ], + "cells": [{"gateway": {"replicas": 0}, "name": "fake_cell"}], "globalLockserver": { "external": { "address": "fake_zk_address", @@ -139,404 +60,6 @@ "vtgate": "docker-paasta.yelpcorp.com:443/vitess_base:v16.0.3", "vttablet": "docker-paasta.yelpcorp.com:443/vitess_base:v16.0.3", }, - "keyspaces": [ - { - "durabilityPolicy": "none", - "name": "fake_keyspaces", - "partitionings": [ - { - "equal": { - "parts": 1, - "shardTemplate": { - "databaseInitScriptSecret": { - "key": "/etc/init_db.sql", - "volumeName": "keyspace-fake-init-script", - }, - "tabletPools": [ - { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "fake_pool", - "operator": "In", - "values": [ - "fake_pool_value" - ], - } - ] - } - ] - } - } - }, - "annotations": { - "paasta.yelp.com/routable_ip": "false", - "smartstack_registrations": '["fake_service.fake_instance"]', - }, - "cell": "fake_cell", - "dataVolumeClaimTemplate": { - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - "storageClassName": "ebs-csi-gp3", - }, - "externalDatastore": { - "credentialsSecret": { - "key": "/etc/credentials.yaml", - "volumeName": "vttablet-fake-credentials", - }, - "database": "fake_keyspaces", - "host": "mysql-fake_cluster-primary.dre-devc", - "port": 3306, - "user": "vt_app", - }, - "extraEnv": [ - { - "name": "VAULT_ADDR", - "value": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", - }, - { - "name": "VAULT_CACERT", - "value": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", - }, - { - "name": "TOPOLOGY_FLAGS", - "value": "--topo_implementation " - "zk2 " - "--topo_global_server_address " - "$fake_zk_address " - "--topo_global_root " - "/vitess-paasta/global", - }, - { - "name": "CELL_TOPOLOGY_SERVERS", - "value": "fake_zk_address", - }, - {"name": "DB", "value": "fake_keyspaces"}, - {"name": "KEYSPACE", "value": "fake_keyspaces"}, - {"name": "WEB_PORT", "value": "15000"}, - {"name": "GRPC_PORT", "value": "15999"}, - {"name": "SHARD", "value": "0"}, - {"name": "EXTERNAL_DB", "value": "1"}, - {"name": "ROLE", "value": "replica"}, - { - "name": "VAULT_ROLEID", - "valueFrom": { - "secretKeyRef": { - "key": "vault-vttablet-approle-roleid", - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", - } - }, - }, - { - "name": "VAULT_SECRETID", - "valueFrom": { - "secretKeyRef": { - "key": "vault-vttablet-approle-secretid", - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", - } - }, - }, - ], - "extraLabels": { - "tablet_type": "fake_keyspaces_migration" - }, - "extraVolumeMounts": [ - { - "mountPath": "/etc/vault/all_cas", - "name": "vault-secrets", - "readOnly": True, - }, - { - "mountPath": "/nail/srv", - "name": "srv-configs", - "readOnly": True, - }, - { - "mountPath": "/nail/etc/srv-configs", - "name": "etc-srv-configs", - "readOnly": True, - }, - { - "mountPath": "etc/credentials.yaml", - "name": "vttablet-fake-credentials", - "readOnly": True, - }, - { - "mountPath": "/etc/init_db.sql", - "name": "keyspace-fake-init-script", - "readOnly": True, - }, - ], - "extraVolumes": [ - { - "hostPath": { - "path": "/nail/etc/vault/all_cas" - }, - "name": "vault-secrets", - }, - { - "name": "srv-configs", - "hostPath": {"path": "/nail/srv"}, - }, - { - "name": "etc-srv-configs", - "hostPath": { - "path": "/nail/etc/srv-configs" - }, - }, - { - "hostPath": {"path": "/dev/null"}, - "name": "vttablet-fake-credentials", - }, - { - "hostPath": {"path": "/dev/null"}, - "name": "keyspace-fake-init-script", - }, - ], - "name": "fake_keyspaces_primary", - "replicas": 1, - "type": "externalmaster", - "vttablet": { - "extraFlags": { - "db-credentials-server": "vault", - "db-credentials-vault-addr": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", - "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", - "db-credentials-vault-tls-ca": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", - "db-credentials-vault-ttl": "60s", - "db_charset": "utf8mb4", - "dba_pool_size": "4", - "disable_active_reparents": "true", - "enable-lag-throttler": "true", - "enforce-tableacl-config": "true", - "grpc_max_message_size": "134217728", - "init_tablet_type": "replica", - "keep_logs": "72h", - "log_err_stacks": "true", - "queryserver-config-schema-reload-time": "1800", - "queryserver-config-strict-table-acl": "true", - "table-acl-config": "/nail/srv/configs/vitess_keyspace_acls/acls_for_fake_keyspaces.json", - "table-acl-config-reload-interval": "60s", - "throttle_check_as_check_self": "true", - "throttle_metrics_query": "select " - "max_replication_delay " - "from " - "max_mysql_replication_delay.read_replication_delay;", - "throttle_metrics_threshold": "3", - "vreplication_heartbeat_update_interval": "60", - "vreplication_tablet_type": "REPLICA", - }, - "resources": { - "limits": { - "cpu": "100m", - "memory": "256Mi", - }, - "requests": { - "cpu": "100m", - "memory": "256Mi", - }, - }, - }, - }, - { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "fake_pool", - "operator": "In", - "values": [ - "fake_pool_value" - ], - } - ] - } - ] - } - } - }, - "annotations": { - "paasta.yelp.com/routable_ip": "false", - "smartstack_registrations": '["fake_service.fake_instance"]', - }, - "cell": "fake_cell", - "dataVolumeClaimTemplate": { - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - "storageClassName": "ebs-csi-gp3", - }, - "externalDatastore": { - "credentialsSecret": { - "key": "/etc/credentials.yaml", - "volumeName": "vttablet-fake-credentials", - }, - "database": "fake_keyspaces", - "host": "mysql-fake_cluster-migration.dre-devc", - "port": 3306, - "user": "vt_app", - }, - "extraEnv": [ - { - "name": "VAULT_ADDR", - "value": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", - }, - { - "name": "VAULT_CACERT", - "value": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", - }, - { - "name": "TOPOLOGY_FLAGS", - "value": "--topo_implementation " - "zk2 " - "--topo_global_server_address " - "$fake_zk_address " - "--topo_global_root " - "/vitess-paasta/global", - }, - { - "name": "CELL_TOPOLOGY_SERVERS", - "value": "fake_zk_address", - }, - {"name": "DB", "value": "fake_keyspaces"}, - {"name": "KEYSPACE", "value": "fake_keyspaces"}, - {"name": "WEB_PORT", "value": "15000"}, - {"name": "GRPC_PORT", "value": "15999"}, - {"name": "SHARD", "value": "0"}, - {"name": "EXTERNAL_DB", "value": "1"}, - {"name": "ROLE", "value": "replica"}, - { - "name": "VAULT_ROLEID", - "valueFrom": { - "secretKeyRef": { - "key": "vault-vttablet-approle-roleid", - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", - } - }, - }, - { - "name": "VAULT_SECRETID", - "valueFrom": { - "secretKeyRef": { - "key": "vault-vttablet-approle-secretid", - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", - } - }, - }, - ], - "extraLabels": { - "tablet_type": "fake_keyspaces_migration" - }, - "extraVolumeMounts": [ - { - "mountPath": "/etc/vault/all_cas", - "name": "vault-secrets", - "readOnly": True, - }, - { - "mountPath": "/nail/srv", - "name": "srv-configs", - "readOnly": True, - }, - { - "mountPath": "/nail/etc/srv-configs", - "name": "etc-srv-configs", - "readOnly": True, - }, - { - "mountPath": "etc/credentials.yaml", - "name": "vttablet-fake-credentials", - "readOnly": True, - }, - { - "mountPath": "/etc/init_db.sql", - "name": "keyspace-fake-init-script", - "readOnly": True, - }, - ], - "extraVolumes": [ - { - "hostPath": { - "path": "/nail/etc/vault/all_cas" - }, - "name": "vault-secrets", - }, - { - "name": "srv-configs", - "hostPath": {"path": "/nail/srv"}, - }, - { - "name": "etc-srv-configs", - "hostPath": { - "path": "/nail/etc/srv-configs" - }, - }, - { - "hostPath": {"path": "/dev/null"}, - "name": "vttablet-fake-credentials", - }, - { - "hostPath": {"path": "/dev/null"}, - "name": "keyspace-fake-init-script", - }, - ], - "name": "fake_keyspaces_migration", - "replicas": 1, - "type": "externalreplica", - "vttablet": { - "extraFlags": { - "db-credentials-server": "vault", - "db-credentials-vault-addr": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", - "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", - "db-credentials-vault-tls-ca": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", - "db-credentials-vault-ttl": "60s", - "db_charset": "utf8mb4", - "dba_pool_size": "4", - "disable_active_reparents": "true", - "enable-lag-throttler": "true", - "enforce-tableacl-config": "true", - "grpc_max_message_size": "134217728", - "init_tablet_type": "replica", - "keep_logs": "72h", - "log_err_stacks": "true", - "queryserver-config-schema-reload-time": "1800", - "queryserver-config-strict-table-acl": "true", - "table-acl-config": "/nail/srv/configs/vitess_keyspace_acls/acls_for_fake_keyspaces.json", - "table-acl-config-reload-interval": "60s", - "throttle_check_as_check_self": "true", - "throttle_metrics_query": "select " - "max_replication_delay " - "from " - "max_mysql_replication_delay.migration_replication_delay;", - "throttle_metrics_threshold": "3", - "vreplication_heartbeat_update_interval": "60", - "vreplication_tablet_type": "REPLICA", - }, - "resources": { - "limits": { - "cpu": "100m", - "memory": "256Mi", - }, - "requests": { - "cpu": "100m", - "memory": "256Mi", - }, - }, - }, - }, - ], - }, - } - } - ], - "turndownPolicy": "Immediate", - } - ], "updateStrategy": {"type": "Immediate"}, "vitessDashboard": { "affinity": { @@ -577,7 +100,7 @@ "disable_active_reparents": "true", "security_policy": "read-only", }, - "extraLabels": {"tablet_type": "fake_keyspaces_migration"}, + "extraLabels": {}, "replicas": 1, "resources": { "limits": {"cpu": "100m", "memory": "256Mi"}, @@ -614,7 +137,7 @@ "cells": ["fake_cell"], "extraEnv": [], "extraFlags": {"grpc-allow-reflection": "true"}, - "extraLabels": {"tablet_type": "fake_keyspaces_migration"}, + "extraLabels": {}, "readOnly": False, "replicas": 1, "webResources": { @@ -626,17 +149,17 @@ @pytest.fixture -def mock_vitess_deployment_config(): +def mock_vitess_cluster_config(): with mock.patch.object( - VitessDeploymentConfig, "get_region", return_value="mock_region-devc" + VitessClusterConfig, "get_region", return_value="mock_region-devc" ), mock.patch.object( - VitessDeploymentConfig, "get_container_env", return_value=[] + VitessClusterConfig, "get_container_env", return_value=[] ), mock.patch.object( - VitessDeploymentConfig, "get_docker_url", return_value="fake_docker_url" + VitessClusterConfig, "get_docker_url", return_value="fake_docker_url" ), mock.patch.object( - VitessDeploymentConfig, "get_cluster", return_value="fake_superregion" + VitessClusterConfig, "get_cluster", return_value="fake_superregion" ), mock.patch.object( - VitessDeploymentConfig, + VitessClusterConfig, "get_kubernetes_metadata", return_value=V1ObjectMeta(labels={}), ): @@ -644,19 +167,19 @@ def mock_vitess_deployment_config(): @mock.patch( - "paasta_tools.vitesscluster_tools.load_vitess_instance_config", + "paasta_tools.vitesscluster_tools.load_vitess_cluster_instance_config", autospec=True, ) @mock.patch( "paasta_tools.vitesscluster_tools.load_system_paasta_config", autospec=True, ) -def test_load_vitess_service_instance_configs( +def test_load_vitess_cluster_instance_configs( mock_load_system_paasta_config, - mock_load_vitess_instance_config, - mock_vitess_deployment_config, + mock_load_vitess_cluster_instance_config, + mock_vitess_cluster_config, ): - mock_load_vitess_instance_config.return_value = VitessDeploymentConfig( + mock_load_vitess_cluster_instance_config.return_value = VitessClusterConfig( service="fake_service", instance="fake_instance", cluster="fake_cluster", @@ -665,13 +188,13 @@ def test_load_vitess_service_instance_configs( soa_dir="fake_soa_dir", ) mock_load_system_paasta_config.return_value = MOCK_SYSTEM_PAASTA_CONFIG - vitess_service_instance_configs = load_vitess_service_instance_configs( + vitess_cluster_instance_configs = load_vitess_cluster_instance_configs( service="fake_service", soa_dir="fake_soa_dir", cluster="fake_cluster", instance="fake_instance", ) - assert vitess_service_instance_configs == VITESS_CONFIG + assert vitess_cluster_instance_configs == VITESS_CONFIG @mock.patch("paasta_tools.vitesscluster_tools.load_v2_deployments_json", autospec=True) @@ -680,12 +203,12 @@ def test_load_vitess_service_instance_configs( autospec=True, ) @mock.patch( - "paasta_tools.vitesscluster_tools.VitessDeploymentConfig", + "paasta_tools.vitesscluster_tools.VitessClusterConfig", autospec=True, ) -def test_load_vitess_instance_config( - mock_vitess_deployment_config, - mock_load_service_instance_config, +def test_load_vitess_cluster_instance_config( + mock_vitess_cluster_config, + mock_load_cluster_instance_config, mock_load_v2_deployments_json, ): mock_config = { @@ -696,7 +219,7 @@ def test_load_vitess_instance_config( "smartstack": {}, "dependencies": {}, } - vitess_deployment_config = load_vitess_instance_config( + vitess_cluster_config = load_vitess_cluster_instance_config( service="fake_vitesscluster_service", instance="fake_instance", cluster="fake_cluster", @@ -706,7 +229,7 @@ def test_load_vitess_instance_config( mock_load_v2_deployments_json.assert_called_with( service="fake_vitesscluster_service", soa_dir="/foo/bar" ) - mock_vitess_deployment_config.assert_called_with( + mock_vitess_cluster_config.assert_called_with( service="fake_vitesscluster_service", instance="fake_instance", cluster="fake_cluster", @@ -715,4 +238,4 @@ def test_load_vitess_instance_config( soa_dir="/foo/bar", ) - assert vitess_deployment_config == mock_vitess_deployment_config.return_value + assert vitess_cluster_config == mock_vitess_cluster_config.return_value From 4872e2ee0aaaf955dec41b8388dbec98fba8f472 Mon Sep 17 00:00:00 2001 From: Vinay Sagar Gonabavi Date: Mon, 26 Aug 2024 09:08:34 -0700 Subject: [PATCH 3/3] Fix a test failure --- tests/test_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 12b8442fbf..47830d1ae9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2363,6 +2363,8 @@ def test_validate_service_instance_invalid(): mock_nrtsearcheks_instances = [("service1", "nrtsearcheks")] mock_monkrelaycluster_instances = [("service1", "monkrelays")] mock_vitesscluster_instances = [("service1", "vitesscluster")] + mock_vitesscell_instances = [("service1", "vitesscell")] + mock_vitesskeyspace_instances = [("service1", "vitesskeyspace")] my_service = "service1" my_instance = "main" fake_cluster = "fake_cluster" @@ -2385,6 +2387,8 @@ def test_validate_service_instance_invalid(): mock_nrtsearcheks_instances, mock_monkrelaycluster_instances, mock_vitesscluster_instances, + mock_vitesscell_instances, + mock_vitesskeyspace_instances, ], ): with raises(