diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index 977d286bb..8f84257ee 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -25,7 +25,7 @@ def run(args): cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args) if env["hub"]: - hub_cm = generate_config_map("hub", env["clusters"], args) + hub_cm = generate_config_map("hub", env, args) wait_for_ramen_hub_operator(env["hub"], args) @@ -38,7 +38,7 @@ def run(args): wait_for_dr_clusters(env["hub"], env["clusters"], args) wait_for_dr_policy(env["hub"], args) else: - dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args) + dr_cluster_cm = generate_config_map("dr-cluster", env, args) for cluster in env["clusters"]: create_ramen_s3_secrets(cluster, s3_secrets) @@ -89,7 +89,9 @@ def create_cloud_credentials_secret(cluster, yaml): kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug) -def generate_config_map(controller, clusters, args): +def generate_config_map(controller, env, args): + clusters = env["clusters"] + volsync = env["features"].get("volsync", True) template = drenv.template(command.resource("configmap.yaml")) return template.substitute( name=f"ramen-{controller}-operator-config", @@ -98,6 +100,7 @@ def generate_config_map(controller, clusters, args): cluster2=clusters[1], minio_url_cluster1=minio.service_url(clusters[0]), minio_url_cluster2=minio.service_url(clusters[1]), + volsync_disabled="false" if volsync else "true", namespace=args.ramen_namespace, ) diff --git a/ramenctl/ramenctl/resources/configmap.yaml b/ramenctl/ramenctl/resources/configmap.yaml index c82136f1f..53b2b7fb4 100644 --- a/ramenctl/ramenctl/resources/configmap.yaml +++ b/ramenctl/ramenctl/resources/configmap.yaml @@ -34,6 +34,8 @@ data: clusterServiceVersionName: ramen-dr-cluster-operator.v0.0.1 kubeObjectProtection: veleroNamespaceName: velero + volSync: + disabled: $volsync_disabled s3StoreProfiles: - s3ProfileName: minio-on-$cluster1 s3Bucket: bucket diff --git a/test/addons/cdi/cr/kustomization.yaml b/test/addons/cdi/cr/kustomization.yaml index 3fd3d08eb..7763740f7 100644 --- a/test/addons/cdi/cr/kustomization.yaml +++ b/test/addons/cdi/cr/kustomization.yaml @@ -4,4 +4,34 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.57.0/cdi-cr.yaml + - https://github.com/kubevirt/containerized-data-importer/releases/download/v1.58.1/cdi-cr.yaml +patches: + # Allow pulling from local insecure registry. + - target: + kind: CDI + name: cdi + patch: |- + apiVersion: cdi.kubevirt.io/v1beta1 + kind: CDI + metadata: + name: not-used + spec: + config: + insecureRegistries: + - host.minikube.internal:5000 + # Incrase certificate duration to avoid certificates renewals while a cluster + # is suspended and resumed. + - target: + kind: CDI + name: cdi + patch: |- + apiVersion: cdi.kubevirt.io/v1beta1 + kind: CDI + metadata: + name: not-used + spec: + certConfig: + ca: + duration: 168h + server: + duration: 168h diff --git a/test/addons/cdi/disk/source.yaml b/test/addons/cdi/disk/source.yaml index cdd40c064..c320d6454 100644 --- a/test/addons/cdi/disk/source.yaml +++ b/test/addons/cdi/disk/source.yaml @@ -9,4 +9,4 @@ metadata: spec: source: registry: - url: "docker://quay.io/alitke/cirros:latest" + url: "docker://quay.io/nirsof/cirros:0.6.2-1" diff --git a/test/addons/cdi/operator/kustomization.yaml b/test/addons/cdi/operator/kustomization.yaml index b894cb979..10ff5811f 100644 --- a/test/addons/cdi/operator/kustomization.yaml +++ b/test/addons/cdi/operator/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.57.0/cdi-operator.yaml +- https://github.com/kubevirt/containerized-data-importer/releases/download/v1.58.1/cdi-operator.yaml diff --git a/test/addons/cdi/start b/test/addons/cdi/start index f9bbf0fa0..86dde20bd 100755 --- a/test/addons/cdi/start +++ b/test/addons/cdi/start @@ -37,6 +37,14 @@ def wait(cluster): "--timeout=600s", context=cluster, ) + print("Waiting until cdi cr finished progressing") + kubectl.wait( + "cdi.cdi.kubevirt.io/cdi", + "--for=condition=progressing=False", + f"--namespace={NAMESPACE}", + "--timeout=300s", + context=cluster, + ) if len(sys.argv) != 2: diff --git a/test/addons/kubevirt/cr/kustomization.yaml b/test/addons/kubevirt/cr/kustomization.yaml index 2f02b94a9..71ea633af 100644 --- a/test/addons/kubevirt/cr/kustomization.yaml +++ b/test/addons/kubevirt/cr/kustomization.yaml @@ -4,4 +4,22 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/kubevirt/releases/download/v1.0.1/kubevirt-cr.yaml + - https://github.com/kubevirt/kubevirt/releases/download/v1.2.0/kubevirt-cr.yaml +patches: + # Incrase certificate duration to avoid certificates renewals while a cluster + # is suspended and resumed. + - target: + kind: KubeVirt + name: kubevirt + patch: |- + apiVersion: kubevirt.io/v1 + kind: Kubevirt + metadata: + name: not-used + spec: + certificateRotateStrategy: + selfSigned: + ca: + duration: 168h + server: + duration: 168h diff --git a/test/addons/kubevirt/operator/kustomization.yaml b/test/addons/kubevirt/operator/kustomization.yaml index 2846de42f..688334562 100644 --- a/test/addons/kubevirt/operator/kustomization.yaml +++ b/test/addons/kubevirt/operator/kustomization.yaml @@ -4,4 +4,4 @@ # yamllint disable rule:line-length --- resources: -- https://github.com/kubevirt/kubevirt/releases/download/v1.0.1/kubevirt-operator.yaml +- https://github.com/kubevirt/kubevirt/releases/download/v1.2.0/kubevirt-operator.yaml diff --git a/test/configs/kubevirt/vm-pvc-k8s-regional.yaml b/test/configs/kubevirt/vm-pvc-k8s-regional.yaml new file mode 100644 index 000000000..f3a07827e --- /dev/null +++ b/test/configs/kubevirt/vm-pvc-k8s-regional.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +repo: https://github.com/ramendr/ocm-ramen-samples.git +path: subscription/kubevirt/vm-pvc-k8s-regional +branch: main +name: vm-pvc +namespace: vm-pvc +dr_policy: dr-policy +pvc_label: vm diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index a7b181901..cad6c86d2 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -30,7 +30,16 @@ def main(): p = argparse.ArgumentParser(prog="drenv") p.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") p.add_argument( - "--skip-tests", dest="run_tests", action="store_false", help="Skip self tests" + "--skip-tests", + dest="run_tests", + action="store_false", + help="Skip addons 'test' hooks", + ) + p.add_argument( + "--skip-addons", + dest="run_addons", + action="store_false", + help="Skip addons 'start' and 'stop' hooks", ) p.add_argument("command", choices=commands, help="Command to run") p.add_argument("--name-prefix", help="Prefix profile names") @@ -57,7 +66,12 @@ def main(): def cmd_start(env, args): start = time.monotonic() logging.info("[%s] Starting environment", env["name"]) - hooks = ["start", "test"] if args.run_tests else ["start"] + + hooks = [] + if args.run_addons: + hooks.append("start") + if args.run_tests: + hooks.append("test") # Delaying `minikube start` ensures cluster start order. execute( @@ -67,7 +81,9 @@ def cmd_start(env, args): hooks=hooks, args=args, ) - execute(run_worker, env["workers"], hooks=hooks) + + if hooks: + execute(run_worker, env["workers"], hooks=hooks) if "ramen" in env: ramen.dump_e2e_config(env) @@ -82,7 +98,8 @@ def cmd_start(env, args): def cmd_stop(env, args): start = time.monotonic() logging.info("[%s] Stopping environment", env["name"]) - execute(stop_cluster, env["profiles"]) + hooks = ["stop"] if args.run_addons else [] + execute(stop_cluster, env["profiles"], hooks=hooks) logging.info( "[%s] Environment stopped in %.2f seconds", env["name"], @@ -107,6 +124,18 @@ def cmd_delete(env, args): ) +def cmd_suspend(env, args): + logging.info("[%s] Suspending environment", env["name"]) + for profile in env["profiles"]: + run("virsh", "-c", "qemu:///system", "suspend", profile["name"]) + + +def cmd_resume(env, args): + logging.info("[%s] Resuming environment", env["name"]) + for profile in env["profiles"]: + run("virsh", "-c", "qemu:///system", "resume", profile["name"]) + + def cmd_dump(env, args): yaml.dump(env, sys.stdout) @@ -152,17 +181,23 @@ def start_cluster(profile, hooks=(), args=None, **options): if is_restart: wait_for_deployments(profile) - execute(run_worker, profile["workers"], max_workers=args.max_workers, hooks=hooks) + if hooks: + execute( + run_worker, + profile["workers"], + max_workers=args.max_workers, + hooks=hooks, + ) -def stop_cluster(profile, **options): +def stop_cluster(profile, hooks=(), **options): cluster_status = cluster.status(profile["name"]) - if cluster_status == cluster.READY: + if cluster_status == cluster.READY and hooks: execute( run_worker, profile["workers"], - hooks=["stop"], + hooks=hooks, reverse=True, allow_failure=True, ) diff --git a/test/envs/regional-dr-hubless.yaml b/test/envs/regional-dr-hubless.yaml index fff2a57f7..2928f7747 100644 --- a/test/envs/regional-dr-hubless.yaml +++ b/test/envs/regional-dr-hubless.yaml @@ -9,6 +9,8 @@ ramen: hub: null clusters: [dr1, dr2] topology: regional-dr + features: + volsync: true templates: - name: "dr-cluster" diff --git a/test/envs/regional-dr-kubevirt.yaml b/test/envs/regional-dr-kubevirt.yaml index eeb7fdbce..a45eca833 100644 --- a/test/envs/regional-dr-kubevirt.yaml +++ b/test/envs/regional-dr-kubevirt.yaml @@ -9,6 +9,8 @@ ramen: hub: hub clusters: [dr1, dr2] topology: regional-dr + features: + volsync: false templates: - name: "dr-cluster" @@ -25,7 +27,6 @@ templates: extra_disks: 1 disk_size: "50g" addons: - - volumesnapshots - csi-hostpath-driver workers: - addons: @@ -57,8 +58,6 @@ templates: - name: ocm-controller - name: cert-manager - name: olm - - name: submariner - args: ["hub", "dr1", "dr2"] profiles: - name: "dr1" @@ -72,6 +71,3 @@ workers: - addons: - name: rbd-mirror args: ["dr1", "dr2"] - - addons: - - name: volsync - args: ["dr1", "dr2"] diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 516ae921c..74c164662 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -9,6 +9,8 @@ ramen: hub: hub clusters: [dr1, dr2] topology: regional-dr + features: + volsync: true templates: - name: "dr-cluster" diff --git a/test/gitlap/README.md b/test/gitlap/README.md new file mode 100644 index 000000000..d71460c68 --- /dev/null +++ b/test/gitlap/README.md @@ -0,0 +1,88 @@ +# Setting up a local git server + +## Initial setup + +1. Install lighttpd + + ``` + sudo dnf install lighttpd + ``` + +1. Create the git repo + + Create a directory where the git repositories will be served: + + ``` + sudo mkdir /var/www/gitlap + cd /var/www/gitlap + sudo git clone --bare https://github.com/nirs/ocm-kubevirt-samples.git + ``` + + Set git repo permissions so you can push changes, and the web server + can serve the repo. + + ``` + sudo chown -R $USER:lighttpd /var/www/gitlap + ``` + +1. Copy the vhost configuration + + ``` + sudo cp gitlap.conf /etc/lighttpd/vhosts.d/ + ``` + +1. Uncomment the vhost include in /etc/lighttpd/lighttpd.conf + + ``` + include conf_dir + "/vhosts.d/*.conf" + ``` + +1. Enable and start the service + + ``` + sudo systemctl enable --now lighttpd + ``` + +1. Allow http access in the libvirt zone + + ``` + sudo firewall-cmd --zone=libvirt --add-service=http --permanent + sudo firewall-cmd --reload + ``` + +## Testing the server + +1. Add entry in /etc/hosts for testing locally + + ``` + 192.168.122.1 host.minikube.internal + ``` + +1. Check that git clone works + + ``` + git clone http://host.minikube.internal/ocm-kubevirt-samples.git + rm -rf ocm-kubevirt-samples + ``` + +1. Check git clone in a minikube cluster + + ``` + minikube ssh -p dr1 + git clone http://host.minikube.internal/ocm-kubevirt-samples.git + rm -rf ocm-kubevirt-samples + ``` + +## Updating the git repo + +1. Add a remote to your working repo + + ``` + git remote add gitlap file:///var/www/gitlap/ocm-kubevirt-samples.git + ``` + +1. Push changes to the remote + + ``` + git push -f gitlap main + ``` diff --git a/test/gitlap/gitlap.conf b/test/gitlap/gitlap.conf new file mode 100644 index 000000000..bfe98fc70 --- /dev/null +++ b/test/gitlap/gitlap.conf @@ -0,0 +1,17 @@ +# Minimal configuration for local git server for minikube clusters. +# +# For more options see: +# https://redmine.lighttpd.net/projects/lighttpd/wiki/How_to_set_up_a_git_server_over_http(s) + +server.modules += ("mod_setenv", "mod_cgi", "mod_alias") + +# `host.minikube.internal` is a special DNS name injected by minikube to all clusters. +# https://minikube.sigs.k8s.io/docs/handbook/host-access/ +$HTTP["host"] == "host.minikube.internal" { + alias.url = ( "" => "/usr/libexec/git-core/git-http-backend" ) + setenv.set-environment = ( + "GIT_PROJECT_ROOT" => "/var/www/gitlap/", + "GIT_HTTP_EXPORT_ALL" => "1" + ) + cgi.assign = ( "" => "" ) +} diff --git a/test/registry/README.md b/test/registry/README.md new file mode 100644 index 000000000..6e3c2ea3e --- /dev/null +++ b/test/registry/README.md @@ -0,0 +1,90 @@ +# Using local registry for minikube clusters + +## Initial setup + +1. Install podman + + ``` + sudo dnf install podman + ``` + +1. Run the registry container + + ``` + podman run --name registry \ + --publish 5000:5000 \ + --volume registry:/var/lib/registry:Z \ + --detach \ + --replace \ + docker.io/library/registry:latest + ``` + + Use `--replace` to replace an existing container, typically left + after reboot the host. + + To run the registry as system service see + [systemd service](#systemd-service). + +1. Allow access to port 5000 in the libvirt zone + + ``` + sudo firewall-cmd --zone=libvirt --add-port=5000/tcp --permanent + sudo firewall-cmd --reload + ``` + +1. Configure podman to allow insecure access + + ``` + sudo cp host.minikube.internal.conf /etc/containers/registries.conf.d/ + ``` + +1. Testing the registry + + ``` + $ curl host.minikube.internal:5000/v2/_catalog + {} + ``` + +## Pushing to the local registry + +1. Pull the image from a remote registry + + ``` + podman pull quay.io/nirsof/cirros:0.6.2-1 + ``` + +1. Push to the local registry + + ``` + podman push quay.io/nirsof/cirros:0.6.2-1 host.minikube.internal:5000/nirsof/cirros:0.6.2-1 + ``` + +## Using images from the local registry + +Example source.yaml: + +``` +--- +apiVersion: cdi.kubevirt.io/v1beta1 +kind: VolumeImportSource +metadata: + name: cirros-source +spec: + source: + registry: + url: "docker://host.minikube.internal:5000/nirsof/cirros:0.6.2-1" +``` + +## Systemd service + +To create a registry service running at boot, install the provided +systemd units and start the service. + +``` +sudo cp systemd/registry.* /etc/containers/systemd/ +sudo systemctl daemon-reload +sudo systemctl start registry.service +``` + +> [!NOTE] +> The service does not need to be enabled. diff --git a/test/registry/host.minikube.internal.conf b/test/registry/host.minikube.internal.conf new file mode 100644 index 000000000..03bc08aa3 --- /dev/null +++ b/test/registry/host.minikube.internal.conf @@ -0,0 +1,4 @@ +# Local registry for serving minikube clusters +[[registry]] +location = 'host.minikube.internal:5000' +insecure = true diff --git a/test/registry/systemd/registry.container b/test/registry/systemd/registry.container new file mode 100644 index 000000000..e7c63bbfe --- /dev/null +++ b/test/registry/systemd/registry.container @@ -0,0 +1,12 @@ +[Unit] +Description=Registry container + +[Container] +Label=app=registry +ContainerName=registry +Image=docker.io/library/registry:latest +PublishPort=5000:5000 +Volume=registry.volume:/var/lib/registry + +[Install] +WantedBy=multi-user.target default.target diff --git a/test/registry/systemd/registry.volume b/test/registry/systemd/registry.volume new file mode 100644 index 000000000..d256357a6 --- /dev/null +++ b/test/registry/systemd/registry.volume @@ -0,0 +1,5 @@ +[Unit] +Description=Registry Volume + +[Volume] +Label=app=registry