enable active passive tests in daily runs (#987)

* re-enable Active Passive in the ROSA scheduled run Signed-off-by: Kamesh Akella <[email protected]> * Fix transaction mode Signed-off-by: Michal Hajas <[email protected]> * Propagate information about active/passive usage to Horreum Signed-off-by: Michal Hajas <[email protected]> * Fix active/passive failover test Signed-off-by: Michal Hajas <[email protected]> * Increase number of users to 100k Signed-off-by: Michal Hajas <[email protected]> --------- Signed-off-by: Kamesh Akella <[email protected]> Signed-off-by: Michal Hajas <[email protected]> Co-authored-by: Michal Hajas <[email protected]>
keycloak · Sep 25, 2024 · df392e3 · df392e3
1 parent 8eaa7b4
commit df392e3
Show file tree

Hide file tree

Showing 11 changed files with 105 additions and 46 deletions.
diff --git a/.github/actions/get-keycloak-url/action.yml b/.github/actions/get-keycloak-url/action.yml
@@ -14,11 +14,6 @@ runs:
       run: |
         KEYCLOAK_URL=https://$(kubectl get routes -n "${{ inputs.project }}" -l app=keycloak -o jsonpath='{.items[*].spec.host}')
         echo "KEYCLOAK_URL=$KEYCLOAK_URL" >> "$GITHUB_ENV"
-    - id: get-keycloak-site-url
-      shell: bash
-      run: |
-        KEYCLOAK_SITE_URL=https://$(kubectl -n "${{ inputs.project }}" get svc accelerator-loadbalancer --template="{{range .status.loadBalancer.ingress}}{{.hostname}}{{end}}")
-        echo "KEYCLOAK_SITE_URL=$KEYCLOAK_SITE_URL" >> "$GITHUB_ENV"
     - id: get-ispn-rest-url
       shell: bash
       run: |

diff --git a/.github/actions/keycloak-create-deployment/action.yml b/.github/actions/keycloak-create-deployment/action.yml
@@ -12,7 +12,7 @@ inputs:
   disableStickySessions:
     description: 'Disable sticky session in OpenShift Route'
     default: 'true'
-  enableKc25Mode:
+  isActivePassive:
     description: 'Set to true when version older than 26 is deployed'
     default: 'false'
   enableExternalInfinispanFeature:
@@ -58,7 +58,7 @@ runs:
         KC_NAMESPACE_PREFIX: ${{ inputs.projectPrefix }}
         KC_INSTANCES: ${{ inputs.replicas }}
         KC_DISABLE_STICKY_SESSION: ${{ inputs.disableStickySessions }}
-        KC_KC25_MODE: ${{ inputs.enableKc25Mode }}
+        KC_IS_ACTIVE_PASSIVE: ${{ inputs.isActivePassive }}
         KC_EXTERNAL_INFINISPAN: ${{ inputs.enableExternalInfinispanFeature }}
         KC_MULTI_SITE: ${{ inputs.enableMultiSiteFeature }}
         KC_CONTAINER_IMAGE: ${{ inputs.image }}

diff --git a/.github/workflows/keycloak-multi-site-health-check.yml b/.github/workflows/keycloak-multi-site-health-check.yml
@@ -11,6 +11,10 @@ on:
         required: true
       region:
         description: 'The AWS region to create both clusters in. Defaults to "vars.AWS_DEFAULT_REGION" if omitted.'
+      activeActive:
+        description: 'When true deploy an Active/Active Keycloak deployment'
+        type: boolean
+        default: false
       expectedInfinispanNodeCount:
         description: 'Expected Infinispan cluster node count'
         default: '3'
@@ -27,6 +31,10 @@ on:
       region:
         description: 'The AWS region to create both clusters in. Defaults to "vars.AWS_DEFAULT_REGION" if omitted.'
         type: string
+      activeActive:
+        description: 'When true deploy an Active/Active Keycloak deployment'
+        type: boolean
+        default: false
       expectedInfinispanNodeCount:
         description: 'Expected Infinispan cluster node count'
         type: string
@@ -59,6 +67,13 @@ jobs:
         with:
           project: ${{ inputs.project }}
 
+      - id: get-keycloak-site-a-url-from-glb
+        if: ${{ inputs.activeActive }}
+        shell: bash
+        run: |
+          KEYCLOAK_SITE_URL=https://$(kubectl -n "${{ inputs.project }}" get svc accelerator-loadbalancer --template="{{range .status.loadBalancer.ingress}}{{.hostname}}{{end}}")
+          echo "KEYCLOAK_SITE_URL=$KEYCLOAK_SITE_URL" >> "$GITHUB_ENV"
+
       - name: Perform Health checks on cluster A
         id: multi_site_health_check_a
         shell: bash
@@ -77,6 +92,13 @@ jobs:
         with:
           project: ${{ inputs.project }}
 
+      - id: get-keycloak-site-b-url-from-glb
+        if: ${{ inputs.activeActive }}
+        shell: bash
+        run: |
+          KEYCLOAK_SITE_URL=https://$(kubectl -n "${{ inputs.project }}" get svc accelerator-loadbalancer --template="{{range .status.loadBalancer.ingress}}{{.hostname}}{{end}}")
+          echo "KEYCLOAK_SITE_URL=$KEYCLOAK_SITE_URL" >> "$GITHUB_ENV"
+
       - name: Perform Health checks on cluster B
         id: multi_site_health_check_b
         shell: bash

diff --git a/.github/workflows/rosa-cluster-auto-provision-on-schedule.yml b/.github/workflows/rosa-cluster-auto-provision-on-schedule.yml
@@ -20,19 +20,20 @@ jobs:
       activeActive: true
     secrets: inherit
 
-  run-multi-site-health-checks-1:
+  run-multi-site-health-checks-after-deploy:
     needs: keycloak-deploy-active-active
     name: Run multi-site health checks after deployment
     uses: ./.github/workflows/keycloak-multi-site-health-check.yml
     with:
+      activeActive: true
       clusterPrefix: gh-keycloak # ${{ env.CLUSTER_PREFIX }} -- unfortunately 'env.' doesn't work here
       project: runner-keycloak
       region: eu-west-1
       expectedInfinispanNodeCount: '3'
     secrets: inherit
 
   run-functional-tests-active-active:
-    needs: run-multi-site-health-checks-1
+    needs: run-multi-site-health-checks-after-deploy
     uses: ./.github/workflows/rosa-run-crossdc-func-tests.yml
     with:
       activeActive: true
@@ -41,32 +42,75 @@ jobs:
       skipRemoteCaches: true
     secrets: inherit
 
-  run-multi-site-health-checks-2:
+  run-multi-site-health-checks-after-functional-tests:
     needs: run-functional-tests-active-active
     name: Run multi-site health checks after functional tests
     uses: ./.github/workflows/keycloak-multi-site-health-check.yml
     with:
+      activeActive: true
       clusterPrefix: gh-keycloak # ${{ env.CLUSTER_PREFIX }} -- unfortunately 'env.' doesn't work here
       project: runner-keycloak
       region: eu-west-1
       expectedInfinispanNodeCount: '3'
     secrets: inherit
 
   run-scaling-benchmark-active-active:
-    needs: run-multi-site-health-checks-2
+    needs: run-multi-site-health-checks-after-functional-tests
     uses: ./.github/workflows/rosa-scaling-benchmark.yml
     with:
       clusterName: gh-keycloak-a # ${{ env.CLUSTER_PREFIX }}-a -- unfortunately 'env.' doesn't work here
       outputArchiveSuffix: 'active-active'
     secrets: inherit
 
-  run-multi-site-health-checks-3:
+  run-multi-site-health-checks-after-benchmarks:
     needs: run-scaling-benchmark-active-active
     name: Run multi-site health checks after benchmarks
     uses: ./.github/workflows/keycloak-multi-site-health-check.yml
     with:
+      activeActive: true
       clusterPrefix: gh-keycloak # ${{ env.CLUSTER_PREFIX }} -- unfortunately 'env.' doesn't work here
       project: runner-keycloak
       region: eu-west-1
       expectedInfinispanNodeCount: '3'
     secrets: inherit
+
+  keycloak-undeploy-active-active:
+    needs: run-multi-site-health-checks-after-benchmarks
+    name: Undeploy Keycloak A/A deployment on the multi-az cluster
+    if: github.event_name != 'schedule' || github.repository == 'keycloak/keycloak-benchmark'
+    uses: ./.github/workflows/rosa-multi-az-cluster-undeploy.yml
+    with:
+      clusterPrefix: gh-keycloak # ${{ env.CLUSTER_PREFIX }} -- unfortunately 'env.' doesn't work here
+      skipAuroraDeletion: true
+      activeActive: true
+    secrets: inherit
+
+  keycloak-deploy-active-passive:
+    needs: keycloak-undeploy-active-active
+    name: Deploy Active/Passive Keycloak with External Infinispan and Persistent Sessions
+    if: github.event_name != 'schedule' || github.repository == 'keycloak/keycloak-benchmark'
+    uses: ./.github/workflows/rosa-multi-az-cluster-create.yml
+    with:
+      clusterPrefix: gh-keycloak # ${{ env.CLUSTER_PREFIX }} -- unfortunately 'env.' doesn't work here
+      enableMultiSiteFeature: true
+      enableExternalInfinispanFeature: false
+      createCluster: false
+    secrets: inherit
+
+  run-functional-tests-active-passive:
+    needs: keycloak-deploy-active-passive
+    uses: ./.github/workflows/rosa-run-crossdc-func-tests.yml
+    with:
+      clusterPrefix: gh-keycloak # ${{ env.CLUSTER_PREFIX }} -- unfortunately 'env.' doesn't work here
+      skipEmbeddedCaches: true
+      skipRemoteCaches: true
+    secrets: inherit
+
+  run-scaling-benchmark-active-passive:
+    needs: run-functional-tests-active-passive
+    uses: ./.github/workflows/rosa-scaling-benchmark.yml
+    with:
+      clusterName: gh-keycloak-a # ${{ env.CLUSTER_PREFIX }}-a -- unfortunately 'env.' doesn't work here
+      skipCreateDataset: true
+      outputArchiveSuffix: 'active-passive'
+    secrets: inherit
diff --git a/.github/workflows/rosa-multi-az-cluster-create.yml b/.github/workflows/rosa-multi-az-cluster-create.yml
@@ -20,10 +20,6 @@ on:
         description: 'When true deploy an Active/Active Keycloak deployment'
         type: boolean
         default: false
-      enableKc25Mode:
-        description: 'Set to true when version older than 26 is deployed'
-        type: boolean
-        default: false
       enableExternalInfinispanFeature:
         description: 'To enable the external Infinispan feature. It disables the embedded caches and only uses the remote caches.'
         type: boolean
@@ -54,10 +50,6 @@ on:
         description: 'When true deploy an Active/Active Keycloak deployment'
         type: boolean
         default: false
-      enableKc25Mode:
-        description: 'Set to true when version older than 26 is deployed'
-        type: boolean
-        default: false
       enableExternalInfinispanFeature:
         description: 'To enable the external Infinispan feature. It disables the embedded caches and only uses the remote caches.'
         type: boolean
@@ -73,7 +65,6 @@ on:
 env:
   CLUSTER_PREFIX: ${{ inputs.clusterPrefix || format('gh-{0}', github.repository_owner) }}
   REGION: ${{ inputs.region || vars.AWS_DEFAULT_REGION }}
-  KC_KC25_MODE: ${{ inputs.enableKc25Mode }}
   KC_EXTERNAL_INFINISPAN: ${{ inputs.enableExternalInfinispanFeature }}
   KC_MULTI_SITE: ${{ inputs.enableMultiSiteFeature }}
 
@@ -196,7 +187,7 @@ jobs:
           KC_INSTANCES: 3
           KC_DISABLE_STICKY_SESSION: true
           KC_CRYOSTAT: false
-          KC_KC25_MODE: ${{ env.KC_KC25_MODE }}
+          KC_IS_ACTIVE_PASSIVE: true
           KC_EXTERNAL_INFINISPAN: ${{ env.KC_EXTERNAL_INFINISPAN }}
           KC_MEMORY_REQUESTS_MB: 3000
           KC_MEMORY_LIMITS_MB: 4000
@@ -205,6 +196,8 @@ jobs:
           KC_DB_POOL_MIN_SIZE: 30
           KC_DATABASE: "aurora-postgres"
           MULTI_AZ: "true"
+          CROSS_DC_XSITE_FAIL_POLICY: "WARN"
+          CROSS_DC_TX_MODE: "NONE"
           KC_REPOSITORY: ${{ inputs.keycloakRepository }}
           KC_BRANCH: ${{ inputs.keycloakBranch }}
 
@@ -236,7 +229,7 @@ jobs:
           KC_INSTANCES: 3
           KC_DISABLE_STICKY_SESSION: true
           KC_CRYOSTAT: false
-          KC_KC25_MODE: ${{ env.KC_KC25_MODE }}
+          KC_IS_ACTIVE_PASSIVE: false
           KC_EXTERNAL_INFINISPAN: ${{ env.KC_EXTERNAL_INFINISPAN }}
           KC_MEMORY_REQUESTS_MB: 3000
           KC_MEMORY_LIMITS_MB: 4000

diff --git a/.github/workflows/rosa-scaling-benchmark.yml b/.github/workflows/rosa-scaling-benchmark.yml
@@ -15,10 +15,10 @@ on:
         description: 'Name of the region where EC2 instances should be installed'
         type: string
         default: 'eu-west-1'
-      numberOfEntitiesInRealm:
-        description: 'Number of entities for the scenario in DB'
+      usersPerRealm:
+        description: 'Number of users for the scenario in DB'
         type: number
-        default: 20000
+        default: 100000
       maxWaitEntityCreation:
         description: 'Maximum number of seconds to wait for creation of entities'
         type: number
@@ -58,10 +58,10 @@ on:
         description: 'Name of the region where EC2 instances should be installed'
         type: string
         default: 'eu-west-1'
-      numberOfEntitiesInRealm:
-        description: 'Number of entities for the scenario in DB'
+      usersPerRealm:
+        description: 'Number of users for the scenario in DB'
         type: number
-        default: 20000
+        default: 100000
       maxWaitEntityCreation:
         description: 'Maximum number of seconds to wait for creation of entities'
         type: number
@@ -93,6 +93,7 @@ env:
   PROJECT_PREFIX: runner- # same as default
   PROJECT: runner-keycloak
   ANSIBLE_CUSTOM_VARS_ARG: '-e @env_rosa_benchmark.yml'
+  CLIENTS_PER_REALM: 20000
 
 jobs:
   run:
@@ -152,14 +153,14 @@ jobs:
           ROSA_CLUSTER_NAME: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }}
           NAMESPACE: ${{ env.PROJECT }}
 
-      - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users and clients
+      - name: Create Keycloak dataset with "${{ inputs.usersPerRealm }}" users and "${{ env.CLIENTS_PER_REALM }}" clients
         if: ${{ !inputs.skipCreateDataset }}
         uses: ./.github/actions/keycloak-create-dataset
         with:
           project: ${{ env.PROJECT }}
-          users: ${{ inputs.numberOfEntitiesInRealm }}
+          users: ${{ inputs.usersPerRealm }}
           clients: 100
-          clientsPerRealm: ${{ inputs.numberOfEntitiesInRealm }}
+          clientsPerRealm: ${{ env.CLIENTS_PER_REALM }}
           createClientForSpecificRealm: true
           maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }}
 
@@ -217,7 +218,7 @@ jobs:
           --ramp-up=20
           --logout-percentage=0
           --measurement=${{ inputs.measurement }}
-          --users-per-realm=${{ inputs.numberOfEntitiesInRealm }}
+          --users-per-realm=${{ inputs.usersPerRealm }}
           --log-http-on-failure
           --sla-error-percentage=0.001
         continue-on-error: true
@@ -299,7 +300,7 @@ jobs:
           --ramp-up=20
           --logout-percentage=100
           --measurement=${{ inputs.measurement }}
-          --users-per-realm=${{ inputs.numberOfEntitiesInRealm }}
+          --users-per-realm=${{ inputs.usersPerRealm }}
           --log-http-on-failure
           --sla-error-percentage=0.001
         continue-on-error: true
@@ -374,7 +375,7 @@ jobs:
           --users-per-sec=${{ inputs.numberOfClientsPerSecond }}
           --ramp-up=20
           --measurement=${{ inputs.measurement }}
-          --users-per-realm=${{ inputs.numberOfEntitiesInRealm }}
+          --users-per-realm=${{ inputs.usersPerRealm }}
           --log-http-on-failure
           --sla-error-percentage=0.001
         continue-on-error: true

diff --git a/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc b/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc
@@ -32,7 +32,6 @@ Deploy OpenShift and ROSA as described in xref:kubernetes-guide::prerequisite/pr
 KC_CPU_REQUESTS=6
 KC_INSTANCES=3
 KC_DISABLE_STICKY_SESSION=true
-KC_KC25_MODE=true
 KC_MEMORY_REQUESTS_MB=3000
 KC_MEMORY_LIMITS_MB=4000
 KC_DB_POOL_INITIAL_SIZE=30

diff --git a/doc/kubernetes/modules/ROOT/pages/customizing-deployment.adoc b/doc/kubernetes/modules/ROOT/pages/customizing-deployment.adoc
@@ -221,9 +221,10 @@ Available options:
 +
 NOTE: This option is implemented only for OpenShift deployments.
 
-[[KC_KC25_MODE,KC_KC25_MODE]]
-KC_KC25_MODE::
-To deploy older Keycloak version prior to Keycloak 26 it is necessary to enable the KC25 mode.
+[[KC_IS_ACTIVE_PASSIVE,KC_IS_ACTIVE_PASSIVE]]
+KC_IS_ACTIVE_PASSIVE::
+To deploy Keycloak in the active/passive setup.
+This should be set to true when active/passive loadbalancer is used.
 
 == Available Benchmark options
 

diff --git a/provision/common/Taskfile.yaml b/provision/common/Taskfile.yaml
@@ -24,7 +24,7 @@ vars:
   KC_REMOTE_STORE_HOST: '{{default "localhost" .KC_REMOTE_STORE_HOST}}'
   KC_REMOTE_STORE_PORT: '{{default "11222" .KC_REMOTE_STORE_PORT}}'
   KC_DISABLE_STICKY_SESSION: '{{default "false" .KC_DISABLE_STICKY_SESSION}}'
-  KC_KC25_MODE: '{{default "false" .KC_KC25_MODE}}'
+  KC_IS_ACTIVE_PASSIVE: '{{default "false" .KC_IS_ACTIVE_PASSIVE}}'
   KC_EXTERNAL_INFINISPAN: '{{default "false" .KC_EXTERNAL_INFINISPAN}}'
   MULTI_AZ: '{{default "false" .MULTI_AZ}}'
   ENV_DATA_JSON_PATH: "{{.ROOT_DIR}}/../environment_data.json"
@@ -72,7 +72,7 @@ tasks:
       - echo {{.KC_REMOTE_STORE_HOST}} > .task/var-KC_REMOTE_STORE_HOST
       - echo {{.KC_REMOTE_STORE_PORT}} > .task/var-KC_REMOTE_STORE_PORT
       - echo {{.KC_DISABLE_STICKY_SESSION}} > .task/var-KC_DISABLE_STICKY_SESSION
-      - echo {{.KC_KC25_MODE}} > .task/var-KC_KC25_MODE
+      - echo {{.KC_IS_ACTIVE_PASSIVE}} > .task/var-KC_IS_ACTIVE_PASSIVE
       - echo {{.KC_EXTERNAL_INFINISPAN}} > .task/var-KC_EXTERNAL_INFINISPAN
       - echo {{.KC_HOSTNAME_OVERRIDE}} > .task/var-KC_HOSTNAME_OVERRIDE
       - echo {{.KC_HEALTH_HOSTNAME}} > .task/var-KC_HEALTH_HOSTNAME
@@ -94,6 +94,7 @@ tasks:
           --arg kc_repo "{{ .KC_REPOSITORY }}" \
           --arg kc_branch "{{ .KC_BRANCH }}" \
           --argjson is_multi_az "{{ .MULTI_AZ }}" \
+          --argjson is_active_passive "{{ .KC_IS_ACTIVE_PASSIVE }}" \
           '. + { "numOfPods": ($num_of_pods|tonumber), "cpuRequestsPerPod": ($cpu_requests_per_pod|tonumber),
                              "cpuLimitsPerPod": (if ($cpu_limits_per_pod | length) == 0 then null else ($cpu_limits_per_pod | tonumber?) end),
                              "stickySessionDisabled": ($sticky_sessions),
@@ -110,6 +111,7 @@ tasks:
                                "max": ($meta_spc_max|tonumber)
                              },
                             "isMultiAZ": ($is_multi_az),
+                            "isActivePassive": ($is_active_passive),
                             "keycloakOperatorTag": ($kc_operator_tag),
                             "keycloakRepository": ($kc_repo),
                             "keycloakBranch": ($kc_branch),
@@ -140,7 +142,7 @@ tasks:
       - test "{{.KC_REMOTE_STORE_HOST}}" == "$(cat .task/var-KC_REMOTE_STORE_HOST)"
       - test "{{.KC_REMOTE_STORE_PORT}}" == "$(cat .task/var-KC_REMOTE_STORE_PORT)"
       - test "{{.KC_DISABLE_STICKY_SESSION}}" == "$(cat .task/var-KC_DISABLE_STICKY_SESSION)"
-      - test "{{.KC_KC25_MODE}}" == "$(cat .task/var-KC_KC25_MODE)"
+      - test "{{.KC_IS_ACTIVE_PASSIVE}}" == "$(cat .task/var-KC_IS_ACTIVE_PASSIVE)"
       - test "{{.KC_EXTERNAL_INFINISPAN}}" == "$(cat .task/var-KC_EXTERNAL_INFINISPAN)"
       - test "{{.KC_HOSTNAME_OVERRIDE}}" == "$(cat .task/var-KC_HOSTNAME_OVERRIDE)"
       - test "{{.KC_HEALTH_HOSTNAME}}" == "$(cat .task/var-KC_HEALTH_HOSTNAME)"

diff --git a/provision/keycloak-tasks/Taskfile.yaml b/provision/keycloak-tasks/Taskfile.yaml
@@ -52,7 +52,7 @@ vars:
   KC_REMOTE_STORE_HOST: '{{default "localhost" .KC_REMOTE_STORE_HOST}}'
   KC_REMOTE_STORE_PORT: '{{default "11222" .KC_REMOTE_STORE_PORT}}'
   KC_DISABLE_STICKY_SESSION: '{{default "false" .KC_DISABLE_STICKY_SESSION}}'
-  KC_KC25_MODE: '{{default "false" .KC_KC25_MODE}}'
+  KC_IS_ACTIVE_PASSIVE: '{{default "false" .KC_IS_ACTIVE_PASSIVE}}'
   KC_EXTERNAL_INFINISPAN: '{{default "false" .KC_EXTERNAL_INFINISPAN}}'
 
 tasks: