[osh-selenium] Upgrade image to latest-ubuntu_focal

+ migrate all Python tests to use Selenium v4 (bionic image had v3 installed):
https://www.selenium.dev/documentation/webdriver/getting_started/upgrade_to_selenium_4/
+ amend selenium role in order to install ChromeDriver compatible with Google Chrome:
https://chromedriver.chromium.org/downloads/version-selection
+ run selenium tests AFTER the charts are deployed

Change-Id: I46200b7dc173bd0e1e6bf3545d9a26c252a21927

grafana/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v9.2.10
 description: OpenStack-Helm Grafana
 name: grafana
-version: 0.1.19
+version: 0.1.20
 home: https://grafana.com/
 sources:
   - https://github.com/grafana/grafana

grafana/templates/bin/_selenium-tests.py.tpl

@@ -22,6 +22,7 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
 from selenium.common.exceptions import TimeoutException
 from selenium.common.exceptions import NoSuchElementException
 
@@ -55,7 +56,9 @@ options = Options()
 options.add_argument('--headless')
 options.add_argument('--no-sandbox')
 options.add_argument('--window-size=1920x1080')
-browser = webdriver.Chrome(chrome_driver, chrome_options=options)
+
+service = Service(executable_path=chrome_driver)
+browser = webdriver.Chrome(service=service, options=options)
 
 logger.info("Attempting to open Grafana dashboard")
 try:
@@ -71,9 +74,9 @@ except TimeoutException:
 
 logger.info("Attempting to log into Grafana dashboard")
 try:
-    browser.find_element_by_name('user').send_keys(username)
+    browser.find_element(By.NAME, 'user').send_keys(username)
-    browser.find_element_by_name('password').send_keys(password)
+    browser.find_element(By.NAME, 'password').send_keys(password)
-    browser.find_element_by_css_selector('[aria-label="Login button"]').click()
+    browser.find_element(By.CSS_SELECTOR, '[aria-label="Login button"]').click()
     logger.info("Successfully logged in to Grafana")
 except NoSuchElementException:
     logger.error("Failed to log in to Grafana")

grafana/values.yaml

@@ -22,7 +22,7 @@ images:
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     db_init: docker.io/openstackhelm/heat:stein-ubuntu_bionic
     grafana_db_session_sync: docker.io/openstackhelm/heat:stein-ubuntu_bionic
-    selenium_tests: docker.io/openstackhelm/osh-selenium:latest-ubuntu_bionic
+    selenium_tests: docker.io/openstackhelm/osh-selenium:latest-ubuntu_focal
     image_repo_sync: docker.io/library/docker:17.07.0
   pull_policy: IfNotPresent
   local_registry:

nagios/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Nagios
 name: nagios
-version: 0.1.6
+version: 0.1.7
 home: https://www.nagios.org
 sources:
   - https://opendev.org/openstack/openstack-helm-addons

nagios/templates/bin/_selenium-tests.py.tpl

@@ -22,6 +22,7 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
 from selenium.common.exceptions import TimeoutException
 from selenium.common.exceptions import NoSuchElementException
 from selenium.common.exceptions import ScreenshotException
@@ -50,7 +51,7 @@ def get_variable(env_var):
 def click_link_by_name(link_name):
     try:
         logger.info("Clicking '{}' link".format(link_name))
-        link = browser.find_element_by_link_text(link_name)
+        link = browser.find_element(By.LINK_TEXT, link_name)
         link.click()
     except NoSuchElementException:
         logger.error("Failed clicking '{}' link".format(link_name))
@@ -78,7 +79,9 @@ options = Options()
 options.add_argument('--headless')
 options.add_argument('--no-sandbox')
 options.add_argument('--window-size=1920x1080')
-browser = webdriver.Chrome(chrome_driver, chrome_options=options)
+
+service = Service(executable_path=chrome_driver)
+browser = webdriver.Chrome(service=service, options=options)
 
 try:
     logger.info('Attempting to connect to Nagios')

nagios/values.yaml

@@ -20,7 +20,7 @@ images:
     apache_proxy: docker.io/library/httpd:2.4
     nagios: docker.io/openstackhelm/nagios:latest-ubuntu_bionic
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
-    selenium_tests: docker.io/openstackhelm/osh-selenium:latest-ubuntu_bionic
+    selenium_tests: docker.io/openstackhelm/osh-selenium:latest-ubuntu_focal
     image_repo_sync: docker.io/library/docker:17.07.0
   pull_policy: IfNotPresent
   local_registry:

releasenotes/notes/grafana.yaml

@@ -20,4 +20,5 @@ grafana:
   - 0.1.17 Fix uid for the user grafana
   - 0.1.18 Migrator job is now mariadb-fail-proof
   - 0.1.19 Update grafana to 9.2.10
+  - 0.1.20 Upgrade osh-selenium image to latest-ubuntu_focal
 ...

releasenotes/notes/nagios.yaml

@@ -7,4 +7,5 @@ nagios:
   - 0.1.4 Update htk requirements
   - 0.1.5 Switch nagios image from xenial to bionic
   - 0.1.6 Added OCI registry authentication
+  - 0.1.7 Upgrade osh-selenium image to latest-ubuntu_focal
 ...

roles/deploy-selenium/tasks/main.yaml

@@ -11,12 +11,12 @@
 # limitations under the License.
 
 ---
-- name: "creating selenium configuration directory"
+- name: Create selenium configuration directory
   file:
     path: /etc/selenium
     state: directory
 
-- name: install selenium dependencies
+- name: Install selenium dependencies
   when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
   apt:
     name: "{{ packages }}"
@@ -26,30 +26,43 @@
       - wget
       - xvfb
 
-- name: install selenium
+- name: Install selenium
   pip:
     name: selenium
     state: latest
     executable: pip3
 
-- name: Get selenium chrome driver
+- name: Add google chrome signing key
-  shell: |-
+  get_url:
-          set -ex
+    url: https://dl-ssl.google.com/linux/linux_signing_key.pub
-          wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+    dest: /etc/apt/trusted.gpg.d/google-chrome.asc
-          sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
-          wget --directory-prefix=/tmp/ https://chromedriver.storage.googleapis.com/2.44/chromedriver_linux64.zip
-  args:
-    executable: /bin/bash
 
-- name: unarchive selenium chrome driver
+- name: Add google chrome repository
-  unarchive:
+  apt_repository:
-    src: /tmp/chromedriver_linux64.zip
+    repo: "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/google-chrome.asc] http://dl.google.com/linux/chrome/deb/ stable main"
-    dest: /etc/selenium
+    filename: google-chrome
-    remote_src: yes
+    state: present
 
-- name: install google chrome
+- name: Install google chrome
   when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
   apt:
     name: google-chrome-stable
     update_cache: yes
+    install_recommends: false
+
+# We need to install ChromeDriver compatible with Google Chrome version
+- name: Get selenium chromedriver archive
+  shell: |-
+    set -ex
+    CHROME_VERSION=$(dpkg -s google-chrome-stable | grep -Po '(?<=^Version: ).*' | awk -F'.' '{print $1"."$2"."$3}')
+    DRIVER_PATH=$(wget -qO- https://chromedriver.storage.googleapis.com | grep -Po "(?<=<Key>)${CHROME_VERSION}[^<]*?chromedriver_linux64\.zip(?=</Key>)" | tail -1)
+    wget -O /tmp/chromedriver.zip "https://chromedriver.storage.googleapis.com/${DRIVER_PATH}"
+  args:
+    executable: /bin/bash
+
+- name: Unarchive selenium chromedriver
+  unarchive:
+    src: /tmp/chromedriver.zip
+    dest: /etc/selenium
+    remote_src: yes
 ...

tools/gate/selenium/grafanaSelenium.py

@@ -23,10 +23,11 @@ st = SeleniumTester('Grafana')
 username = st.get_variable('GRAFANA_USER')
 password = st.get_variable('GRAFANA_PASSWORD')
 grafana_uri = st.get_variable('GRAFANA_URI')
+grafana_url = 'http://{0}'.format(grafana_uri)
 
 try:
     st.logger.info('Attempting to connect to Grafana')
-    st.browser.get(grafana_uri)
+    st.browser.get(grafana_url)
     el = WebDriverWait(st.browser, 15).until(
         EC.title_contains('Grafana')
     )
@@ -36,15 +37,15 @@ except TimeoutException:
     st.browser.quit()
     sys.exit(1)
 
-logger.info("Attempting to log into Grafana dashboard")
+st.logger.info("Attempting to log into Grafana dashboard")
 try:
-    browser.find_element_by_name('user').send_keys(username)
+    st.browser.find_element(By.NAME, 'user').send_keys(username)
-    browser.find_element_by_name('password').send_keys(password)
+    st.browser.find_element(By.NAME, 'password').send_keys(password)
-    browser.find_element_by_class_name('css-6ntnx5-button').click()
+    st.browser.find_element(By.CLASS_NAME, 'css-1mhnkuh').click()
-    logger.info("Successfully logged in to Grafana")
+    st.logger.info("Successfully logged in to Grafana")
 except NoSuchElementException:
-    logger.error("Failed to log in to Grafana")
+    st.logger.error("Failed to log in to Grafana")
-    browser.quit()
+    st.browser.quit()
     sys.exit(1)
 
 st.browser.quit()

tools/gate/selenium/seleniumtester.py

@@ -75,7 +75,7 @@ class SeleniumTester():
                 EC.presence_of_element_located((By.LINK_TEXT, link_name))
             )
             self.logger.info("Clicking '{}' link".format(link_name))
-            link = self.browser.find_element_by_link_text(link_name)
+            link = self.browser.find_element(By.LINK_TEXT, link_name)
             link.click()
         except (TimeoutException, NoSuchElementException):
             self.logger.error("Failed clicking '{}' link".format(link_name))

zuul.d/jobs.yaml

@@ -225,7 +225,7 @@
         - - ./tools/deployment/osh-infra-monitoring/110-grafana.sh
           - ./tools/deployment/osh-infra-monitoring/120-nagios.sh
           - ./tools/deployment/osh-infra-monitoring/170-postgresql.sh
-          - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true
+        - - ./tools/deployment/osh-infra-monitoring/600-grafana-selenium.sh || true
           - ./tools/deployment/osh-infra-monitoring/610-prometheus-selenium.sh || true
           - ./tools/deployment/osh-infra-monitoring/620-nagios-selenium.sh || true
 
@@ -623,7 +623,7 @@
         - - ./tools/deployment/osh-infra-monitoring-tls/110-grafana.sh
           - ./tools/deployment/osh-infra-monitoring-tls/120-nagios.sh
           - ./tools/deployment/osh-infra-monitoring-tls/170-postgresql.sh
-          - ./tools/deployment/osh-infra-monitoring-tls/600-grafana-selenium.sh || true
+        - - ./tools/deployment/osh-infra-monitoring-tls/600-grafana-selenium.sh || true
           - ./tools/deployment/osh-infra-monitoring-tls/610-prometheus-selenium.sh || true
           - ./tools/deployment/osh-infra-monitoring-tls/620-nagios-selenium.sh || true
 - job: