From a5f06418b6a591f89588914cec60b899dd9f72b5 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 18 Apr 2022 08:53:53 -0700 Subject: [PATCH] Remove config management for subunit2sql workers The openstack health service is being shutdown and retired. That services was the only service that relied on the subunit2sql workers. This means we can shutdown and retire the subunit2sql workers. This is one step of that process. Change-Id: Ibd02faaeba888dfcd1f512f4dd3a7d768497fc16 --- doc/source/logstash.rst | 54 ++++--------------- hiera/common.yaml | 3 -- inventory/service/group_vars/logstash.yaml | 1 - inventory/service/groups.yaml | 4 -- manifests/site.pp | 12 ----- .../manifests/subunit_worker.pp | 49 ----------------- .../logstash/jenkins-subunit-worker.yaml.erb | 9 ---- 7 files changed, 10 insertions(+), 122 deletions(-) delete mode 100644 modules/openstack_project/manifests/subunit_worker.pp delete mode 100644 modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb diff --git a/doc/source/logstash.rst b/doc/source/logstash.rst index 5fd7ea918a..630ec57fbd 100644 --- a/doc/source/logstash.rst +++ b/doc/source/logstash.rst @@ -75,16 +75,16 @@ bottleneck very quickly. This looks something like: zuul post-logs playbook | | - gearman-client --------------- - / | \ | - / | \ | - gearman gearman gearman subunit gearman - worker1 worker2 worker3 worker01 - | | | | - logstash logstash logstash | - indexer1 indexer2 indexer3 | - \ | / subunit2sql - \ | / DB + gearman-client + / | \ + / | \ + gearman gearman gearman + worker1 worker2 worker3 + | | | + logstash logstash logstash + indexer1 indexer2 indexer3 + \ | / + \ | / elasticsearch cluster | @@ -113,19 +113,6 @@ can be found at * https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-client.py * https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-worker.py -Subunit Gearman Worker ----------------------- - -Using the same mechanism as the Log pushers there is an additional class of -gearman worker that takes the subunit output from test runs and stores them in -a subunit2SQL database. Right now this is only done with the subunit output -from gate and periodic queue tempest runs. - -If you are interested in technical details the source of this script can be -found at: - -* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/subunit-gearman-worker.py - Logstash -------- @@ -215,27 +202,6 @@ be reached at http://logstash.openstack.org. See :ref:`query-logstash` for more info on using Kibana to perform queries. -subunit2SQL ------------ -subunit2SQL is a python project for taking subunit v2 streams and storing them -in a SQL database. More information on the subunit protocol can be found here: -https://github.com/testing-cabal/subunit/blob/master/README - -subunit2sql provides a database schema, several utilities for interacting with -the database, and a python library to build tooling on top of the database. -More information about using subunit2sql can be found at: -http://docs.openstack.org/developer/subunit2sql/ - -Our instance of the subunit2SQL database is running on a MySQL database and has -been configured to be remotely accessible to allow for public querying. The -public query access is provided with the following credentials:: - - username: query - password: query - hostname: logstash.openstack.org - database name: subunit2sql - database port: 3306 - simpleproxy ----------- Simpleproxy is a simple tcp proxy which allows forwarding tcp connections from diff --git a/hiera/common.yaml b/hiera/common.yaml index 5e43077428..5eadefe33a 100644 --- a/hiera/common.yaml +++ b/hiera/common.yaml @@ -33,7 +33,6 @@ cacti_hosts: - gitea08.opendev.org - grafana01.opendev.org - graphite02.opendev.org -- health.openstack.org - jvb01.opendev.org - jvb02.opendev.org - kdc03.openstack.org @@ -94,8 +93,6 @@ cacti_hosts: - status01.openstack.org - storyboard01.opendev.org - storyboard-dev01.opendev.org -- subunit-worker01.openstack.org -- subunit-worker02.openstack.org - translate.openstack.org - translate-dev.openstack.org - wiki.openstack.org diff --git a/inventory/service/group_vars/logstash.yaml b/inventory/service/group_vars/logstash.yaml index 91e4cf90d0..4295a978c1 100644 --- a/inventory/service/group_vars/logstash.yaml +++ b/inventory/service/group_vars/logstash.yaml @@ -3,5 +3,4 @@ iptables_extra_public_tcp_ports: - 3306 iptables_extra_allowed_groups: - {'protocol': 'tcp', 'port': '4730', 'group': 'logstash-worker'} - - {'protocol': 'tcp', 'port': '4730', 'group': 'subunit-worker'} - {'protocol': 'tcp', 'port': '4730', 'group': 'zuul-executor'} diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index 66754358d1..e2037f961f 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -139,7 +139,6 @@ groups: - status*.open*.org - storyboard-dev[0-9]*.opendev.org - storyboard[0-9]*.opendev.org - - subunit-worker[0-9]*.open*.org - translate-dev[0-9]*.open*.org - translate[0-9]*.open*.org puppet4: @@ -152,7 +151,6 @@ groups: - status*.open*.org - storyboard[0-9]*.opendev.org - storyboard-dev[0-9]*.opendev.org - - subunit-worker[0-9]*.open*.org - translate[0-9]*.open*.org - translate-dev[0-9]*.open*.org refstack: @@ -172,8 +170,6 @@ groups: - storyboard[0-9]*.opendev.org storyboard-dev: - storyboard-dev[0-9]*.opendev.org - subunit-worker: - - subunit-worker[0-9]*.open*.org translate-dev: - translate-dev[0-9]*.open*.org translate: diff --git a/manifests/site.pp b/manifests/site.pp index c3220b1e35..41fe4d31d0 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -61,18 +61,6 @@ node /^logstash-worker\d+\.open.*\.org$/ { } } -# Node-OS: xenial -node /^subunit-worker\d+\.open.*\.org$/ { - $group = "subunit-worker" - class { 'openstack_project::server': } - class { 'openstack_project::subunit_worker': - subunit2sql_db_host => hiera('subunit2sql_db_host', ''), - subunit2sql_db_pass => hiera('subunit2sql_db_password', ''), - mqtt_pass => hiera('mqtt_service_user_password'), - mqtt_ca_cert_contents => hiera('mosquitto_tls_ca_file'), - } -} - # Node-OS: xenial node /^elasticsearch\d+\.open.*\.org$/ { $group = "elasticsearch" diff --git a/modules/openstack_project/manifests/subunit_worker.pp b/modules/openstack_project/manifests/subunit_worker.pp deleted file mode 100644 index f789e1283e..0000000000 --- a/modules/openstack_project/manifests/subunit_worker.pp +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# subunit2sql worker glue class. -# -class openstack_project::subunit_worker ( - $subunit2sql_db_host, - $subunit2sql_db_pass, - $mqtt_user = 'infra', - $mqtt_pass = undef, - $mqtt_ca_cert_contents = undef, -) { - - file { '/etc/subunit2sql/subunit-woker.yaml': - ensure => file, - owner => 'root', - group => 'root', - mode => '0555', - content => template('openstack_project/logstash/jenkins-subunit-worker.yaml.erb'), - } - - file { '/etc/subunit2sql/mqtt-root-CA.pem.crt': - ensure => present, - content => $mqtt_ca_cert_contents, - replace => true, - owner => 'subunit', - group => 'subunit', - mode => '0555', - } - - include subunit2sql - subunit2sql::worker { 'A': - config_file => '/etc/subunit2sql/subunit-woker.yaml', - db_host => $subunit2sql_db_host, - db_pass => $subunit2sql_db_pass, - } - -} diff --git a/modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb b/modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb deleted file mode 100644 index 5d5bb7a954..0000000000 --- a/modules/openstack_project/templates/logstash/jenkins-subunit-worker.yaml.erb +++ /dev/null @@ -1,9 +0,0 @@ -gearman-host: logstash.openstack.org -gearman-port: 4730 -config: /etc/subunit2sql/subunit2sql.conf -#mqtt-host: firehose.openstack.org -#mqtt-port: 8883 -#mqtt-topic: gearman-subunit/<%= @hostname %> -#mqtt-user: <%= @mqtt_user %> -#mqtt-pass: <%= @mqtt_pass %> -#mqtt-ca-certs: /etc/subunit2sql/mqtt-root-CA.pem.crt