From 30c22e2286ce78362899cb1854edb1ff081da774 Mon Sep 17 00:00:00 2001
From: ricolin <rlin@vexxhost.com>
Date: Thu, 2 Nov 2023 14:39:50 +0800
Subject: [PATCH] Placement: Support uWSGI for API server
Currently Placement API server still using eventlet-based HTTP servers,
it is generally considered more performant and flexible to run them
using a generic HTTP server that supports WSGI.
Change-Id: I7c0d57a210f1a2d02d989cd8c0d25798bfabfa35
---
placement/.helmignore | 1 +
placement/Chart.yaml | 2 +-
placement/templates/bin/_placement-api.sh.tpl | 10 +++-
placement/templates/configmap-etc.yaml | 11 +++-
placement/templates/deployment.yaml | 8 ++-
placement/values.yaml | 59 ++++++-------------
placement/values_overrides/tls.yaml | 29 ++++-----
releasenotes/notes/placement.yaml | 1 +
.../deployment/component/common/openstack.sh | 3 +-
9 files changed, 57 insertions(+), 67 deletions(-)
create mode 100644 placement/.helmignore
diff --git a/placement/.helmignore b/placement/.helmignore
new file mode 100644
index 0000000000..b54c347b85
--- /dev/null
+++ b/placement/.helmignore
@@ -0,0 +1 @@
+values_overrides
diff --git a/placement/Chart.yaml b/placement/Chart.yaml
index 38b9668338..3e3d688bfa 100644
--- a/placement/Chart.yaml
+++ b/placement/Chart.yaml
@@ -16,7 +16,7 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Placement
name: placement
-version: 0.3.14
+version: 0.3.15
home: https://docs.openstack.org/placement/latest/
icon: https://www.openstack.org/themes/openstack/images/project-mascots/Placement/OpenStack_Project_Placement_vertical.png
sources:
diff --git a/placement/templates/bin/_placement-api.sh.tpl b/placement/templates/bin/_placement-api.sh.tpl
index 2b1b12d143..f59947aaf8 100644
--- a/placement/templates/bin/_placement-api.sh.tpl
+++ b/placement/templates/bin/_placement-api.sh.tpl
@@ -20,9 +20,8 @@ set -ex
COMMAND="${@:-start}"
function start () {
-
+{{- if .Values.manifests.certificates }}
cp -a $(type -p placement-api) /var/www/cgi-bin/placement/
-
if [ -f /etc/apache2/envvars ]; then
# Loading Apache2 ENV variables
source /etc/apache2/envvars
@@ -46,13 +45,20 @@ function start () {
{{- end }}
{{- end }}
exec {{ .Values.conf.software.apache2.binary }} {{ .Values.conf.software.apache2.start_parameters }}
+{{- else }}
+ exec uwsgi --ini /etc/placement/placement-api-uwsgi.ini
+{{- end }}
}
function stop () {
+{{- if .Values.manifests.certificates }}
if [ -f /etc/apache2/envvars ]; then
source /etc/apache2/envvars
fi
{{ .Values.conf.software.apache2.binary }} -k graceful-stop
+{{- else }}
+ kill -TERM 1
+{{- end }}
}
$COMMAND
diff --git a/placement/templates/configmap-etc.yaml b/placement/templates/configmap-etc.yaml
index c5880af245..efd651613d 100644
--- a/placement/templates/configmap-etc.yaml
+++ b/placement/templates/configmap-etc.yaml
@@ -57,6 +57,12 @@ limitations under the License.
{{- if empty .Values.conf.placement.keystone_authtoken.memcache_secret_key -}}
{{- $_ := set .Values.conf.placement.keystone_authtoken "memcache_secret_key" ( default ( randAlphaNum 64 ) .Values.endpoints.oslo_cache.auth.memcache_secret_key ) -}}
{{- end -}}
+
+{{- if empty (index .Values.conf.placement_api_uwsgi.uwsgi "http-socket") -}}
+{{- $http_socket_port := tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | toString }}
+{{- $http_socket := printf "0.0.0.0:%s" $http_socket_port }}
+{{- $_ := set .Values.conf.placement_api_uwsgi.uwsgi "http-socket" $http_socket -}}
+{{- end -}}
---
apiVersion: v1
kind: Secret
@@ -66,6 +72,9 @@ type: Opaque
data:
policy.yaml: {{ toYaml .Values.conf.policy | b64enc }}
placement.conf: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.placement | b64enc }}
- logging.conf: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.logging | b64enc }}
+ placement-api-uwsgi.ini: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.placement_api_uwsgi | b64enc }}
+{{- if .Values.manifests.certificates }}
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.wsgi_placement "key" "wsgi-placement.conf" "format" "Secret" ) | indent 2 }}
+{{- end }}
+ logging.conf: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.logging | b64enc }}
{{- end }}
diff --git a/placement/templates/deployment.yaml b/placement/templates/deployment.yaml
index 08450f36e5..dc726dfc6a 100644
--- a/placement/templates/deployment.yaml
+++ b/placement/templates/deployment.yaml
@@ -100,14 +100,14 @@ spec:
scheme: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
path: /
port: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- initialDelaySeconds: 15
+ initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
scheme: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
path: /
port: {{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- initialDelaySeconds: 50
+ initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: pod-tmp
@@ -122,6 +122,10 @@ spec:
mountPath: /etc/placement/placement.conf
subPath: placement.conf
readOnly: true
+ - name: placement-etc
+ mountPath: /etc/placement/placement-api-uwsgi.ini
+ subPath: placement-api-uwsgi.ini
+ readOnly: true
{{- if .Values.conf.placement.DEFAULT.log_config_append }}
- name: placement-etc
mountPath: {{ .Values.conf.placement.DEFAULT.log_config_append }}
diff --git a/placement/values.yaml b/placement/values.yaml
index a411250c0d..7e9497a1e5 100644
--- a/placement/values.yaml
+++ b/placement/values.yaml
@@ -61,18 +61,6 @@ network:
port: 30778
conf:
- software:
- apache2:
- binary: apache2
- start_parameters: -DFOREGROUND
- # Enable/Disable modules
- # a2enmod:
- # - headers
- # - rewrite
- # a2dismod:
- # - status
- a2enmod: null
- a2dismod: null
policy: {}
placement:
DEFAULT:
@@ -148,36 +136,23 @@ conf:
formatter_default:
format: "%(message)s"
datefmt: "%Y-%m-%d %H:%M:%S"
- wsgi_placement: |
- Listen 0.0.0.0:{{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
- LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy
- SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
- CustomLog /dev/stdout combined env=!forwarded
- CustomLog /dev/stdout proxy env=forwarded
- <VirtualHost *:{{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}>
- WSGIDaemonProcess placement-api processes=4 threads=1 user=placement group=placement display-name=%{GROUP}
- WSGIProcessGroup placement-api
- WSGIScriptAlias / /var/www/cgi-bin/placement/placement-api
- WSGIApplicationGroup %{GLOBAL}
- WSGIPassAuthorization On
- <IfVersion >= 2.4>
- ErrorLogFormat "%{cu}t %M"
- </IfVersion>
- ErrorLog /dev/stdout
- SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
- CustomLog /dev/stdout combined env=!forwarded
- CustomLog /dev/stdout proxy env=forwarded
- </VirtualHost>
- Alias /placement /var/www/cgi-bin/placement/placement-api
- <Location /placement>
- SetHandler wsgi-script
- Options +ExecCGI
- WSGIProcessGroup placement-api
- WSGIApplicationGroup %{GLOBAL}
- WSGIPassAuthorization On
- </Location>
-
+ placement_api_uwsgi:
+ uwsgi:
+ processes: 1
+ add-header: "Connection: close"
+ buffer-size: 65535
+ die-on-term: true
+ enable-threads: true
+ exit-on-reload: false
+ hook-master-start: unix_signal:15 gracefully_kill_them_all
+ lazy-apps: true
+ log-x-forwarded-for: true
+ master: true
+ procname-prefix-spaced: "placement-api:"
+ route-user-agent: '^kube-probe.* donotlog:'
+ thunder-lock: true
+ worker-reload-mercy: 80
+ wsgi-file: /var/lib/openstack/bin/placement-api
endpoints:
cluster_domain_suffix: cluster.local
local_image_registry:
diff --git a/placement/values_overrides/tls.yaml b/placement/values_overrides/tls.yaml
index 514b66091e..1d511395c9 100644
--- a/placement/values_overrides/tls.yaml
+++ b/placement/values_overrides/tls.yaml
@@ -7,21 +7,23 @@ network:
conf:
software:
apache2:
+ binary: apache2
+ start_parameters: -DFOREGROUND
+ site_dir: /etc/apache2/sites-enabled
+ conf_dir: /etc/apache2/conf-enabled
+ mods_dir: /etc/apache2/mods-available
a2enmod:
- ssl
+ a2dismod: null
placement:
keystone_authtoken:
cafile: /etc/placement/certs/ca.crt
wsgi_placement: |
- Listen 0.0.0.0:{{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
- LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy
- SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
- CustomLog /dev/stdout combined env=!forwarded
- CustomLog /dev/stdout proxy env=forwarded
- <VirtualHost *:{{ tuple "placement" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}>
+ {{- $portInt := tuple "placement" "service" "api" $ | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+ Listen {{ $portInt }}
+ <VirtualHost *:{{ $portInt }}>
ServerName {{ printf "%s.%s.svc.%s" "placement-api" .Release.Namespace .Values.endpoints.cluster_domain_suffix }}
- WSGIDaemonProcess placement-api processes=4 threads=1 user=placement group=placement display-name=%{GROUP}
+ WSGIDaemonProcess placement-api processes=1 threads=1 user=placement group=placement display-name=%{GROUP}
WSGIProcessGroup placement-api
WSGIScriptAlias / /var/www/cgi-bin/placement/placement-api
WSGIApplicationGroup %{GLOBAL}
@@ -29,11 +31,10 @@ conf:
<IfVersion >= 2.4>
ErrorLogFormat "%{cu}t %M"
</IfVersion>
- ErrorLog /dev/stdout
SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+ ErrorLog /dev/stdout
CustomLog /dev/stdout combined env=!forwarded
CustomLog /dev/stdout proxy env=forwarded
-
SSLEngine on
SSLCertificateFile /etc/placement/certs/tls.crt
SSLCertificateKeyFile /etc/placement/certs/tls.key
@@ -41,14 +42,6 @@ conf:
SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
SSLHonorCipherOrder on
</VirtualHost>
- Alias /placement /var/www/cgi-bin/placement/placement-api
- <Location /placement>
- SetHandler wsgi-script
- Options +ExecCGI
- WSGIProcessGroup placement-api
- WSGIApplicationGroup %{GLOBAL}
- WSGIPassAuthorization On
- </Location>
endpoints:
identity:
auth:
diff --git a/releasenotes/notes/placement.yaml b/releasenotes/notes/placement.yaml
index d62862f45d..7618064538 100644
--- a/releasenotes/notes/placement.yaml
+++ b/releasenotes/notes/placement.yaml
@@ -37,4 +37,5 @@ placement:
- 0.3.12 Add 2024.1 overrides
- 0.3.13 Enable custom annotations for Openstack secrets
- 0.3.14 Update images used by default
+ - 0.3.15 Uses uWSGI for API service
...
diff --git a/tools/deployment/component/common/openstack.sh b/tools/deployment/component/common/openstack.sh
index d1dc15a7c5..b5138d9691 100755
--- a/tools/deployment/component/common/openstack.sh
+++ b/tools/deployment/component/common/openstack.sh
@@ -120,7 +120,8 @@ helm upgrade --install $release ${OSH_HELM_REPO}/openstack \
--set nova.conf.ceph.enabled=${CEPH_ENABLED} \
--values=/tmp/neutron.yaml \
--values=/tmp/glance.yaml \
- --namespace=$namespace
+ --namespace=$namespace \
+ --timeout=1200s
# If compute kit installed using Tungsten Fubric, it will be alive when Tunsten Fabric become active.
if [[ "$FEATURE_GATES" =~ (,|^)tf(,|$) ]]; then