Add OVN network logging parser

Change-Id: I03a1c600c161536e693743219912199fabc1e5a5

ovn/templates/bin/_ovn-network-logging-parser.sh.tpl

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+COMMAND="${@:-start}"
+
+function start () {
+  exec uwsgi --ini /etc/neutron/neutron-ovn-network-logging-parser-uwsgi.ini
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND

ovn/templates/clusterrolebinding-controller.yaml

@@ -25,3 +25,6 @@ subjects:
 - kind: ServiceAccount
   name: ovn-controller
   namespace: {{ .Release.Namespace }}
+- kind: ServiceAccount
+  name: ovn-controller-gw
+  namespace: {{ .Release.Namespace }}

ovn/templates/configmap-bin.yaml

@@ -28,6 +28,8 @@ data:
 {{- end }}
   ovn-controller-init.sh: |
 {{ tuple "bin/_ovn-controller-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ovn-network-logging-parser.sh: |
+{{ tuple "bin/_ovn-network-logging-parser.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
 {{- end }}
 

ovn/templates/configmap-etc.yaml

@@ -17,6 +17,12 @@ limitations under the License.
 {{- $envAll := index . 1 }}
 {{- with $envAll }}
 
+{{- if empty (index .Values.conf.ovn_network_logging_parser_uwsgi.uwsgi "http-socket") -}}
+{{- $http_socket_port := tuple "ovn_logging_parser" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | toString }}
+{{- $http_socket := printf "0.0.0.0:%s" $http_socket_port }}
+{{- $_ := set .Values.conf.ovn_network_logging_parser_uwsgi.uwsgi "http-socket" $http_socket -}}
+{{- end -}}
+
 ---
 apiVersion: v1
 kind: Secret
@@ -25,7 +31,7 @@ metadata:
 type: Opaque
 data:
   auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }}
-
+  neutron-ovn-network-logging-parser-uwsgi.ini: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.ovn_network_logging_parser_uwsgi | b64enc }}
 {{- end }}
 {{- end }}
 

ovn/templates/daemonset-controller.yaml

@@ -133,6 +133,52 @@ spec:
               mountPath: /var/log/ovn
             - name: run-openvswitch
               mountPath: /run/ovn
+        {{- if .Values.pod.sidecars.vector }}
+        - name: vector
+{{ tuple $envAll "vector" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.vector | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "ovn_controller" "container" "vector" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - vector
+            - --config
+            - /etc/vector/vector.toml
+          volumeMounts:
+            - name: vector-config
+              mountPath: /etc/vector
+            - name: logs
+              mountPath: /logs
+            - name: vector-data
+              mountPath: /var/lib/vector
+        {{- end }}
+        {{- if .Values.pod.sidecars.ovn_logging_parser }}
+        - name: log-parser
+{{ tuple $envAll "ovn_logging_parser" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.ovn_logging_parser | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "ovn_controller" "container" "ovn_logging_parser" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /tmp/ovn-network-logging-parser.sh
+            - start
+          env:
+            - name: VECTOR_HTTP_ENDPOINT
+              value: http://localhost:5001
+          ports:
+            - name: http
+              containerPort: {{ tuple "ovn_logging_parser" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+              protocol: TCP
+          volumeMounts:
+            - name: neutron-etc
+              mountPath: /etc/neutron/neutron.conf
+              subPath: neutron.conf
+              readOnly: true
+            - name: ovn-bin
+              mountPath: /tmp/ovn-network-logging-parser.sh
+              subPath: ovn-network-logging-parser.sh
+              readOnly: true
+            - name: ovn-etc
+              mountPath: /etc/neutron/neutron-ovn-network-logging-parser-uwsgi.ini
+              subPath: neutron-ovn-network-logging-parser-uwsgi.ini
+              readOnly: true
+        {{- end }}
       volumes:
         - name: ovn-bin
           configMap:
@@ -156,6 +202,19 @@ spec:
             type: DirectoryOrCreate
         - name: gw-enabled
           emptyDir: {}
+        {{- if .Values.pod.sidecars.vector }}
+        - name: vector-config
+          secret:
+            secretName: ovn-vector-config
+        - name: vector-data
+          emptyDir: {}
+        {{- end }}
+        {{- if .Values.pod.sidecars.ovn_logging_parser }}
+        - name: neutron-etc
+          secret:
+            secretName: neutron-etc
+            defaultMode: 0444
+        {{- end }}
 {{- end }}
 {{- end }}
 

ovn/templates/rolebinding-controller.yaml

@@ -25,4 +25,7 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: ovn-controller
-  namespace: {{ .Release.Namespace }}
+  namespace: {{ .Release.Namespace }}
+- kind: ServiceAccount
+  name: ovn-controller-gw
+  namespace: {{ .Release.Namespace }}

ovn/templates/secret-vector.yaml

@@ -0,0 +1,26 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.pod.sidecars.vector }}
+{{- $envAll := . }}
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ovn-vector-config
+type: Opaque
+data:
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.vector "key" "vector.toml" "format" "Secret" ) | indent 2 }}
+{{- end }}

ovn/values.yaml

@@ -27,6 +27,8 @@ images:
     ovn_controller_kubectl: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_jammy
     dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal
     image_repo_sync: docker.io/library/docker:17.07.0
+    vector: docker.io/timberio/vector:0.39.0-debian
+    ovn_logging_parser: docker.io/openstackhelm/neutron:2024.1-ubuntu_jammy
   pull_policy: "IfNotPresent"
   local_registry:
     active: false
@@ -83,6 +85,55 @@ conf:
   #   br-public: eth1
   auto_bridge_add: {}
   ovs_user_name: openvswitch
+  ovn_network_logging_parser_uwsgi:
+    uwsgi:
+      add-header: "Connection: close"
+      buffer-size: 65535
+      die-on-term: true
+      enable-threads: true
+      exit-on-reload: false
+      hook-master-start: unix_signal:15 gracefully_kill_them_all
+      lazy-apps: true
+      log-x-forwarded-for: true
+      master: true
+      processes: 1
+      procname-prefix-spaced: "neutron-ovn-network-logging-parser:"
+      route-user-agent: '^kube-probe.* donotlog:'
+      thunder-lock: true
+      worker-reload-mercy: 80
+      wsgi-file: /var/lib/openstack/bin/neutron-ovn-network-logging-parser-wsgi
+  vector: |
+    [sources.file_logs]
+    type = "file"
+    include = [ "/logs/ovn-controller.log" ]
+
+    [sinks.ovn_log_parser_in]
+    type = "http"
+    inputs = ["file_logs"]
+    uri = "{{ tuple "ovn_logging_parser" "default" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}"
+    encoding.codec = "json"
+    method = "post"
+
+    [sources.ovn_log_parser_out]
+    type = "http_server"
+    address = "0.0.0.0:5001"
+    encoding = "json"
+
+    [transforms.parse_log_message]
+    type = "remap"
+    inputs = ["ovn_log_parser_out"]
+    source = '''
+      del(.source_type)
+      del(.path)
+    '''
+
+    [sinks.loki_sink]
+    type = "loki"
+    labels.event_source = "network_logs"
+    inputs = ["parse_log_message"]
+    endpoint = "http://loki.monitoring:3100"
+    encoding.codec = "json"
+    tenant_id = "{{`{{ project_id }}`}}"
 
 pod:
   # NOTE: should be same as nova.pod.use_fqdn.compute
@@ -103,6 +154,12 @@ pod:
         controller:
           readOnlyRootFilesystem: true
           privileged: true
+        ovn_logging_parser:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+        vector:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
   tolerations:
     ovn_ovsdb_nb:
       enabled: false
@@ -217,6 +274,20 @@ pod:
       limits:
         memory: "1024Mi"
         cpu: "2000m"
+    ovn_logging_parser:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "256Mi"
+        cpu: "500m"
+    vector:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "256Mi"
+        cpu: "500m"
     jobs:
       image_repo_sync:
         requests:
@@ -226,6 +297,10 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
 
+  sidecars:
+    ovn_logging_parser: false
+    vector: false
+
 secrets:
   oci_image_registry:
     ovn: ovn-oci-image-registry-key
@@ -284,6 +359,22 @@ endpoints:
         default: 6642
       raft:
         default: 6644
+  ovn_logging_parser:
+    name: ovn-logging-parser
+    namespace: null
+    hosts:
+      default: localhost
+    host_fqdn_override:
+      default: localhost
+    scheme:
+      default: 'http'
+      service: 'http'
+    path:
+      default: "/logs"
+    port:
+      api:
+        default: 9697
+        service: 9697
 
 network_policy:
   ovn_ovsdb_nb:

releasenotes/notes/ovn-a82eced671495a3d.yaml

@@ -0,0 +1,4 @@
+---
+ovn:
+  - Add OVN network logging parser
+...