airship/maas
Code Issues Proposed changes

syslog listener for physical nodes

Browse Source 
- Add a new pod running syslog to receive syslog
  messages containing the console logs of bootstrapping
  nodes. This aids in troubleshooting without requiring
  accessing the OOB console.
- Add a UDP forwarder to the MAAS ingress controller
  as nodes attempt to send syslogs to UDP 514 of the region
  controller

Change-Id: I3f508225f4394a90c6f2534a51f262b42c1afa4e
This commit is contained in:
Scott Hussey 2018-12-26 12:02:21 -06:00 committed by Scott Hussey
parent 922db8aee1
commit 5b2a7a2439
13 changed files with 333 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
charts/maas/templates/bin/_start-syslog.sh.tpl Normal file
View File

@ -0,0 +1,31 @@
#!/bin/bash
{{/*
Copyright 2019 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.*/}}
set -x
RSYSLOG_BIN=${RSYSLOG_BIN:-"/usr/sbin/rsyslogd"}
RSYSLOG_CONFFILE=${RSYSLOG_CONFFILE:-"/etc/rsyslog.conf"}
LOGFILE=${LOGFILE:-"/var/log/maas/nodeboot.log"}
$RSYSLOG_BIN -f "$RSYSLOG_CONFFILE"
# Handle race waiting for rsyslogd to start logging
while true
do
tail -f "$LOGFILE"
echo "Waiting for log file to exist."
sleep 10
done

2
charts/maas/templates/configmap-bin.yaml
View File

@ -47,3 +47,5 @@ data:
{{ tuple "bin/_maas-ingress-errors.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
maas-vip-configure: |
{{ tuple "bin/_maas-vip-configure.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
start-syslog.sh: |
{{ tuple "bin/_start-syslog.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

6
charts/maas/templates/configmap-etc.yaml
View File

@ -46,3 +46,9 @@ data:
{{- end }}
sleep-inittab: |
::sysinit:/tmp/maas-vip-configure.sh sleep
rsyslog.conf: |
{{ tuple "etc/_rsyslog.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
logrotate.conf: |
{{ tuple "etc/_logrotate.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
logrotate.cron: |
{{ tuple "etc/_logrotate.cron.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

1
charts/maas/templates/configmap-ingress.yaml
View File

@ -31,6 +31,7 @@ metadata:
name: maas-ingress-services-udp
data:
'53': "{{- .Release.Namespace -}}/{{- tuple "maas_region" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}}:region-dns"
"514": "{{- .Release.Namespace -}}/{{- tuple "maas_syslog" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}}:syslog"
...
---
apiVersion: v1

13
charts/maas/templates/etc/_logrotate.conf.tpl Normal file
View File

@ -0,0 +1,13 @@
{{ printf "%s/%s" .Values.conf.syslog.logpath .Values.conf.syslog.logfile }}
{
rotate {{ .Values.conf.syslog.logrotate.rotate }}
size {{ .Values.conf.syslog.logrotate.size }}
missingok
delaycompress
compress
nocreate
nomail
postrotate
killall -s HUP rsyslogd >/dev/null 2>&1 || true
endscript
}

1
charts/maas/templates/etc/_logrotate.cron.tpl Normal file
View File

@ -0,0 +1 @@
@hourly root /usr/sbin/logrotate /etc/logrotate.conf

11
charts/maas/templates/etc/_rsyslog.conf.tpl Normal file
View File

@ -0,0 +1,11 @@
# Enable the udp server for installation logging
$ModLoad imudp
$UDPServerRun {{ tuple "maas_region" "podport" "region_syslog" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
#$ModLoad imtcp # load TCP listener
# Reduce message repetition
$RepeatedMsgReduction on
##$RepeatedMsgContainsOriginalMsg on
:fromhost-ip, !isequal, "127.0.0.1" {{ .Values.conf.syslog.logpath }}/{{ .Values.conf.syslog.logfile }}
& ~

3
charts/maas/templates/service-region.yaml
View File

@ -38,6 +38,9 @@ spec:
port: 53
targetPort: 53
protocol: "UDP"
- name: region-syslog
port: 514
targetPort: {{ tuple "maas_region" "podport" "region_syslog" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
selector:
{{ tuple $envAll "maas" "region" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ if or .Values.network.region_proxy.node_port.enabled .Values.network.region_api.node_port.enabled }}

31
charts/maas/templates/service-syslog.yaml Normal file
View File

@ -0,0 +1,31 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.syslog_service -}}
{{- $envAll := . }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "maas_syslog" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
spec:
ports:
- name: syslog
port: 514
protocol: "UDP"
targetPort: {{ tuple "maas_syslog" "podport" "syslog" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
selector:
{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
...
{{- end -}}

129
charts/maas/templates/statefulset-maas-syslog.yaml Normal file
View File

@ -0,0 +1,129 @@
{{/*
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. */}}
{{- if .Values.manifests.syslog_statefulset }}
{{- $envAll := . }}
{{- $serviceAccountName := "maas-syslog" }}
{{- $mounts_maas_syslog := .Values.pod.mounts.maas_syslog.syslog }}
{{- $mounts_maas_syslog_init := .Values.pod.mounts.maas_syslog.init_container }}
{{ tuple $envAll "syslog" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: maas-syslog
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
replicas: {{ .Values.pod.replicas.syslog }}
podManagementPolicy: 'Parallel'
updateStrategy:
type: 'RollingUpdate'
selector:
matchLabels:
{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
template:
metadata:
labels:
{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
{{ dict "envAll" $envAll "podName" "maas-syslog" "containerNames" (list "syslog") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
serviceAccountName: {{ $serviceAccountName }}
shareProcessNamespace: true
affinity:
{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
nodeSelector:
{{ .Values.labels.syslog.node_selector_key }}: {{ .Values.labels.syslog.node_selector_value }}
initContainers:
{{ tuple $envAll "maas_syslog" $mounts_maas_syslog_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: syslog
image: {{ .Values.images.tags.maas_syslog }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/start-syslog.sh
env:
- name: RSYSLOG_CONFFILE
value: "/etc/rsyslog.conf"
- name: LOGFILE
value: {{ printf "%s/%s" .Values.conf.syslog.logpath .Values.conf.syslog.logfile | quote }}
volumeMounts:
- mountPath: /tmp
name: pod-tmp
- mountPath: /etc/rsyslog.conf
name: maas-etc
subPath: rsyslog.conf
readOnly: true
- mountPath: /tmp/start-syslog.sh
name: maas-bin
subPath: start-syslog.sh
readOnly: true
- mountPath: {{ .Values.conf.syslog.logpath }}
name: syslog-archive
readOnly: false
- name: logrotate
image: {{ .Values.images.tags.maas_syslog }}
imagePullPolicy: {{ .Values.images.pull_policy }}
# Run cron in the foreground and only log failed cronjobs (when logrotate fails)
command:
- "cron"
- "-f"
- "-L"
- "4"
volumeMounts:
- mountPath: /tmp
name: pod-tmp
- mountPath: /etc/logrotate.conf
name: maas-etc
subPath: logrotate.conf
readOnly: true
- mountPath: /etc/crontab
name: maas-etc
subPath: logrotate.cron
readOnly: true
- mountPath: /var/lib/logrotate
name: logrotate-state
readOnly: false
- mountPath: {{ .Values.conf.syslog.logpath }}
name: syslog-archive
readOnly: false
volumes:
- name: pod-tmp
emptyDir: {}
- name: logrotate-state
emptyDir: {}
- name: maas-etc
configMap:
name: maas-etc
defaultMode: 0444
- name: maas-bin
configMap:
name: maas-bin
defaultMode: 0555
volumeClaimTemplates:
- metadata:
name: syslog-archive
annotations:
{{ .Values.storage.syslog.pvc.class_path }}: {{ .Values.storage.syslog.pvc.class_name }}
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.storage.syslog.pvc.size }}
...
{{- end }}

11
charts/maas/templates/statefulset-rack.yaml
View File

@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.rack_deployment }}
{{- if .Values.manifests.rack_statefulset }}
{{- $envAll := . }}
{{- $serviceAccountName := "maas-rack" }}
{{- $mounts_maas_rack := .Values.pod.mounts.maas_rack }}
@ -23,13 +23,18 @@ limitations under the License.
{{ tuple $envAll "rack_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: apps/v1beta1
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: maas-rack
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll "maas" "rack" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
selector:
matchLabels:
{{ tuple $envAll "maas" "rack" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
serviceName: maas-rack
replicas: {{ .Values.pod.replicas.rack }}
podManagementPolicy: 'Parallel'
@ -90,7 +95,7 @@ spec:
- 'SYS_TIME'
readinessProbe:
initialDelaySeconds: 60
periodSeconds: 300
periodSeconds: 60
timeoutSeconds: 60
exec:
command:

44
charts/maas/templates/statefulset-region.yaml
View File

@ -20,13 +20,18 @@
{{ tuple $envAll "region_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: apps/v1beta1
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: maas-region
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll "maas" "region" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
selector:
matchLabels:
{{ tuple $envAll "maas" "region" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
serviceName: maas-region-ui
replicas: {{ .Values.pod.replicas.region }}
updateStrategy:
@ -53,11 +58,36 @@ spec:
- name: maas-cache
image: {{ .Values.images.tags.maas_cache }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{- end }}
{{- if .Values.manifests.maas_syslog }}
- name: maas-syslog
image: {{ .Values.images.tags.maas_region }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/start-syslog.sh
env:
- name: RSYSLOG_CONFFILE
value: "/etc/rsyslog.conf"
- name: LOGFILE
value: {{ printf "%s/%s" .Values.conf.syslog.logpath .Values.conf.syslog.logfile | quote }}
volueMounts:
- mountPath: /etc/rsyslog.conf
name: maas-etc
subPath: rsyslog.conf
readOnly: true
- mountPath: /tmp/start-syslog.sh
name: maas-bin
subPath: start-syslog.sh
readOnly: true
- mountPath: {{ .Values.conf.syslog.logpath }}
name: syslog-archive
readOnly: false
{{- end }}
- name: maas-region
image: {{ .Values.images.tags.maas_region }}
imagePullPolicy: {{ .Values.images.pull_policy }}
tty: true
{{ tuple $envAll $envAll.Values.pod.resources.maas_region | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
ports:
- name: region-api
@ -149,5 +179,17 @@ spec:
secret:
secretName: {{ .Values.secrets.maas_region.name }}
{{- if $mounts_maas_region.volumes }}{{ toYaml $mounts_maas_region.volumes | indent 8 }}{{ end }}
{{- if .Values.manifests.maas_syslog }}
volumeClaimTemplates:
- metadata:
name: syslog-archive
annotations:
{{ .Values.storage.syslog.pvc.class_path }}: {{ .Values.storage.syslog.pvc.class_name }}
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: {{ .Values.storage.syslog.pvc.size }}
{{- end }}
...
{{- end }}

55
charts/maas/values.yaml
View File

@ -20,6 +20,7 @@
dependencies:
static:
maas_ingress: {}
maas_syslog: {}
rack_controller:
services:
- service: maas_region
@ -76,7 +77,9 @@ dependencies:
manifests:
region_statefulset: true
rack_deployment: true
rack_statefulset: true
syslog_statefulset: true
syslog_service: true
test_maas_api: true
secret_ssh_key: false
ingress_region: true
@ -96,6 +99,7 @@ images:
ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.20.0
ingress_vip: docker.io/busybox:latest
error_pages: gcr.io/google_containers/defaultbackend:1.0
maas_syslog: quay.io/airshipit/maas-region-controller:latest
pull_policy: IfNotPresent
local_registry:
# TODO(portdirect): this chart does not yet support local image cacheing
@ -120,6 +124,9 @@ labels:
ingress:
node_selector_key: ucp-control-plane
node_selector_value: enabled
syslog:
node_selector_key: ucp-control-plane
node_selector_value: enabled
network:
maas_ingress:
@ -141,6 +148,11 @@ network:
enabled: false
storage:
syslog:
pvc:
class_path: volume.beta.kubernetes.io/storage-class
class_name: general
size: 1Gi
rackd:
pvc:
class_path: volume.beta.kubernetes.io/storage-class
@ -164,6 +176,14 @@ conf:
bootaction_url: null
cache:
enabled: false
syslog:
logpath: /var/log/maas
logfile: nodeboot.log
logrotate:
# How many rotated logs to keep
rotate: '30'
# Size threshold when a log should rotate
size: '100M'
maas:
override:
append:
@ -254,7 +274,16 @@ pod:
maas-rack: localhost/docker-default
maas-region:
maas-region: localhost/docker-default
maas-syslog:
syslog: localhost/docker-default
security_context:
maas-syslog:
pod:
runAsUser: 99
container:
syslog:
runAsUser: 99
readOnlyRootFilesystem: true
ingress_errors:
pod:
runAsUser: 99
@ -282,9 +311,13 @@ pod:
maas_ingress:
init_container: []
maas_region:
maas_syslog:
init_container: []
syslog: []
replicas:
rack: 2
region: 2
syslog: 1
resources:
enabled: false
test:
@ -308,6 +341,13 @@ pod:
requests:
memory: "128Mi"
cpu: "500m"
maas_syslog:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
maas_ingress_vip:
limits:
memory: "128Mi"
@ -389,6 +429,16 @@ endpoints:
podport: 10254
status:
podport: 18080
maas_syslog:
hosts:
default: maas-syslog
host_fqdn_override:
default: null
port:
syslog:
public: 514
podport: 514
maas_db:
auth:
admin:
@ -432,6 +482,9 @@ endpoints:
# comment them out and let the default rule
# podport: 8000
# public: 8000
region_syslog:
default: 514
podport: 514
host_fqdn_override:
default: null
physicalprovisioner: