sip/pkg/services/jumphost.go

/*
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     https://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package services

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"net/url"
	"strings"

	"github.com/go-logr/logr"
	"golang.org/x/crypto/ssh"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
	"sigs.k8s.io/controller-runtime/pkg/client"

	airshipv1 "sipcluster/pkg/api/v1"
	bmh "sipcluster/pkg/bmh"
)

const (
	JumpHostServiceName = "jumphost"

	subPathHosts     = "hosts"
	subPathSSHConfig = "ssh_config"

	sshDir             = "/home/ubuntu/.ssh"
	authorizedKeysFile = "authorized_keys"

	mountPathData               = "/etc/opt/sip"
	mountPathScripts            = "/opt/sip/bin"
	mountPathHosts              = mountPathData + "/" + subPathHosts
	mountPathSSHConfig          = sshDir + "/config"
	mountPathSSH                = sshDir + "/" + authorizedKeysFile
	mountPathNodeSSHPrivateKeys = mountPathData + "/" + nameNodeSSHPrivateKeysVolume

	nameDataVolume               = "data"
	nameScriptsVolume            = "scripts"
	nameAuthorizedKeysVolume     = "authorized-keys"
	nameNodeSSHPrivateKeysVolume = "ssh-private-keys"
)

// JumpHost is an InfrastructureService that provides SSH and power-management capabilities for sub-clusters.
type jumpHost struct {
	client   client.Client
	sipName  types.NamespacedName
	logger   logr.Logger
	config   airshipv1.JumpHostService
	machines *bmh.MachineList
}

func newJumpHost(name, namespace string, logger logr.Logger, config airshipv1.JumpHostService,
	machines *bmh.MachineList, client client.Client) InfraService {
	return jumpHost{
		sipName: types.NamespacedName{
			Name:      name,
			Namespace: namespace,
		},
		logger:   logger,
		config:   config,
		machines: machines,
		client:   client,
	}
}

// Deploy creates a JumpHost service in the base cluster.
func (jh jumpHost) Deploy() error {
	instance := JumpHostServiceName + "-" + jh.sipName.Name
	labels := map[string]string{
		// See https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels
		"app.kubernetes.io/part-of":   "sip",
		"app.kubernetes.io/name":      JumpHostServiceName,
		"app.kubernetes.io/component": JumpHostServiceName,
		"app.kubernetes.io/instance":  instance,
	}

	hostAliases := jh.generateHostAliases()

	// TODO: Validate Service becomes ready.
	service := jh.generateService(instance, labels)
	jh.logger.Info("Applying service", "service", service.GetNamespace()+"/"+service.GetName())
	err := applyRuntimeObject(client.ObjectKey{Name: service.GetName(), Namespace: service.GetNamespace()},
		service, jh.client)
	if err != nil {
		return err
	}

	secret, err := jh.generateSecret(instance, labels, hostAliases)
	if err != nil {
		return err
	}

	jh.logger.Info("Applying secret", "secret", secret.GetNamespace()+"/"+secret.GetName())
	err = applyRuntimeObject(client.ObjectKey{Name: secret.GetName(), Namespace: secret.GetNamespace()},
		secret, jh.client)
	if err != nil {
		return err
	}

	configMap, err := jh.generateConfigMap(instance, labels)
	if err != nil {
		return err
	}
	jh.logger.Info("Applying configmap", "configmap", configMap.GetNamespace()+"/"+configMap.GetName())
	err = applyRuntimeObject(client.ObjectKey{Name: configMap.GetName(), Namespace: configMap.GetNamespace()},
		configMap, jh.client)
	if err != nil {
		return err
	}

	// TODO: Validate Deployment becomes ready.
	deployment := jh.generateDeployment(instance, labels, hostAliases)
	jh.logger.Info("Applying deployment", "deployment", deployment.GetNamespace()+"/"+deployment.GetName())
	err = applyRuntimeObject(client.ObjectKey{Name: deployment.GetName(), Namespace: deployment.GetNamespace()},
		deployment, jh.client)
	if err != nil {
		return err
	}

	return nil
}

func (jh jumpHost) generateDeployment(instance string, labels map[string]string,
	hostAliases []corev1.HostAlias) *appsv1.Deployment {
	deployment := &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      instance,
			Namespace: jh.sipName.Namespace,
			Labels:    labels,
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: int32Ptr(1),
			Selector: &metav1.LabelSelector{
				MatchLabels: labels,
			},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: labels,
				},
				Spec: corev1.PodSpec{
					InitContainers: []corev1.Container{
						{
							Name:            "ssh-perms",
							Image:           jh.config.Image,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Command:         []string{"/bin/sh", "-c"},
							Args: []string{
								fmt.Sprintf("mkdir -m700 %s", sshDir),
							},
						},
					},
					Containers: []corev1.Container{
						{
							Name:            JumpHostServiceName,
							Image:           jh.config.Image,
							ImagePullPolicy: corev1.PullIfNotPresent,
							Ports: []corev1.ContainerPort{
								{
									Name:          "ssh",
									ContainerPort: 22,
								},
							},
							VolumeMounts: []corev1.VolumeMount{
								{
									Name:      nameDataVolume,
									MountPath: mountPathHosts,
									SubPath:   subPathHosts,
								},
								{
									Name:      nameScriptsVolume,
									MountPath: mountPathScripts,
								},
								{
									Name:      nameDataVolume,
									MountPath: mountPathSSHConfig,
									SubPath:   subPathSSHConfig,
								},
								{
									Name:      nameNodeSSHPrivateKeysVolume,
									MountPath: mountPathNodeSSHPrivateKeys,
								},
								{
									Name:      nameAuthorizedKeysVolume,
									MountPath: mountPathSSH,
									SubPath:   authorizedKeysFile,
								},
							},
							LivenessProbe: &corev1.Probe{
								Handler: corev1.Handler{
									TCPSocket: &corev1.TCPSocketAction{
										Port: intstr.FromString("ssh"),
									},
								},
								InitialDelaySeconds: 15,
								PeriodSeconds:       10,
							},
							ReadinessProbe: &corev1.Probe{
								Handler: corev1.Handler{
									TCPSocket: &corev1.TCPSocketAction{
										Port: intstr.FromString("ssh"),
									},
								},
								InitialDelaySeconds: 5,
								PeriodSeconds:       20,
							},
						},
					},
					SecurityContext: &corev1.PodSecurityContext{
						FSGroup: int64Ptr(1000),
					},
					Volumes: []corev1.Volume{
						{
							Name: nameDataVolume,
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									SecretName: instance,
								},
							},
						},
						{
							Name: nameAuthorizedKeysVolume,
							VolumeSource: corev1.VolumeSource{
								ConfigMap: &corev1.ConfigMapVolumeSource{
									LocalObjectReference: corev1.LocalObjectReference{
										Name: instance,
									},
									DefaultMode: int32Ptr(0700),
									Items: []corev1.KeyToPath{
										{
											Key:  nameAuthorizedKeysVolume,
											Path: authorizedKeysFile,
											Mode: int32Ptr(0600),
										},
									},
								},
							},
						},
						{
							Name: nameScriptsVolume,
							VolumeSource: corev1.VolumeSource{
								ConfigMap: &corev1.ConfigMapVolumeSource{
									LocalObjectReference: corev1.LocalObjectReference{
										Name: instance,
									},
									DefaultMode: int32Ptr(0777),
								},
							},
						},
						{
							Name: nameNodeSSHPrivateKeysVolume,
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									SecretName: jh.config.NodeSSHPrivateKeys,
								},
							},
						},
					},
					HostAliases: hostAliases,
				},
			},
		},
	}

	// Set NO_PROXY env variables when Redfish proxy setting is false (Default: false).
	var proxy bool
	if jh.config.BMC != nil {
		proxy = jh.config.BMC.Proxy
	}
	if proxy == false {
		// TODO: We may need to identify the container with Redfish functionality in the future if some
		// containers require communication over a proxy server.
		for _, container := range deployment.Spec.Template.Spec.Containers {
			container.Env = []corev1.EnvVar{
				{
					Name:  "NO_PROXY",
					Value: "*",
				},
				{
					Name:  "no_proxy",
					Value: "*",
				},
			}
		}
	}

	return deployment
}

func (jh jumpHost) generateConfigMap(instance string, labels map[string]string) (*corev1.ConfigMap, error) {
	for _, key := range jh.config.SSHAuthorizedKeys {
		_, _, _, _, err := ssh.ParseAuthorizedKey([]byte(key))
		if err != nil {
			return nil, ErrInvalidAuthorizedKeyFormat{SSHErr: err.Error(), Key: key}
		}
	}

	return &corev1.ConfigMap{
		TypeMeta: metav1.TypeMeta{
			APIVersion: corev1.SchemeGroupVersion.String(),
			Kind:       "ConfigMap",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      instance,
			Namespace: jh.sipName.Namespace,
			Labels:    labels,
		},
		Data: map[string]string{
			nameAuthorizedKeysVolume: strings.Join(jh.config.SSHAuthorizedKeys, "\n"),
			"host":                   fmt.Sprintf(rebootScript, mountPathHosts),
		},
	}, nil
}

func (jh jumpHost) generateSecret(instance string, labels map[string]string, hostAliases []corev1.HostAlias) (
	*corev1.Secret, error) {
	hostData, err := generateHostList(*jh.machines)
	if err != nil {
		return nil, err
	}
	sshConfig, err := jh.generateSSHConfig(hostAliases)
	if err != nil {
		return nil, err
	}

	secret := &corev1.Secret{
		TypeMeta: metav1.TypeMeta{
			APIVersion: corev1.SchemeGroupVersion.String(),
			Kind:       "Secret",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      instance,
			Namespace: jh.sipName.Namespace,
			Labels:    labels,
		},
		Data: map[string][]byte{
			subPathHosts:     hostData,
			subPathSSHConfig: sshConfig,
		},
	}

	return secret, nil
}

func (jh jumpHost) generateSSHConfig(hostAliases []corev1.HostAlias) ([]byte, error) {
	key := types.NamespacedName{
		Namespace: jh.sipName.Namespace,
		Name:      jh.config.NodeSSHPrivateKeys,
	}
	secret := &corev1.Secret{}
	if err := jh.client.Get(context.Background(), key, secret); err != nil {
		return nil, err
	}

	identityFiles := []string{}
	for k := range secret.Data {
		identityFiles = append(identityFiles, mountPathNodeSSHPrivateKeys+"/"+k)
	}
	hostNames := []string{}
	for _, hostAlias := range hostAliases {
		hostNames = append(hostNames, hostAlias.Hostnames[0])
	}

	tmpl, err := template.New("ssh-config").Parse(sshConfigTemplate)
	if err != nil {
		return nil, err
	}

	data := sshConfigTemplateData{
		IdentityFiles: identityFiles,
		HostNames:     hostNames,
	}

	w := bytes.NewBuffer([]byte{})
	if err := tmpl.Execute(w, data); err != nil {
		return nil, err
	}

	rendered := w.Bytes()
	return rendered, nil
}

type sshConfigTemplateData struct {
	IdentityFiles []string
	HostNames     []string
}

const sshConfigTemplate = `
Host *
{{- range .IdentityFiles }}
	IdentityFile {{ . }}
{{ end -}}

{{- range .HostNames }}
Host {{ . }}
	HostName {{ . }}
{{ end -}}
`

func (jh jumpHost) generateHostAliases() []corev1.HostAlias {
	hostAliases := []corev1.HostAlias{}
	for _, machine := range jh.machines.Machines {
		namespace := machine.BMH.Namespace
		name := machine.BMH.Name
		ip, exists := machine.Data.IPOnInterface[jh.config.NodeInterface]
		if !exists {
			jh.logger.Info("Machine does not have ip to be aliased",
				"interface", jh.config.NodeInterface,
				"machine", namespace+"/"+name,
			)
			continue
		}
		hostname := machine.BMH.Name
		hostAliases = append(hostAliases, corev1.HostAlias{IP: ip, Hostnames: []string{hostname}})
	}
	return hostAliases
}

func (jh jumpHost) generateService(instance string, labels map[string]string) *corev1.Service {
	return &corev1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name:      instance,
			Namespace: jh.sipName.Namespace,
		},
		Spec: corev1.ServiceSpec{
			Ports: []corev1.ServicePort{
				{
					Name:     "ssh",
					Port:     22,
					NodePort: int32(jh.config.NodePort),
				},
			},
			Selector: labels,
			Type:     corev1.ServiceTypeNodePort,
		},
	}
}

// Finalize removes a deployed JumpHost service.
func (jh jumpHost) Finalize() error {
	// TODO(drewwalters96): Add logic to cleanup SIPCluster JumpHost pod.
	return nil
}

type host struct {
	Name string `json:"name"`
	BMC  bmc    `json:"bmc"`
}

type bmc struct {
	IP       string `json:"ip"`
	Username string `json:"username"`
	Password string `json:"password"`
}

// generateHostList creates a list of hosts in JSON format to be mounted as a config map to the jump host pod and used
// to power cycle sub-cluster nodes.
func generateHostList(machineList bmh.MachineList) ([]byte, error) {
	hosts := make([]host, len(machineList.Machines))
	for name, machine := range machineList.Machines {
		managementIP, err := getManagementIP(machine.BMH.Spec.BMC.Address)
		if err != nil {
			return nil, err
		}

		h := host{
			Name: name,
			BMC: bmc{
				IP:       managementIP,
				Username: machine.Data.BMCUsername,
				Password: machine.Data.BMCPassword,
			},
		}

		hosts = append(hosts, h)
	}

	out, err := json.Marshal(hosts)
	if err != nil {
		return nil, err
	}

	return out, nil
}

// getManagementIP parses the BMC IP address from a Redfish fully qualified domain name. For example, the input
// redfish+https://127.0.0.1/redfish/v1/Systems/System.Embedded.1 yields 127.0.0.1.
func getManagementIP(redfishURL string) (string, error) {
	parsedURL, err := url.Parse(redfishURL)
	if err != nil {
		return "", ErrMalformedRedfishAddress{Address: redfishURL}
	}

	return parsedURL.Host, nil
}

var rebootScript = `#!/bin/sh

# Support Infrastructure Provider (SIP) Host Utility
# DO NOT MODIFY: generated by SIP

HOSTS_FILE="%s"

LIST_COMMAND="list"
REBOOT_COMMAND="reboot"

help() {
  echo "Support Infrastructure Provider (SIP) Host Utility"
  echo ""
  echo "Usage: ${LIST_COMMAND}                      list hosts"
  echo "       ${REBOOT_COMMAND} [host name]        reboot host"
}

dep_check() {
  if [ "$(which jq)" = "" ]; then
    echo "Missing package 'jq'. Update your JumpHost image to include 'jq' and 'redfishtool'."
    exit 1
  fi

  if [ "$(which redfishtool)" = "" ]; then
    echo "Missing package 'redfishtool'. Update your JumpHost image to include 'jq' and 'redfishtool'."
    exit 1
  fi
}

get_bmc_info() {
  for host in $(jq -r -c '.[]' ${HOSTS_FILE}); do
    if [ "$(echo "$host" | jq -r '.name')" = "$1" ]; then
      addr=$(echo "$host" | jq -r '.bmc.ip')
      user=$(echo "$host" | jq -r '.bmc.username')
      pass=$(echo "$host" | jq -r '.bmc.password')
    fi
  done
}

reboot() {
  get_bmc_info "$1"
  if [ "${addr}" = "" ] || [ "${user}" = "" ] || [ "$pass" = "" ]; then
    echo "Invalid host '$1'. Use the '${LIST_COMMAND}' command to view hosts."
    exit 1
  fi

  echo "Rebooting host '$1'"
  redfishtool -r "${addr}" -u "${user}" -p "${pass}" \
    Systems reset GracefulRestart -vvvvv
  exit 0
}

case $1 in
  "${LIST_COMMAND}")
    dep_check
    jq -r '.[].name' ${HOSTS_FILE}
    ;;
  "${REBOOT_COMMAND}")
    if [ "$2" = "" ]; then
      printf "Host name required.\n\n"
      help
      exit 1
    fi
    dep_check
    reboot "$2"
    ;;
  *)
    help
    ;;
esac
`

/*

The SIP Cluster operator will manufacture a jump host pod specifically for this
tenant cluster.  Much like we did above for master nodes by extracting IP
addresses, we would need to extract the `oam-ipv4` ip address for all nodes and
create a configmap to bind mount into the pod so it understands what host IPs
represent the clusters.

The expectation is the Jump Pod runs `sshd` protected by `uam` to allow
operators to SSH directly to the Jump Pod and authenticate via UAM to
immediately access their cluster.

It will provide the following functionality over SSH:

- The Jump Pod will be fronted by a `NodePort` service to allow incoming ssh.
- The Jump Pod will be UAM secured (for SSH)
- Bind mount in cluster-specific SSH key for cluster
- Ability to Power Cycle the cluster VMs
- A kubectl binary and kubeconfig (cluster-admin) for the cluster
- SSH access to the cluster node VMs
- Libvirt console logs for the VMs
  - We will secure libvirt with tls and provide keys to every jump host
    with curated interfaces to extract logs remotely for all VMs for their
    clusters.

*/