stackube/pkg/openstack/loadbalancer.go

package openstack

import (
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/golang/glog"
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/listeners"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools"
	"github.com/gophercloud/gophercloud/pagination"
)

const (
	defaultMonitorDelay   = 10
	defaultMonitorRetry   = 3
	defaultMonotorTimeout = 3

	// loadbalancerActive* is configuration of exponential backoff for
	// going into ACTIVE loadbalancer provisioning status. Starting with 1
	// seconds, multiplying by 1.2 with each step and taking 19 steps at maximum
	// it will time out after 128s, which roughly corresponds to 120s
	loadbalancerActiveInitDealy = 1 * time.Second
	loadbalancerActiveFactor    = 1.2
	loadbalancerActiveSteps     = 19

	// loadbalancerDelete* is configuration of exponential backoff for
	// waiting for delete operation to complete. Starting with 1
	// seconds, multiplying by 1.2 with each step and taking 13 steps at maximum
	// it will time out after 32s, which roughly corresponds to 30s
	loadbalancerDeleteInitDealy = 1 * time.Second
	loadbalancerDeleteFactor    = 1.2
	loadbalancerDeleteSteps     = 13

	activeStatus = "ACTIVE"
	errorStatus  = "ERROR"
)

// LoadBalancer contains all essential information of kubernetes service.
type LoadBalancer struct {
	Name            string
	ServicePort     int
	TenantID        string
	SubnetID        string
	Protocol        string
	InternalIP      string
	ExternalIP      string
	SessionAffinity bool
	Endpoints       []Endpoint
}

// Endpoint represents a container endpoint.
type Endpoint struct {
	Address string
	Port    int
}

// LoadBalancerStatus contains the status of a load balancer.
type LoadBalancerStatus struct {
	InternalIP string
	ExternalIP string
}

// EnsureLoadBalancer ensures a load balancer is created.
func (os *Client) EnsureLoadBalancer(lb *LoadBalancer) (*LoadBalancerStatus, error) {
	// removes old one if already exists.
	loadbalancer, err := os.getLoadBalanceByName(lb.Name)
	if err != nil {
		if isNotFound(err) {
			// create a new one.
			lbOpts := loadbalancers.CreateOpts{
				Name:        lb.Name,
				Description: "Stackube service",
				VipSubnetID: lb.SubnetID,
				TenantID:    lb.TenantID,
			}
			loadbalancer, err = loadbalancers.Create(os.Network, lbOpts).Extract()
			if err != nil {
				glog.Errorf("Create load balancer %q failed: %v", lb.Name, err)
				return nil, err
			}
		}
	} else {
		glog.V(3).Infof("LoadBalancer %s already exists", lb.Name)
	}

	status, err := os.waitLoadBalancerStatus(loadbalancer.ID)
	if err != nil {
		glog.Errorf("Waiting for load balancer provision failed: %v", err)
		return nil, err
	}

	glog.V(3).Infof("Load balancer %q becomes %q", lb.Name, status)

	// get old listeners
	var listener *listeners.Listener
	oldListeners, err := os.getListenersByLoadBalancerID(loadbalancer.ID)
	if err != nil {
		return nil, fmt.Errorf("error getting LB %s listeners: %v", loadbalancer.Name, err)
	}
	for i := range oldListeners {
		l := oldListeners[i]
		if l.ProtocolPort == lb.ServicePort {
			listener = &l
		} else {
			// delete the obsolete listener
			if err := os.ensureListenerDeleted(loadbalancer.ID, l); err != nil {
				return nil, fmt.Errorf("error deleting listener %q: %v", l.Name, err)
			}
			os.waitLoadBalancerStatus(loadbalancer.ID)
		}
	}

	// create the listener.
	if listener == nil {
		lisOpts := listeners.CreateOpts{
			LoadbalancerID: loadbalancer.ID,
			// Only tcp is supported now.
			Protocol:     listeners.ProtocolTCP,
			ProtocolPort: lb.ServicePort,
			TenantID:     lb.TenantID,
			Name:         lb.Name,
		}
		listener, err = listeners.Create(os.Network, lisOpts).Extract()
		if err != nil {
			glog.Errorf("Create listener %q failed: %v", lb.Name, err)
			return nil, err
		}
		os.waitLoadBalancerStatus(loadbalancer.ID)
	}

	// create the load balancer pool.
	pool, err := os.getPoolByListenerID(loadbalancer.ID, listener.ID)
	if err != nil && !isNotFound(err) {
		return nil, fmt.Errorf("error getting pool for listener %q: %v", listener.ID, err)
	}
	if pool == nil {
		poolOpts := pools.CreateOpts{
			Name:       lb.Name,
			ListenerID: listener.ID,
			Protocol:   pools.ProtocolTCP,
			LBMethod:   pools.LBMethodRoundRobin,
			TenantID:   lb.TenantID,
		}
		if lb.SessionAffinity {
			poolOpts.Persistence = &pools.SessionPersistence{Type: "SOURCE_IP"}
		}
		pool, err = pools.Create(os.Network, poolOpts).Extract()
		if err != nil {
			glog.Errorf("Create pool %q failed: %v", lb.Name, err)
			return nil, err
		}
		os.waitLoadBalancerStatus(loadbalancer.ID)
	}

	// create load balancer members.
	members, err := os.getMembersByPoolID(pool.ID)
	if err != nil && !isNotFound(err) {
		return nil, fmt.Errorf("error getting members for pool %q: %v", pool.ID, err)
	}
	for _, ep := range lb.Endpoints {
		if !memberExists(members, ep.Address, ep.Port) {
			memberName := fmt.Sprintf("%s-%s-%d", lb.Name, ep.Address, ep.Port)
			_, err = pools.CreateMember(os.Network, pool.ID, pools.CreateMemberOpts{
				Name:         memberName,
				ProtocolPort: ep.Port,
				Address:      ep.Address,
				SubnetID:     lb.SubnetID,
			}).Extract()
			if err != nil {
				glog.Errorf("Create member %q failed: %v", memberName, err)
				return nil, err
			}
			os.waitLoadBalancerStatus(loadbalancer.ID)
		} else {
			members = popMember(members, ep.Address, ep.Port)
		}
	}
	// delete obsolete members
	for _, member := range members {
		glog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID,
			pool.ID, member.Address)
		err := pools.DeleteMember(os.Network, pool.ID, member.ID).ExtractErr()
		if err != nil && !isNotFound(err) {
			return nil, fmt.Errorf("error deleting member %s for pool %s address %s: %v",
				member.ID, pool.ID, member.Address, err)
		}
	}

	// create loadbalancer monitor.
	if pool.MonitorID == "" {
		_, err = monitors.Create(os.Network, monitors.CreateOpts{
			Name:       lb.Name,
			Type:       monitors.TypeTCP,
			PoolID:     pool.ID,
			TenantID:   lb.TenantID,
			Delay:      defaultMonitorDelay,
			Timeout:    defaultMonotorTimeout,
			MaxRetries: defaultMonitorRetry,
		}).Extract()
		if err != nil {
			glog.Errorf("Create monitor for pool %q failed: %v", pool.ID, err)
			return nil, err
		}
	}

	// associate external IP for the vip.
	fip, err := os.associateFloatingIP(lb.TenantID, loadbalancer.VipPortID, lb.ExternalIP)
	if err != nil {
		glog.Errorf("associateFloatingIP for port %q failed: %v", loadbalancer.VipPortID, err)
		return nil, err
	}

	return &LoadBalancerStatus{
		InternalIP: loadbalancer.VipAddress,
		ExternalIP: fip,
	}, nil
}

// GetLoadBalancer gets a load balancer by name.
func (os *Client) GetLoadBalancer(name string) (*LoadBalancer, error) {
	// get load balancer
	lb, err := os.getLoadBalanceByName(name)
	if err != nil {
		return nil, err
	}

	// get listener
	listener, err := os.getListenerByName(name)
	if err != nil {
		return nil, err
	}

	// get members
	endpoints := make([]Endpoint, 0)
	for _, pool := range listener.Pools {
		for _, m := range pool.Members {
			endpoints = append(endpoints, Endpoint{
				Address: m.Address,
				Port:    m.ProtocolPort,
			})
		}
	}

	return &LoadBalancer{
		Name:            lb.Name,
		ServicePort:     listener.ProtocolPort,
		TenantID:        lb.TenantID,
		SubnetID:        lb.VipSubnetID,
		Protocol:        listener.Protocol,
		InternalIP:      lb.VipAddress,
		SessionAffinity: listener.Pools[0].Persistence.Type != "",
		Endpoints:       endpoints,
	}, nil
}

// LoadBalancerExist returns whether a load balancer has already been exist.
func (os *Client) LoadBalancerExist(name string) (bool, error) {
	_, err := os.getLoadBalanceByName(name)
	if err != nil {
		if isNotFound(err) {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

// EnsureLoadBalancerDeleted ensures a load balancer is deleted.
func (os *Client) EnsureLoadBalancerDeleted(name string) error {
	// get load balancer
	lb, err := os.getLoadBalanceByName(name)
	if err != nil {
		if isNotFound(err) {
			return nil
		}

		return err
	}

	// delete floatingip
	floatingIP, err := os.getFloatingIPByPortID(lb.VipPortID)
	if err != nil && !isNotFound(err) {
		return fmt.Errorf("error getting floating ip by port %q: %v", lb.VipPortID, err)
	}
	if floatingIP != nil {
		err = floatingips.Delete(os.Network, floatingIP.ID).ExtractErr()
		if err != nil && !isNotFound(err) {
			return fmt.Errorf("error deleting floating ip %q: %v", floatingIP.ID, err)
		}
	}

	// get listeners and corelative pools and members
	var poolIDs []string
	var monitorIDs []string
	var memberIDs []string
	listenerList, err := os.getListenersByLoadBalancerID(lb.ID)
	if err != nil {
		return fmt.Errorf("Error getting load balancer %s listeners: %v", lb.ID, err)
	}

	for _, listener := range listenerList {
		pool, err := os.getPoolByListenerID(lb.ID, listener.ID)
		if err != nil && !isNotFound(err) {
			return fmt.Errorf("error getting pool for listener %s: %v", listener.ID, err)
		}
		poolIDs = append(poolIDs, pool.ID)
		if pool.MonitorID != "" {
			monitorIDs = append(monitorIDs, pool.MonitorID)
		}
	}
	for _, pool := range poolIDs {
		membersList, err := os.getMembersByPoolID(pool)
		if err != nil && !isNotFound(err) {
			return fmt.Errorf("Error getting pool members %s: %v", pool, err)
		}
		for _, member := range membersList {
			memberIDs = append(memberIDs, member.ID)
		}
	}

	// delete all monitors
	for _, monitorID := range monitorIDs {
		err := monitors.Delete(os.Network, monitorID).ExtractErr()
		if err != nil && !isNotFound(err) {
			return err
		}
		os.waitLoadBalancerStatus(lb.ID)
	}

	// delete all members and pools
	for _, poolID := range poolIDs {
		// delete all members for this pool
		for _, memberID := range memberIDs {
			err := pools.DeleteMember(os.Network, poolID, memberID).ExtractErr()
			if err != nil && !isNotFound(err) {
				return err
			}
			os.waitLoadBalancerStatus(lb.ID)
		}

		// delete pool
		err := pools.Delete(os.Network, poolID).ExtractErr()
		if err != nil && !isNotFound(err) {
			return err
		}
		os.waitLoadBalancerStatus(lb.ID)
	}

	// delete all listeners
	for _, listener := range listenerList {
		err := listeners.Delete(os.Network, listener.ID).ExtractErr()
		if err != nil && !isNotFound(err) {
			return err
		}
		os.waitLoadBalancerStatus(lb.ID)
	}

	// delete the load balancer
	err = loadbalancers.Delete(os.Network, lb.ID).ExtractErr()
	if err != nil && !isNotFound(err) {
		return err
	}
	os.waitLoadBalancerStatus(lb.ID)

	return nil
}

func (os *Client) ensureListenerDeleted(loadbalancerID string, listener listeners.Listener) error {
	for _, pool := range listener.Pools {
		for _, member := range pool.Members {
			// delete member
			if err := pools.DeleteMember(os.Network, pool.ID, member.ID).ExtractErr(); err != nil && !isNotFound(err) {
				return err
			}
			os.waitLoadBalancerStatus(loadbalancerID)
		}

		// delete monitor
		if err := monitors.Delete(os.Network, pool.MonitorID).ExtractErr(); err != nil && !isNotFound(err) {
			return err
		}
		os.waitLoadBalancerStatus(loadbalancerID)

		// delete pool
		if err := pools.Delete(os.Network, pool.ID).ExtractErr(); err != nil && !isNotFound(err) {
			return err
		}
		os.waitLoadBalancerStatus(loadbalancerID)
	}

	// delete listener
	if err := listeners.Delete(os.Network, listener.ID).ExtractErr(); err != nil && !isNotFound(err) {
		return err
	}
	os.waitLoadBalancerStatus(loadbalancerID)

	return nil
}

func (os *Client) waitLoadBalancerStatus(loadbalancerID string) (string, error) {
	backoff := wait.Backoff{
		Duration: loadbalancerActiveInitDealy,
		Factor:   loadbalancerActiveFactor,
		Steps:    loadbalancerActiveSteps,
	}

	var provisioningStatus string
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		loadbalancer, err := loadbalancers.Get(os.Network, loadbalancerID).Extract()
		if err != nil {
			return false, err
		}
		provisioningStatus = loadbalancer.ProvisioningStatus
		if loadbalancer.ProvisioningStatus == activeStatus {
			return true, nil
		} else if loadbalancer.ProvisioningStatus == errorStatus {
			return true, fmt.Errorf("Loadbalancer has gone into ERROR state")
		} else {
			return false, nil
		}

	})

	if err == wait.ErrWaitTimeout {
		err = fmt.Errorf("Loadbalancer failed to go into ACTIVE provisioning status within alloted time")
	}
	return provisioningStatus, err
}

func (os *Client) waitLoadbalancerDeleted(loadbalancerID string) error {
	backoff := wait.Backoff{
		Duration: loadbalancerDeleteInitDealy,
		Factor:   loadbalancerDeleteFactor,
		Steps:    loadbalancerDeleteSteps,
	}
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		_, err := loadbalancers.Get(os.Network, loadbalancerID).Extract()
		if err != nil {
			if err == ErrNotFound {
				return true, nil
			} else {
				return false, err
			}
		} else {
			return false, nil
		}
	})

	if err == wait.ErrWaitTimeout {
		err = fmt.Errorf("Loadbalancer failed to delete within the alloted time")
	}

	return err
}

func (os *Client) getListenersByLoadBalancerID(id string) ([]listeners.Listener, error) {
	var existingListeners []listeners.Listener
	err := listeners.List(os.Network, listeners.ListOpts{LoadbalancerID: id}).EachPage(func(page pagination.Page) (bool, error) {
		listenerList, err := listeners.ExtractListeners(page)
		if err != nil {
			return false, err
		}
		for _, l := range listenerList {
			for _, lb := range l.Loadbalancers {
				if lb.ID == id {
					existingListeners = append(existingListeners, l)
					break
				}
			}
		}

		return true, nil
	})
	if err != nil {
		return nil, err
	}

	return existingListeners, nil
}

func (os *Client) getLoadBalanceByName(name string) (*loadbalancers.LoadBalancer, error) {
	var lb *loadbalancers.LoadBalancer

	opts := loadbalancers.ListOpts{Name: name}
	pager := loadbalancers.List(os.Network, opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		lbs, err := loadbalancers.ExtractLoadBalancers(page)
		if err != nil {
			return false, err
		}

		switch len(lbs) {
		case 0:
			return false, ErrNotFound
		case 1:
			lb = &lbs[0]
			return true, nil
		default:
			return false, ErrMultipleResults
		}
	})
	if err != nil {
		return nil, err
	}

	if lb == nil {
		return nil, ErrNotFound
	}

	return lb, nil
}

func (os *Client) getPoolByListenerID(loadbalancerID string, listenerID string) (*pools.Pool, error) {
	listenerPools := make([]pools.Pool, 0, 1)
	err := pools.List(os.Network, pools.ListOpts{LoadbalancerID: loadbalancerID}).EachPage(
		func(page pagination.Page) (bool, error) {
			poolsList, err := pools.ExtractPools(page)
			if err != nil {
				return false, err
			}
			for _, p := range poolsList {
				for _, l := range p.Listeners {
					if l.ID == listenerID {
						listenerPools = append(listenerPools, p)
					}
				}
			}
			if len(listenerPools) > 1 {
				return false, ErrMultipleResults
			}
			return true, nil
		})
	if err != nil {
		if isNotFound(err) {
			return nil, ErrNotFound
		}
		return nil, err
	}

	if len(listenerPools) == 0 {
		return nil, ErrNotFound
	} else if len(listenerPools) > 1 {
		return nil, ErrMultipleResults
	}

	return &listenerPools[0], nil
}

// getPoolByName gets openstack pool by name.
func (os *Client) getPoolByName(name string) (*pools.Pool, error) {
	var pool *pools.Pool

	opts := pools.ListOpts{Name: name}
	pager := pools.List(os.Network, opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		ps, err := pools.ExtractPools(page)
		if err != nil {
			return false, err
		}

		switch len(ps) {
		case 0:
			return false, ErrNotFound
		case 1:
			pool = &ps[0]
			return true, nil
		default:
			return false, ErrMultipleResults
		}
	})
	if err != nil {
		return nil, err
	}

	if pool == nil {
		return nil, ErrNotFound
	}

	return pool, nil
}

func (os *Client) getListenerByName(name string) (*listeners.Listener, error) {
	var listener *listeners.Listener

	opts := listeners.ListOpts{Name: name}
	pager := listeners.List(os.Network, opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		lists, err := listeners.ExtractListeners(page)
		if err != nil {
			return false, err
		}

		switch len(lists) {
		case 0:
			return false, ErrNotFound
		case 1:
			listener = &lists[0]
			return true, nil
		default:
			return false, ErrMultipleResults
		}
	})
	if err != nil {
		return nil, err
	}

	if listener == nil {
		return nil, ErrNotFound
	}

	return listener, nil
}

func (os *Client) getMembersByPoolID(id string) ([]pools.Member, error) {
	var members []pools.Member
	err := pools.ListMembers(os.Network, id, pools.ListMembersOpts{}).EachPage(func(page pagination.Page) (bool, error) {
		membersList, err := pools.ExtractMembers(page)
		if err != nil {
			return false, err
		}
		members = append(members, membersList...)

		return true, nil
	})
	if err != nil {
		return nil, err
	}

	return members, nil
}

func (os *Client) getFloatingIPByPortID(portID string) (*floatingips.FloatingIP, error) {
	opts := floatingips.ListOpts{
		PortID: portID,
	}
	pager := floatingips.List(os.Network, opts)

	floatingIPList := make([]floatingips.FloatingIP, 0, 1)

	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		f, err := floatingips.ExtractFloatingIPs(page)
		if err != nil {
			return false, err
		}
		floatingIPList = append(floatingIPList, f...)
		if len(floatingIPList) > 1 {
			return false, ErrMultipleResults
		}
		return true, nil
	})
	if err != nil {
		if isNotFound(err) {
			return nil, ErrNotFound
		}
		return nil, err
	}

	if len(floatingIPList) == 0 {
		return nil, ErrNotFound
	} else if len(floatingIPList) > 1 {
		return nil, ErrMultipleResults
	}

	return &floatingIPList[0], nil
}

// Check if a member exists for node
func memberExists(members []pools.Member, addr string, port int) bool {
	for _, member := range members {
		if member.Address == addr && member.ProtocolPort == port {
			return true
		}
	}

	return false
}

func (os *Client) associateFloatingIP(tenantID, portID, floatingIPAddress string) (string, error) {
	var fip *floatingips.FloatingIP
	opts := floatingips.ListOpts{FloatingIP: floatingIPAddress}
	pager := floatingips.List(os.Network, opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		floatingipList, err := floatingips.ExtractFloatingIPs(page)
		if err != nil {
			return false, err
		}

		if len(floatingipList) > 0 {
			fip = &floatingipList[0]
		}

		return true, nil
	})
	if err != nil {
		return "", err
	}

	if fip != nil {
		if fip.PortID != "" {
			if fip.PortID == portID {
				glog.V(3).Infof("FIP %q has already been associated with port %q", floatingIPAddress, portID)
				return fip.FloatingIP, nil
			}
			// fip has already been used
			return fip.FloatingIP, fmt.Errorf("FloatingIP %v is already been binded to %v", floatingIPAddress, fip.PortID)
		}

		// Update floatingip
		floatOpts := floatingips.UpdateOpts{PortID: &portID}
		_, err = floatingips.Update(os.Network, fip.ID, floatOpts).Extract()
		if err != nil {
			glog.Errorf("Bind floatingip %v to %v failed: %v", floatingIPAddress, portID, err)
			return "", err
		}
	} else {
		// Create floatingip
		opts := floatingips.CreateOpts{
			FloatingNetworkID: os.ExtNetID,
			TenantID:          tenantID,
			FloatingIP:        floatingIPAddress,
			PortID:            portID,
		}
		fip, err = floatingips.Create(os.Network, opts).Extract()
		if err != nil {
			glog.Errorf("Create floatingip failed: %v", err)
			return "", err
		}
	}

	return fip.FloatingIP, nil
}

func popMember(members []pools.Member, addr string, port int) []pools.Member {
	for i, member := range members {
		if member.Address == addr && member.ProtocolPort == port {
			members[i] = members[len(members)-1]
			members = members[:len(members)-1]
		}
	}

	return members
}

func isNotFound(err error) bool {
	if err == ErrNotFound {
		return true
	}

	if _, ok := err.(*gophercloud.ErrResourceNotFound); ok {
		return true
	}

	if _, ok := err.(*gophercloud.ErrDefault404); ok {
		return true
	}

	return false
}