From 19e7fdafe3a32118cef276c90ba307523909a1a9 Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Thu, 27 Feb 2025 17:38:10 -0600 Subject: [PATCH] Allow to specify GNP to access to a software repo Sometimes armada isn't able to download the chart due to restricting network policies in k8s cluster. This patch allows to specify chart name which contains updated network policies and temporaty GNP template to fetch the new network policies and apply them. Change-Id: I7a9feb2ecd460618d7606bc8a31fdb2e97a31f85 Signed-off-by: Ruslan Aliev --- go.mod | 2 + go.sum | 10 ++ .../armada-operator/Dockerfile.ubuntu_jammy | 2 +- pkg/controller/armadachart_controller.go | 161 +++++++++++++++++- 4 files changed, 171 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 41da756..17e691b 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 + github.com/tigera/api v0.0.0-20230406222214-ca74195900cb helm.sh/helm/v3 v3.16.4 k8s.io/api v0.31.5 k8s.io/apiextensions-apiserver v0.31.5 @@ -84,6 +85,7 @@ require ( github.com/huandu/xstrings v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jinzhu/copier v0.3.5 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index cdf3493..b35073f 100644 --- a/go.sum +++ b/go.sum @@ -220,6 +220,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -294,6 +296,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= @@ -365,6 +371,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tigera/api v0.0.0-20230406222214-ca74195900cb h1:Y7r5Al3V235KaEoAzGBz9RYXEbwDu8CPaZoCq2PlD8w= +github.com/tigera/api v0.0.0-20230406222214-ca74195900cb/go.mod h1:ZZghiX3CUsBAc0osBjRvV6y/eun2ObYdvSbjqXAoj/w= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -510,6 +518,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/images/armada-operator/Dockerfile.ubuntu_jammy b/images/armada-operator/Dockerfile.ubuntu_jammy index bcd3ae3..4b8d002 100644 --- a/images/armada-operator/Dockerfile.ubuntu_jammy +++ b/images/armada-operator/Dockerfile.ubuntu_jammy @@ -1,6 +1,6 @@ ARG FROM=quay.io/airshipit/ubuntu:jammy # Build the manager binary -FROM quay.io/airshipit/golang:1.23.1-bullseye as builder +FROM quay.io/airshipit/golang:1.23.1-bullseye AS builder ARG TARGETOS ARG TARGETARCH diff --git a/pkg/controller/armadachart_controller.go b/pkg/controller/armadachart_controller.go index 3e97a09..cb58812 100644 --- a/pkg/controller/armadachart_controller.go +++ b/pkg/controller/armadachart_controller.go @@ -22,15 +22,20 @@ import ( "errors" "fmt" "io" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "net" "net/http" + "net/url" "os" + "slices" + "strings" "time" "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hashicorp/go-retryablehttp" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + calico "github.com/tigera/api/pkg/client/clientset_generated/clientset" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" @@ -40,12 +45,15 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apiextension "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -468,10 +476,38 @@ func (r *ArmadaChartReconciler) loadHelmChart(ctx context.Context, hr armadav1.A return nil, fmt.Errorf("failed to create a new request: %w", err) } - resp, err := r.httpClient.Do(req) + var resp *http.Response + resp, err = r.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to download artifact, error: %w", err) + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + log.Info(fmt.Sprintf("failed to download artifact due to %s, attempting to apply temporary GNP", err.Error())) // remove + + urlInfo, err := url.Parse(source) + if err != nil { + return nil, err + } + hostname := strings.TrimPrefix(urlInfo.Hostname(), "www.") + + ips, err := net.LookupIP(hostname) + if err != nil { + return nil, err + } + + var nets []string + for _, ip := range ips { + nets = append(nets, fmt.Sprintf("%s/32", ip.String())) + } + + resp, err = r.downloadWithGNP(hr.Spec.ChartName, req, nets, log) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("failed to download artifact, error: %w", err) + } } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { @@ -486,6 +522,125 @@ func (r *ArmadaChartReconciler) loadHelmChart(ctx context.Context, hr armadav1.A return loader.Load(f.Name()) } +func (r *ArmadaChartReconciler) getGNPTemplate(chartName string, log logr.Logger) ([]byte, error) { + operatorNamespace := os.Getenv("NAMESPACE") + configGetter, err := r.buildRESTClientGetter(operatorNamespace, log) + if err != nil { + return nil, err + } + restConfig, err := configGetter.ToRESTConfig() + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, err + } + + configMap, err := clientSet.CoreV1().ConfigMaps(operatorNamespace).Get(context.TODO(), "armada-gnp", v1.GetOptions{}) + if err != nil { + return nil, err + } + + if val, ok := configMap.Data[chartName]; ok { + return []byte(val), nil + } + + return nil, errors.New(fmt.Sprintf("failed to download chart %s, no GNP policy found", chartName)) +} + +func (r *ArmadaChartReconciler) getGNP(chartName string, log logr.Logger) (*v3.GlobalNetworkPolicy, error) { + sch := runtime.NewScheme() + if err := v3.AddToScheme(sch); err != nil { + return nil, err + } + + cfgMapData, err := r.getGNPTemplate(chartName, log) + if err != nil { + return nil, err + } + + obj, _, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(cfgMapData, nil, nil) + if err != nil { + return nil, err + } + + return obj.(*v3.GlobalNetworkPolicy), nil +} + +func (r *ArmadaChartReconciler) downloadWithGNP(chartName string, req *retryablehttp.Request, nets []string, log logr.Logger) (*http.Response, error) { + armadaGNP, err := r.getGNP(chartName, log) + if err != nil { + return nil, err + } + + clientGetter, err := r.buildRESTClientGetter("default", log) + if err != nil { + return nil, err + } + restConfig, err := clientGetter.ToRESTConfig() + if err != nil { + return nil, err + } + + calicoClient, err := calico.NewForConfig(restConfig) + if err != nil { + return nil, err + } + + // consider using wait.PollUntilContextTimeout + deleteGNP := func(e error) error { + return errors.Join(e, calicoClient.ProjectcalicoV3().GlobalNetworkPolicies().Delete(context.TODO(), armadaGNP.Name, v1.DeleteOptions{})) + } + + appendNets := func(ruleType string, rules *[]v3.Rule) { + for i, rule := range *rules { + for _, n := range nets { + if !slices.Contains(rule.Destination.Nets, n) { + log.Info(fmt.Sprintf("appending net %s to the %s rule policy type", n, ruleType)) + (*rules)[i].Destination.Nets = append((*rules)[i].Destination.Nets, n) + } + } + } + } + + addNetsToRule := func(ruleType string) { + switch ruleType { + case "Egress": + appendNets(ruleType, &armadaGNP.Spec.Egress) + case "Ingress": + appendNets(ruleType, &armadaGNP.Spec.Ingress) + } + } + + for _, policyType := range armadaGNP.Spec.Types { + addNetsToRule(string(policyType)) + } + + oldGNP, err := calicoClient.ProjectcalicoV3().GlobalNetworkPolicies().Get(context.TODO(), armadaGNP.Name, v1.GetOptions{}) + if err == nil && oldGNP != nil { + log.Info("armada gnp already exists, deleting") + err = deleteGNP(nil) + if err != nil { + return nil, err + } + } + + _, err = calicoClient.ProjectcalicoV3().GlobalNetworkPolicies().Create(context.TODO(), armadaGNP, v1.CreateOptions{}) + if err != nil { + return nil, err + } + log.Info("armada gnp has been created, waiting 5 second for the rules to apply") + + time.Sleep(5 * time.Second) + resp, err := r.httpClient.Do(req) + if err != nil { + return nil, deleteGNP(err) + } + + return resp, deleteGNP(nil) +} + func (r *ArmadaChartReconciler) buildRESTClientGetter(namespace string, l logr.Logger) (genericclioptions.RESTClientGetter, error) { opts := []kube.Option{ kube.WithNamespace(namespace),