From f026321abb6eb66380c4f6f06bdea35f14cb94f8 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Wed, 15 Apr 2020 23:21:27 -0400 Subject: [PATCH] Added base for KOPF operator, switching Memcache Change-Id: I6fdd8c69c35480f3844787eb4d5275174be027ee --- .gitignore | 2 + api/v1alpha1/memcached_types.go | 57 ---- api/v1alpha1/zz_generated.deepcopy.go | 103 ------- chart/templates/deployment.yaml | 17 +- controllers/memcached_controller.go | 279 ------------------ images/openstack-operator/Dockerfile | 17 +- main.go | 13 - openstack_operator/__init__.py | 0 openstack_operator/memcached.py | 72 +++++ openstack_operator/objects.py | 68 +++++ .../templates/memcached/deployment.yml.j2 | 89 ++++++ .../templates/memcached/mcrouter.yml.j2 | 33 +++ .../templates/memcached/podmonitor.yml.j2 | 29 ++ .../templates/memcached/prometheusrule.yml.j2 | 39 +++ openstack_operator/utils.py | 126 ++++++++ requirements.txt | 2 + setup.cfg | 6 + setup.py | 19 ++ test-requirements.txt | 2 + tox.ini | 21 +- zuul.d/misc-jobs.yaml | 10 +- 21 files changed, 540 insertions(+), 464 deletions(-) delete mode 100755 api/v1alpha1/memcached_types.go delete mode 100755 controllers/memcached_controller.go create mode 100644 openstack_operator/__init__.py create mode 100644 openstack_operator/memcached.py create mode 100644 openstack_operator/objects.py create mode 100644 openstack_operator/templates/memcached/deployment.yml.j2 create mode 100644 openstack_operator/templates/memcached/mcrouter.yml.j2 create mode 100644 openstack_operator/templates/memcached/podmonitor.yml.j2 create mode 100644 openstack_operator/templates/memcached/prometheusrule.yml.j2 create mode 100644 openstack_operator/utils.py create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt diff --git a/.gitignore b/.gitignore index c4fce659..7f9fe117 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ bin doc/build .tox +__pycache__ +*.egg* diff --git a/api/v1alpha1/memcached_types.go b/api/v1alpha1/memcached_types.go deleted file mode 100755 index 3e7212d6..00000000 --- a/api/v1alpha1/memcached_types.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020 VEXXHOST, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package v1alpha1 - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// MemcachedSpec defines the desired state of Memcached -type MemcachedSpec struct { - // +kubebuilder:validation:Required - // +kubebuilder:validation:Default=64 - Megabytes int `json:"megabytes"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Tolerations []v1.Toleration `json:"tolerations,omitempty"` -} - -// MemcachedStatus defines the observed state of Memcached -type MemcachedStatus struct { -} - -// +kubebuilder:object:root=true - -// Memcached is the Schema for the memcacheds API -type Memcached struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec MemcachedSpec `json:"spec,omitempty"` - Status MemcachedStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// MemcachedList contains a list of Memcached -type MemcachedList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Memcached `json:"items"` -} - -func init() { - SchemeBuilder.Register(&Memcached{}, &MemcachedList{}) -} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 281d1dd2..df5b3427 100755 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -153,109 +153,6 @@ func (in *McrouterStatus) DeepCopy() *McrouterStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Memcached) DeepCopyInto(out *Memcached) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Memcached. -func (in *Memcached) DeepCopy() *Memcached { - if in == nil { - return nil - } - out := new(Memcached) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Memcached) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemcachedList) DeepCopyInto(out *MemcachedList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Memcached, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemcachedList. -func (in *MemcachedList) DeepCopy() *MemcachedList { - if in == nil { - return nil - } - out := new(MemcachedList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MemcachedList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemcachedSpec) DeepCopyInto(out *MemcachedSpec) { - *out = *in - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemcachedSpec. -func (in *MemcachedSpec) DeepCopy() *MemcachedSpec { - if in == nil { - return nil - } - out := new(MemcachedSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemcachedStatus) DeepCopyInto(out *MemcachedStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemcachedStatus. -func (in *MemcachedStatus) DeepCopy() *MemcachedStatus { - if in == nil { - return nil - } - out := new(MemcachedStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Rabbitmq) DeepCopyInto(out *Rabbitmq) { *out = *in diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index e8da1184..4eb80c5d 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -20,6 +20,21 @@ spec: containers: - name: operator image: vexxhost/openstack-operator:latest + command: ["/usr/local/bin/kopf"] + args: + - run + - -m + - openstack_operator.memcached + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + - name: go-operator + image: vexxhost/openstack-operator:latest + command: ["/manager"] args: - --enable-leader-election resources: @@ -36,4 +51,4 @@ spec: {{- with .Values.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/controllers/memcached_controller.go b/controllers/memcached_controller.go deleted file mode 100755 index 0d0f96ab..00000000 --- a/controllers/memcached_controller.go +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2020 VEXXHOST, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controllers - -import ( - "context" - "fmt" - "sort" - "strconv" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrastructurev1alpha1 "opendev.org/vexxhost/openstack-operator/api/v1alpha1" - - monitoringv1 "opendev.org/vexxhost/openstack-operator/api/monitoring/v1" - "opendev.org/vexxhost/openstack-operator/builders" - "opendev.org/vexxhost/openstack-operator/utils/baseutils" - "opendev.org/vexxhost/openstack-operator/utils/k8sutils" -) - -// MemcachedReconciler reconciles a Memcached object -type MemcachedReconciler struct { - client.Client - Log logr.Logger - Scheme *runtime.Scheme -} - -// +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=memcacheds,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=infrastructure.vexxhost.cloud,resources=memcacheds/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete - -// Reconcile does the reconcilication of Memcached instances -func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - ctx := context.Background() - log := r.Log.WithValues("memcached", req.NamespacedName) - - var memcached infrastructurev1alpha1.Memcached - if err := r.Get(ctx, req.NamespacedName, &memcached); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } - - // Labels - typeLabels := baseutils.MergeMapsWithoutOverwrite(map[string]string{ - "app.kubernetes.io/name": "memcached", - "app.kubernetes.io/managed-by": "openstack-operator", - }, memcached.Labels) - - labels := map[string]string{ - "app.kubernetes.io/name": "memcached", - "app.kubernetes.io/managed-by": "openstack-operator", - "app.kubernetes.io/instance": req.Name, - } - mcrouterLabels := baseutils.MergeMapsWithoutOverwrite(map[string]string{ - "app.kubernetes.io/name": "memcached", - "app.kubernetes.io/managed-by": "openstack-operator", - "app.kubernetes.io/instance": req.Name, - }, memcached.Labels) - - // Deployment - if res, err := r.ReconcileDeployment(ctx, req, &memcached, log, labels); err != nil || res != (ctrl.Result{}) { - return res, err - } - - // PodMonitor - if res, err := r.ReconcilePodMonitor(ctx, req, &memcached, log, typeLabels); err != nil || res != (ctrl.Result{}) { - return res, err - } - - // Alertrule - if res, err := r.ReconcilePrometheusRule(ctx, req, &memcached, log, typeLabels); err != nil || res != (ctrl.Result{}) { - return res, err - } - - // Mcrouter - if res, err := r.ReconcileMcrouter(ctx, req, &memcached, log, labels, mcrouterLabels); err != nil || res != (ctrl.Result{}) { - return res, err - } - - return ctrl.Result{}, nil -} - -// SetupWithManager initializes the controller with primary manager -func (r *MemcachedReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&infrastructurev1alpha1.Memcached{}). - Owns(&appsv1.Deployment{}). - Owns(&infrastructurev1alpha1.Mcrouter{}). - Owns(&monitoringv1.PodMonitor{}). - Owns(&monitoringv1.PrometheusRule{}). - Complete(r) -} - -// ReconcilePodMonitor reconciles the podMonitor -func (r *MemcachedReconciler) ReconcilePodMonitor(ctx context.Context, req ctrl.Request, memcached *infrastructurev1alpha1.Memcached, log logr.Logger, labels map[string]string) (ctrl.Result, error) { - podMonitor := &monitoringv1.PodMonitor{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "monitoring.coreos.com/v1", - Kind: "PodMonitor", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: req.Namespace, - Name: "memcached-podmonitor", - }, - } - op, err := k8sutils.CreateOrUpdate(ctx, r, podMonitor, func() error { - return builders.PodMonitor(podMonitor, memcached, r.Scheme). - Labels(labels). - Selector(map[string]string{ - "app.kubernetes.io/name": "memcached", - }). - PodMetricsEndpoints( - builders.PodMetricsEndpoint(). - Port("metrics"). - Path("/metrics"). - Interval("15s"), - ).Build() - - }) - if err != nil { - return ctrl.Result{}, err - } - log.WithValues("resource", "podmonitor").WithValues("op", op).Info("Reconciled") - return ctrl.Result{}, nil -} - -// ReconcilePrometheusRule reconciles the prometheusRule -func (r *MemcachedReconciler) ReconcilePrometheusRule(ctx context.Context, req ctrl.Request, memcached *infrastructurev1alpha1.Memcached, log logr.Logger, labels map[string]string) (ctrl.Result, error) { - alertRule := &monitoringv1.PrometheusRule{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: req.Namespace, - Name: "memcached-alertrule", - }, - } - op, err := k8sutils.CreateOrUpdate(ctx, r, alertRule, func() error { - - return builders.PrometheusRule(alertRule, memcached, r.Scheme). - Labels(labels). - RuleGroups(builders.RuleGroup(). - Name("memcached-rule"). - Rules( - builders.Rule(). - Alert("MemcachedDown"). - Priority(3). - Expr("memcached_up == 0"), - - builders.Rule(). - Alert("MemcachedMaxConnections"). - Priority(3). - Expr("memcached_current_connections/memcached_max_connections * 100 > 95"), - builders.Rule(). - Alert("MemcachedMaxConnections"). - Priority(4). - Expr("memcached_current_connections/memcached_max_connections * 100 > 90"), - ). - Interval("1m")). - Build() - }) - if err != nil { - return ctrl.Result{}, err - } - log.WithValues("resource", "memcached-alertrule").WithValues("op", op).Info("Reconciled") - return ctrl.Result{}, nil -} - -// ReconcileMcrouter reconciles the mcrouter -func (r *MemcachedReconciler) ReconcileMcrouter(ctx context.Context, req ctrl.Request, memcached *infrastructurev1alpha1.Memcached, log logr.Logger, labels map[string]string, mcrouterLabels map[string]string) (ctrl.Result, error) { - // Get the memcached pod list - pods := &corev1.PodList{} - if err := r.List(ctx, pods, client.InNamespace(req.Namespace), client.MatchingLabels(labels)); err != nil { - return ctrl.Result{}, err - } - // Generate list of pod IP addresses - servers := []string{} - for _, pod := range pods.Items { - // NOTE(mnaser): It's not possible that there is no pod IP assiged yet - if len(pod.Status.PodIP) == 0 { - continue - } - server := fmt.Sprintf("%s:11211", pod.Status.PodIP) - servers = append(servers, server) - } - // If we don't have any servers, requeue. - if len(servers) == 0 { - return ctrl.Result{Requeue: true}, nil - } - // Make sure that they're sorted so we're idempotent - sort.Strings(servers) - mcrouter := &infrastructurev1alpha1.Mcrouter{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: req.Namespace, - Name: fmt.Sprintf("memcached-%s", req.Name), - }, - } - op, err := k8sutils.CreateOrUpdate(ctx, r, mcrouter, func() error { - return builders.Mcrouter(mcrouter, memcached, r.Scheme). - Labels(mcrouterLabels). - NodeSelector(memcached.Spec.NodeSelector). - Tolerations(memcached.Spec.Tolerations). - Route("PoolRoute|default"). - Pool("default", builders.McrouterPoolSpec().Servers(servers)). - Build() - }) - if err != nil { - return ctrl.Result{}, err - } - log.WithValues("resource", "Mcrouter").WithValues("op", op).Info("Reconciled") - return ctrl.Result{}, nil -} - -// ReconcileDeployment reconciles the deployment -func (r *MemcachedReconciler) ReconcileDeployment(ctx context.Context, req ctrl.Request, memcached *infrastructurev1alpha1.Memcached, log logr.Logger, labels map[string]string) (ctrl.Result, error) { - // Calculate size per shared - size := memcached.Spec.Megabytes / 2 - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: req.Namespace, - Name: fmt.Sprintf("memcached-%s", req.Name), - Labels: labels, - }, - } - op, err := k8sutils.CreateOrUpdate(ctx, r, deployment, func() error { - return builders.Deployment(deployment, memcached, r.Scheme). - Labels(labels). - Replicas(2). - PodTemplateSpec( - builders.PodTemplateSpec(). - Labels(labels). - PodSpec( - builders.PodSpec(). - NodeSelector(memcached.Spec.NodeSelector). - Tolerations(memcached.Spec.Tolerations). - Containers( - builders.Container("memcached", "vexxhost/memcached:latest"). - Args("-m", strconv.Itoa(size)). - Port("memcached", 11211).PortProbe("memcached", 10, 30). - Resources(1000, int64(size), 500, 1.10). - SecurityContext( - builders.SecurityContext(). - RunAsUser(1001), - ), - builders.Container("exporter", "vexxhost/memcached-exporter:latest"). - Port("metrics", 9150).HTTPProbe("metrics", "/metrics", 10, 30). - Resources(500, 128, 500, 2). - SecurityContext( - builders.SecurityContext(). - RunAsUser(1001), - ), - ), - ), - ). - Build() - }) - if err != nil { - return ctrl.Result{}, err - } - log.WithValues("resource", "Deployment").WithValues("op", op).Info("Reconciled") - return ctrl.Result{}, nil -} diff --git a/images/openstack-operator/Dockerfile b/images/openstack-operator/Dockerfile index 4df9ef0f..4eda2e13 100644 --- a/images/openstack-operator/Dockerfile +++ b/images/openstack-operator/Dockerfile @@ -38,11 +38,14 @@ COPY version/ version/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -ldflags "-X version.Revision=${REV}" -a -o manager main.go -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot -WORKDIR / -COPY --from=builder /workspace/manager . -USER nonroot:nonroot +# NOTE(mnaser): Rename python-builder to builder +FROM docker.io/opendevorg/python-builder as python-builder +COPY . /tmp/src +RUN assemble -ENTRYPOINT ["/manager"] +FROM docker.io/opendevorg/python-base +COPY --from=python-builder /output/ /output +RUN /output/install-from-bindep + +# TODO(mnaser): Drop this +COPY --from=builder /workspace/manager /manager diff --git a/main.go b/main.go index c2c2cfbf..74afc085 100755 --- a/main.go +++ b/main.go @@ -75,7 +75,6 @@ func main() { // Setup controllers with manager setupMcrouterReconciler(mgr) - setupMemcachedReconciler(mgr) setupRabbitmqReconciler(mgr) setupZoneReconciler(mgr, designateClientBuilder) setupDesignateReconciler(mgr, designateClientBuilder) @@ -126,18 +125,6 @@ func setupMcrouterReconciler(mgr ctrl.Manager) { } } -// setupMemcachedReconciler setups the Memcached controller with manager -func setupMemcachedReconciler(mgr ctrl.Manager) { - if err := (&controllers.MemcachedReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Memcached"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Memcached") - os.Exit(1) - } -} - // setupRabbitmqReconciler setups the Rabbitmq controller with manager func setupRabbitmqReconciler(mgr ctrl.Manager) { if err := (&controllers.RabbitmqReconciler{ diff --git a/openstack_operator/__init__.py b/openstack_operator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openstack_operator/memcached.py b/openstack_operator/memcached.py new file mode 100644 index 00000000..2d7e4960 --- /dev/null +++ b/openstack_operator/memcached.py @@ -0,0 +1,72 @@ +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Memcached Operator + +This module maintains the operator for Memcached, it takes care of creating +the appropriate deployments, Mcrouter, pod monitors and Prometheus rules. +""" + +import kopf + +from openstack_operator import utils + + +@kopf.on.resume('infrastructure.vexxhost.cloud', 'v1alpha1', 'memcacheds') +@kopf.on.create('infrastructure.vexxhost.cloud', 'v1alpha1', 'memcacheds') +def create_or_resume(name, spec, **_): + """Create and re-sync any Memcached instances + + This function is called when a new resource is created but also when we + start the service up for the first time. + """ + + utils.create_or_update('memcached/deployment.yml.j2', + name=name, spec=spec) + utils.create_or_update('memcached/podmonitor.yml.j2', + name=name, spec=spec) + utils.create_or_update('memcached/prometheusrule.yml.j2', + name=name, spec=spec) + + +@kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'memcacheds') +def update(name, spec, **_): + """Update a Memcached + + This function updates the deployment for Memcached if there are any + changes that happen within it. + """ + + utils.create_or_update('memcached/deployment.yml.j2', + name=name, spec=spec) + + +@kopf.on.event('apps', 'v1', 'deployments', labels={ + 'app.kubernetes.io/managed-by': 'openstack-operator', + 'app.kubernetes.io/name': 'memcached', +}) +def deployment_event(namespace, meta, spec, **_): + """Create and re-sync Mcrouter instances + + This function takes care of watching for the readyReplicas on the + Deployments for Memcached to both update and synchronize the Mcrouter. + """ + + name = meta['labels']['app.kubernetes.io/instance'] + selector = spec['selector']['matchLabels'] + servers = utils.get_ready_pod_ips(namespace, selector) + + utils.create_or_update('memcached/mcrouter.yml.j2', + name=name, servers=servers, + spec=spec['template']['spec']) diff --git a/openstack_operator/objects.py b/openstack_operator/objects.py new file mode 100644 index 00000000..72b99029 --- /dev/null +++ b/openstack_operator/objects.py @@ -0,0 +1,68 @@ +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Kubernetes Objects + +This module maintains a list of all of the Kubernetes objects that are used +by the operator. It also includes a few of the custom ones that we use which +are not part of ``pykube``. + +It also inclues a ``dict`` with mappings which allows doing reverse-lookups +from combinations of apiVersion and kind to the exact model. +""" + +from pykube.objects import Deployment +from pykube.objects import NamespacedAPIObject +from pykube.objects import Pod + + +class Mcrouter(NamespacedAPIObject): + """Mcrouter Kubernetes object""" + + version = "infrastructure.vexxhost.cloud/v1alpha1" + endpoint = "mcrouters" + kind = "Mcrouter" + + +class PodMonitor(NamespacedAPIObject): + """PodMonitor Kubernetes object""" + + version = "monitoring.coreos.com/v1" + endpoint = "podmonitors" + kind = "PodMonitor" + + +class PrometheusRule(NamespacedAPIObject): + """PrometheusRule Kubernetes object""" + + version = "monitoring.coreos.com/v1" + endpoint = "prometheusrules" + kind = "PrometheusRule" + + +MAPPING = { + "v1": { + "Pod": Pod, + }, + "apps/v1": { + "Deployment": Deployment, + }, + "infrastructure.vexxhost.cloud/v1alpha1": { + "Mcrouter": Mcrouter, + }, + "monitoring.coreos.com/v1": { + "PodMonitor": PodMonitor, + "PrometheusRule": PrometheusRule, + } +} diff --git a/openstack_operator/templates/memcached/deployment.yml.j2 b/openstack_operator/templates/memcached/deployment.yml.j2 new file mode 100644 index 00000000..3a7da83b --- /dev/null +++ b/openstack_operator/templates/memcached/deployment.yml.j2 @@ -0,0 +1,89 @@ +--- +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached-{{ name }} + labels: + {{ labels("memcached", name) | indent(4) }} +spec: + replicas: 2 + selector: + matchLabels: + {{ labels("memcached", name) | indent(6) }} + template: + metadata: + labels: + {{ labels("memcached", name) | indent(8) }} + spec: + containers: + - name: memcached + image: vexxhost/memcached:latest + args: ["-m", "{{ (spec.megabytes / 2) | int }}"] + imagePullPolicy: Always + ports: + - name: memcached + containerPort: 11211 + livenessProbe: + tcpSocket: + port: memcached + readinessProbe: + tcpSocket: + port: memcached + resources: + limits: + cpu: 50m + ephemeral-storage: 50M + memory: {{ (spec.megabytes / 2) | int + 64 }}M + requests: + cpu: 10m + ephemeral-storage: 50M + memory: {{ (spec.megabytes / 2) | int }}M + securityContext: + runAsUser: 1001 + - name: exporter + image: vexxhost/memcached_exporter:latest + imagePullPolicy: Always + ports: + - name: metrics + containerPort: 9150 + livenessProbe: + httpGet: + path: /metrics + port: metrics + readinessProbe: + httpGet: + path: /metrics + port: metrics + resources: + limits: + cpu: 100m + ephemeral-storage: 10M + memory: 64Mi + requests: + cpu: 50m + ephemeral-storage: 10M + memory: 32Mi + securityContext: + runAsUser: 1001 +{% if 'nodeSelector' in spec %} + nodeSelector: + {{ spec.nodeSelector | to_yaml | indent(8) }} +{% endif %} +{% if 'tolerations' in spec %} + tolerations: + {{ spec.tolerations | to_yaml | indent(8) }} +{% endif %} diff --git a/openstack_operator/templates/memcached/mcrouter.yml.j2 b/openstack_operator/templates/memcached/mcrouter.yml.j2 new file mode 100644 index 00000000..a8a08f4d --- /dev/null +++ b/openstack_operator/templates/memcached/mcrouter.yml.j2 @@ -0,0 +1,33 @@ +--- +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: infrastructure.vexxhost.cloud/v1alpha1 +kind: Mcrouter +metadata: + name: memcached-{{ name }} +spec: + pools: + default: + servers: + {{ servers | to_yaml | indent(8) }} + route: PoolRoute|default +{% if 'nodeSelector' in spec %} + nodeSelector: + {{ spec.nodeSelector | to_yaml | indent(4) }} +{% endif %} +{% if 'tolerations' in spec %} + tolerations: + {{ spec.tolerations | to_yaml | indent(4) }} +{% endif %} diff --git a/openstack_operator/templates/memcached/podmonitor.yml.j2 b/openstack_operator/templates/memcached/podmonitor.yml.j2 new file mode 100644 index 00000000..75cfdfc7 --- /dev/null +++ b/openstack_operator/templates/memcached/podmonitor.yml.j2 @@ -0,0 +1,29 @@ +--- +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: memcached-{{ name }} + labels: + {{ labels("memcached", name) | indent(4) }} +spec: + podMetricsEndpoints: + - interval: 15s + path: /metrics + port: metrics + selector: + matchLabels: + {{ labels("memcached", name) | indent(6) }} diff --git a/openstack_operator/templates/memcached/prometheusrule.yml.j2 b/openstack_operator/templates/memcached/prometheusrule.yml.j2 new file mode 100644 index 00000000..21ed4248 --- /dev/null +++ b/openstack_operator/templates/memcached/prometheusrule.yml.j2 @@ -0,0 +1,39 @@ +--- +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: memcached-{{ name }} + labels: + {{ labels("memcached", name) | indent(4) }} +spec: + groups: + - name: down + rules: + - alert: MemcachedDown + expr: memcached_up == 0 + annotations: + priority: P3 + - name: connection-limits + rules: + - alert: MemcachedMaxConnections + expr: memcached_current_connections/memcached_max_connections * 100 > 95 + annotations: + priority: P3 + - alert: MemcachedMaxConnections + expr: memcached_current_connections/memcached_max_connections * 100 > 90 + annotations: + priority: P4 diff --git a/openstack_operator/utils.py b/openstack_operator/utils.py new file mode 100644 index 00000000..12b89c4f --- /dev/null +++ b/openstack_operator/utils.py @@ -0,0 +1,126 @@ +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities + +The module contains a few useful utilities which we refactor out in order +to be able to use them across all different operators. +""" + +import copy +import operator +import os + +import jinja2 +import kopf +import pykube +import yaml + +from openstack_operator import objects + + +DIR_PATH = os.path.dirname(os.path.realpath(__file__)) + + +def to_yaml(value): + """Return a YAML string from a dictionary.""" + return yaml.safe_dump(value) + + +def labels(app, instance): + """Return standard labels for the operator.""" + return yaml.safe_dump({ + 'app.kubernetes.io/managed-by': 'openstack-operator', + 'app.kubernetes.io/name': app, + 'app.kubernetes.io/instance': instance, + }).strip() + + +ENV = jinja2.Environment( + loader=jinja2.FileSystemLoader("%s/templates" % DIR_PATH) +) +ENV.filters['to_yaml'] = to_yaml +ENV.globals['labels'] = labels + + +def create_or_update(template, **kwargs): + """Create or update a Kubernetes resource. + + This function is called with a template and the args to pass to that + template and it will generate a Kubernetes object, with that + object, it will try and check if it exists. If it does, it will run + an update, if not, it will create the new object. + """ + + resource = generate_object(template, **kwargs) + obj = copy.deepcopy(resource.obj) + + # Try to get the remote record + try: + resource.reload() + resource.obj = obj + resource.update() + except pykube.exceptions.HTTPError as exc: + if exc.code != 404: + raise + resource.create() + + +def generate_yaml(template, **kwargs): + """Generate dictionary from YAML template. + + This takes a Jinja2 template, renders it using all the passed ``kwargs`` + and then runs ``adopt`` as well to prepare it to be committed to the + cluster. + """ + + template = ENV.get_template(template) + yamldoc = template.render(**kwargs) + doc = yaml.safe_load(yamldoc) + kopf.adopt(doc) + + return doc + + +def generate_object(template, **kwargs): + """Generate Kubernetes object + + This function renders a Jinja2 template provided into a Kubernetes object + based on the ``apiVersion`` and ``kind`` from the generated YAML file. + """ + + doc = generate_yaml(template, **kwargs) + + api_version = doc['apiVersion'] + kind = doc['kind'] + resource = objects.MAPPING[api_version][kind] + + api = pykube.HTTPClient(pykube.KubeConfig.from_env()) + return resource(api, doc) + + +def get_ready_pod_ips(namespace, selector): + """Get list of all ready pod IPs. + + This is a helper function which given a selector, will retrieve all pods + and return the IP addresses of the ones that are ready. + """ + + api = pykube.HTTPClient(pykube.KubeConfig.from_env()) + pods = objects.Pod.objects(api).filter(namespace=namespace, + selector=selector) + ready_pods = filter(operator.attrgetter("ready"), pods) + servers = sorted([p.obj["status"]["podIP"] for p in ready_pods]) + + return servers diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..48fc86de --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +kopf +Jinja2 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..0b449645 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[metadata] +name = openstack-operator + +[files] +packages = + openstack_operator diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..fd3c0078 --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..897e59f5 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +flake8 +pylint diff --git a/tox.ini b/tox.ini index cb8000a2..a2bb08ae 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,8 @@ [tox] -minversion = 2.0 -skipsdist = True +minversion = 3.1.1 + +[testenv] +usedevelop = True [testenv:update-zuul-jobs] deps = @@ -8,6 +10,21 @@ deps = commands = {toxinidir}/hack/update-zuul-jobs.py +[testenv:linters] +basepython = python3.7 +deps = + -rtest-requirements.txt + -rrequirements.txt +commands = + pylint openstack_operator + flake8 openstack_operator + +[testenv:kopf] +deps = + -rrequirements.txt +commands = + kopf run {posargs} + [testenv:docs] deps = -r{toxinidir}/doc/requirements.txt diff --git a/zuul.d/misc-jobs.yaml b/zuul.d/misc-jobs.yaml index cd161085..26927f04 100644 --- a/zuul.d/misc-jobs.yaml +++ b/zuul.d/misc-jobs.yaml @@ -4,16 +4,22 @@ vars: zuul_work_dir: "{{ zuul.project.src_dir }}/chart" +- job: + name: openstack-operator:linters:tox + parent: tox-linters + vars: + python_version: 3.7 + - project: templates: - publish-opendev-tox-docs check: jobs: - - golangci-lint - golang-go-test - openstack-operator:linters:chart + - openstack-operator:linters:tox gate: jobs: - - golangci-lint - golang-go-test - openstack-operator:linters:chart + - openstack-operator:linters:tox