WIP

config/samples/airship_v1beta1_sipcluster.yaml

@@ -1,21 +1,23 @@
 apiVersion: airship.airshipit.org/v1
 kind: SIPCluster
 metadata:
-  name: sipcluster-sample
+  name: sipcluster-test1
-  namespace: clusterA
+  namespace: sip-cluster-system
 spec:
+    config:
+        cluster-name: cname
     nodes:
       worker:
           vm-flavor: 'airshipit.org/vino-flavor=worker'
           scheduling-constraints: ['per-node'] # Support dont'care option.
           count:
-              active: 3 #driven by capi node number
+              active: 1 #driven by capi node number
               standby: 1 #slew for upgrades etc
       master:
           vm-flavor: 'airshipit.org/vino-flavor=master'
           scheduling-constraints: ['per-node','per-rack']
           count:
-              active: 3
+              active: 1
               standby: 1
     infra:
         loadbalancer:

config/samples/bmh/baremetalhosts.metal3.io.yaml

@@ -0,0 +1,565 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
+  name: baremetalhosts.metal3.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.operationalStatus
+    description: Operational status
+    name: Status
+    type: string
+  - JSONPath: .status.provisioning.state
+    description: Provisioning status
+    name: Provisioning Status
+    type: string
+  - JSONPath: .spec.consumerRef.name
+    description: Consumer using this host
+    name: Consumer
+    type: string
+  - JSONPath: .spec.bmc.address
+    description: Address of management controller
+    name: BMC
+    type: string
+  - JSONPath: .status.hardwareProfile
+    description: The type of hardware detected
+    name: Hardware Profile
+    type: string
+  - JSONPath: .spec.online
+    description: Whether the host is online or not
+    name: Online
+    type: string
+  - JSONPath: .status.errorMessage
+    description: Most recent error
+    name: Error
+    type: string
+  group: metal3.io
+  names:
+    kind: BareMetalHost
+    listKind: BareMetalHostList
+    plural: baremetalhosts
+    shortNames:
+    - bmh
+    - bmhost
+    singular: baremetalhost
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: BareMetalHost is the Schema for the baremetalhosts API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: BareMetalHostSpec defines the desired state of BareMetalHost
+          properties:
+            bmc:
+              description: How do we connect to the BMC?
+              properties:
+                address:
+                  description: Address holds the URL for accessing the controller
+                    on the network.
+                  type: string
+                credentialsName:
+                  description: The name of the secret containing the BMC credentials
+                    (requires keys "username" and "password").
+                  type: string
+                disableCertificateVerification:
+                  description: DisableCertificateVerification disables verification
+                    of server certificates when using HTTPS to connect to the BMC.
+                    This is required when the server certificate is self-signed, but
+                    is insecure because it allows a man-in-the-middle to intercept
+                    the connection.
+                  type: boolean
+              required:
+              - address
+              - credentialsName
+              type: object
+            bootMACAddress:
+              description: Which MAC address will PXE boot? This is optional for some
+                types, but required for libvirt VMs driven by vbmc.
+              pattern: '[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}'
+              type: string
+            consumerRef:
+              description: ConsumerRef can be used to store information about something
+                that is using a host. When it is not empty, the host is considered
+                "in use".
+              properties:
+                apiVersion:
+                  description: API version of the referent.
+                  type: string
+                fieldPath:
+                  description: 'If referring to a piece of an object instead of an
+                    entire object, this string should contain a valid JSON/Go field
+                    access statement, such as desiredState.manifest.containers[2].
+                    For example, if the object reference is to a container within
+                    a pod, this would take on a value like: "spec.containers{name}"
+                    (where "name" refers to the name of the container that triggered
+                    the event) or if no container name is specified "spec.containers[2]"
+                    (container with index 2 in this pod). This syntax is chosen only
+                    to have some well-defined way of referencing a part of an object.
+                    TODO: this design is not final and this field is subject to change
+                    in the future.'
+                  type: string
+                kind:
+                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                  type: string
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                  type: string
+                namespace:
+                  description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                  type: string
+                resourceVersion:
+                  description: 'Specific resourceVersion to which this reference is
+                    made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                  type: string
+                uid:
+                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                  type: string
+              type: object
+            description:
+              description: Description is a human-entered text used to help identify
+                the host
+              type: string
+            externallyProvisioned:
+              description: ExternallyProvisioned means something else is managing
+                the image running on the host and the operator should only manage
+                the power status and hardware inventory inspection. If the Image field
+                is filled in, this field is ignored.
+              type: boolean
+            hardwareProfile:
+              description: What is the name of the hardware profile for this host?
+                It should only be necessary to set this when inspection cannot automatically
+                determine the profile.
+              type: string
+            image:
+              description: Image holds the details of the image to be provisioned.
+              properties:
+                checksum:
+                  description: Checksum is the checksum for the image.
+                  type: string
+                url:
+                  description: URL is a location of an image to deploy.
+                  type: string
+              required:
+              - checksum
+              - url
+              type: object
+            networkData:
+              description: NetworkData holds the reference to the Secret containing
+                content of network_data.json which is passed to Config Drive
+              properties:
+                name:
+                  description: Name is unique within a namespace to reference a secret
+                    resource.
+                  type: string
+                namespace:
+                  description: Namespace defines the space within which the secret
+                    name must be unique.
+                  type: string
+              type: object
+            online:
+              description: Should the server be online?
+              type: boolean
+            taints:
+              description: Taints is the full, authoritative list of taints to apply
+                to the corresponding Machine. This list will overwrite any modifications
+                made to the Machine on an ongoing basis.
+              items:
+                description: The node this Taint is attached to has the "effect" on
+                  any pod that does not tolerate the Taint.
+                properties:
+                  effect:
+                    description: Required. The effect of the taint on pods that do
+                      not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule
+                      and NoExecute.
+                    type: string
+                  key:
+                    description: Required. The taint key to be applied to a node.
+                    type: string
+                  timeAdded:
+                    description: TimeAdded represents the time at which the taint
+                      was added. It is only written for NoExecute taints.
+                    format: date-time
+                    type: string
+                  value:
+                    description: Required. The taint value corresponding to the taint
+                      key.
+                    type: string
+                required:
+                - effect
+                - key
+                type: object
+              type: array
+            userData:
+              description: UserData holds the reference to the Secret containing the
+                user data to be passed to the host before it boots.
+              properties:
+                name:
+                  description: Name is unique within a namespace to reference a secret
+                    resource.
+                  type: string
+                namespace:
+                  description: Namespace defines the space within which the secret
+                    name must be unique.
+                  type: string
+              type: object
+          required:
+          - online
+          type: object
+        status:
+          description: BareMetalHostStatus defines the observed state of BareMetalHost
+          properties:
+            errorMessage:
+              description: the last error message reported by the provisioning subsystem
+              type: string
+            errorType:
+              description: ErrorType indicates the type of failure encountered when
+                the OperationalStatus is OperationalStatusError
+              enum:
+              - registration error
+              - inspection error
+              - provisioning error
+              - power management error
+              type: string
+            goodCredentials:
+              description: the last credentials we were able to validate as working
+              properties:
+                credentials:
+                  description: SecretReference represents a Secret Reference. It has
+                    enough information to retrieve secret in any namespace
+                  properties:
+                    name:
+                      description: Name is unique within a namespace to reference
+                        a secret resource.
+                      type: string
+                    namespace:
+                      description: Namespace defines the space within which the secret
+                        name must be unique.
+                      type: string
+                  type: object
+                credentialsVersion:
+                  type: string
+              type: object
+            hardware:
+              description: The hardware discovered to exist on the host.
+              properties:
+                cpu:
+                  description: CPU describes one processor on the host.
+                  properties:
+                    arch:
+                      type: string
+                    clockMegahertz:
+                      description: ClockSpeed is a clock speed in MHz
+                    count:
+                      type: integer
+                    flags:
+                      items:
+                        type: string
+                      type: array
+                    model:
+                      type: string
+                  required:
+                  - arch
+                  - clockMegahertz
+                  - count
+                  - flags
+                  - model
+                  type: object
+                firmware:
+                  description: Firmware describes the firmware on the host.
+                  properties:
+                    bios:
+                      description: The BIOS for this firmware
+                      properties:
+                        date:
+                          description: The release/build date for this BIOS
+                          type: string
+                        vendor:
+                          description: The vendor name for this BIOS
+                          type: string
+                        version:
+                          description: The version of the BIOS
+                          type: string
+                      required:
+                      - date
+                      - vendor
+                      - version
+                      type: object
+                  required:
+                  - bios
+                  type: object
+                hostname:
+                  type: string
+                nics:
+                  items:
+                    description: NIC describes one network interface on the host.
+                    properties:
+                      ip:
+                        description: The IP address of the device
+                        type: string
+                      mac:
+                        description: The device MAC addr
+                        pattern: '[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}'
+                        type: string
+                      model:
+                        description: The name of the model, e.g. "virt-io"
+                        type: string
+                      name:
+                        description: The name of the NIC, e.g. "nic-1"
+                        type: string
+                      pxe:
+                        description: Whether the NIC is PXE Bootable
+                        type: boolean
+                      speedGbps:
+                        description: The speed of the device
+                        type: integer
+                      vlanId:
+                        description: The untagged VLAN ID
+                        format: int32
+                        type: integer
+                      vlans:
+                        description: The VLANs available
+                        items:
+                          description: VLAN represents the name and ID of a VLAN
+                          properties:
+                            id:
+                              description: VLANID is a 12-bit 802.1Q VLAN identifier
+                              format: int32
+                              type: integer
+                            name:
+                              type: string
+                          required:
+                          - id
+                          type: object
+                        type: array
+                    required:
+                    - ip
+                    - mac
+                    - model
+                    - name
+                    - pxe
+                    - speedGbps
+                    - vlanId
+                    type: object
+                  type: array
+                ramMebibytes:
+                  type: integer
+                storage:
+                  items:
+                    description: Storage describes one storage device (disk, SSD,
+                      etc.) on the host.
+                    properties:
+                      hctl:
+                        description: The SCSI location of the device
+                        type: string
+                      model:
+                        description: Hardware model
+                        type: string
+                      name:
+                        description: A name for the disk, e.g. "disk 1 (boot)"
+                        type: string
+                      rotational:
+                        description: Whether this disk represents rotational storage
+                        type: boolean
+                      serialNumber:
+                        description: The serial number of the device
+                        type: string
+                      sizeBytes:
+                        description: The size of the disk in Bytes
+                        format: int64
+                        type: integer
+                      vendor:
+                        description: The name of the vendor of the device
+                        type: string
+                      wwn:
+                        description: The WWN of the device
+                        type: string
+                      wwnVendorExtension:
+                        description: The WWN Vendor extension of the device
+                        type: string
+                      wwnWithExtension:
+                        description: The WWN with the extension
+                        type: string
+                    required:
+                    - name
+                    - rotational
+                    - serialNumber
+                    - sizeBytes
+                    type: object
+                  type: array
+                systemVendor:
+                  description: HardwareSystemVendor stores details about the whole
+                    hardware system.
+                  properties:
+                    manufacturer:
+                      type: string
+                    productName:
+                      type: string
+                    serialNumber:
+                      type: string
+                  required:
+                  - manufacturer
+                  - productName
+                  - serialNumber
+                  type: object
+              required:
+              - cpu
+              - firmware
+              - hostname
+              - nics
+              - ramMebibytes
+              - storage
+              - systemVendor
+              type: object
+            hardwareProfile:
+              description: The name of the profile matching the hardware details.
+              type: string
+            lastUpdated:
+              description: LastUpdated identifies when this status was last observed.
+              format: date-time
+              type: string
+            operationHistory:
+              description: OperationHistory holds information about operations performed
+                on this host.
+              properties:
+                deprovision:
+                  description: OperationMetric contains metadata about an operation
+                    (inspection, provisioning, etc.) used for tracking metrics.
+                  properties:
+                    end:
+                      format: date-time
+                      nullable: true
+                      type: string
+                    start:
+                      format: date-time
+                      nullable: true
+                      type: string
+                  type: object
+                inspect:
+                  description: OperationMetric contains metadata about an operation
+                    (inspection, provisioning, etc.) used for tracking metrics.
+                  properties:
+                    end:
+                      format: date-time
+                      nullable: true
+                      type: string
+                    start:
+                      format: date-time
+                      nullable: true
+                      type: string
+                  type: object
+                provision:
+                  description: OperationMetric contains metadata about an operation
+                    (inspection, provisioning, etc.) used for tracking metrics.
+                  properties:
+                    end:
+                      format: date-time
+                      nullable: true
+                      type: string
+                    start:
+                      format: date-time
+                      nullable: true
+                      type: string
+                  type: object
+                register:
+                  description: OperationMetric contains metadata about an operation
+                    (inspection, provisioning, etc.) used for tracking metrics.
+                  properties:
+                    end:
+                      format: date-time
+                      nullable: true
+                      type: string
+                    start:
+                      format: date-time
+                      nullable: true
+                      type: string
+                  type: object
+              type: object
+            operationalStatus:
+              description: OperationalStatus holds the status of the host
+              enum:
+              - ""
+              - OK
+              - discovered
+              - error
+              type: string
+            poweredOn:
+              description: indicator for whether or not the host is powered on
+              type: boolean
+            provisioning:
+              description: Information tracked by the provisioner.
+              properties:
+                ID:
+                  description: The machine's UUID from the underlying provisioning
+                    tool
+                  type: string
+                image:
+                  description: Image holds the details of the last image successfully
+                    provisioned to the host.
+                  properties:
+                    checksum:
+                      description: Checksum is the checksum for the image.
+                      type: string
+                    url:
+                      description: URL is a location of an image to deploy.
+                      type: string
+                  required:
+                  - checksum
+                  - url
+                  type: object
+                state:
+                  description: An indiciator for what the provisioner is doing with
+                    the host.
+                  type: string
+              required:
+              - ID
+              - state
+              type: object
+            triedCredentials:
+              description: the last credentials we sent to the provisioning backend
+              properties:
+                credentials:
+                  description: SecretReference represents a Secret Reference. It has
+                    enough information to retrieve secret in any namespace
+                  properties:
+                    name:
+                      description: Name is unique within a namespace to reference
+                        a secret resource.
+                      type: string
+                    namespace:
+                      description: Namespace defines the space within which the secret
+                        name must be unique.
+                      type: string
+                  type: object
+                credentialsVersion:
+                  type: string
+              type: object
+          required:
+          - errorMessage
+          - hardwareProfile
+          - operationHistory
+          - operationalStatus
+          - poweredOn
+          - provisioning
+          type: object
+      type: object
+  version: v1alpha1
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+

config/samples/bmh/bmhLabels

@@ -0,0 +1,6 @@
+k label baremetalhosts -n metal3 --overwrite rdm9r006o001 sip.airshipit.org/rack=r006 sip.airshipit.org/server=rdm9r006o001 airshipit.org/vino-flavor=master sip.airshipit.org/sip-scheduled:false
+k label baremetalhosts -n metal3 --overwrite rdm9r006o002 sip.airshipit.org/rack=r006 sip.airshipit.org/server=rdm9r006o002 airshipit.org/vino-flavor=master sip.airshipit.org/sip-scheduled:false
+k label baremetalhosts -n metal3 --overwrite rdm9r007o001 sip.airshipit.org/rack=r007 sip.airshipit.org/server=rdm9r007o001 airshipit.org/vino-flavor=master sip.airshipit.org/sip-scheduled:false
+k label baremetalhosts -n metal3 --overwrite rdm9r007o002 sip.airshipit.org/rack=r007 sip.airshipit.org/server=rdm9r007o002 airshipit.org/vino-flavor=worker sip.airshipit.org/sip-scheduled:false
+k label baremetalhosts -n metal3 --overwrite rdm9r008c002 sip.airshipit.org/rack=r008 sip.airshipit.org/server=rdm9r008c002 airshipit.org/vino-flavor=worker sip.airshipit.org/sip-scheduled:false
+k label baremetalhosts -n metal3 --overwrite rdm9r009c002 sip.airshipit.org/rack=r009 sip.airshipit.org/server=rdm9r009c002 airshipit.org/vino-flavor=worker sip.airshipit.org/sip-scheduled:false

go.mod

@@ -3,6 +3,7 @@ module sipcluster
 go 1.13
 
 require (
+	github.com/PaesslerAG/jsonpath v0.1.1
 	github.com/fluxcd/helm-controller/api v0.1.3
 	github.com/go-logr/logr v0.2.1
 	github.com/metal3-io/baremetal-operator v0.0.0-20201014161845-a6d4f1fc3228

go.sum

@@ -28,6 +28,11 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
+github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
+github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
+github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=

main.go

@@ -29,6 +29,8 @@ import (
 	airshipv1 "sipcluster/pkg/api/v1"
 	"sipcluster/pkg/controllers"
 
+	corev1 "k8s.io/api/core/v1"
+
 	// +kubebuilder:scaffold:imports
 
 	metal3 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
@@ -47,6 +49,10 @@ func init() {
 
 	// Add Metal3 CRD
 	_ = metal3.AddToScheme(scheme)
+
+	// Add Kubernetes Coree??
+	_ = corev1.AddToScheme(scheme)
+
 }
 
 func main() {

pkg/api/v1/sipcluster_types.go

@@ -28,6 +28,7 @@ type SIPClusterSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
+	Config *SipConfig `json:"config,omitempty"`
 	// Nodes are the list of Nodes objects workers, or master that definee eexpectations
 	// of the Tenant cluster
 	// VmRole is either Control or Workers
@@ -42,6 +43,12 @@ type SIPClusterSpec struct {
 	InfraServices map[InfraService]InfraConfig `json:"infra"`
 }
 
+// SIPClusterSpec defines the desired state of SIPCluster
+type SipConfig struct {
+	// Cluster NAme to be used for labeling vBMH
+	ClusterName string `json:"cluster-name,omitempty"`
+}
+
 // VmRoles defines the states the provisioner will report
 // the tenant has having.
 type InfraService string

pkg/api/v1/zz_generated.deepcopy.go

@@ -153,6 +153,11 @@ func (in *SIPClusterList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SIPClusterSpec) DeepCopyInto(out *SIPClusterSpec) {
 	*out = *in
+	if in.Config != nil {
+		in, out := &in.Config, &out.Config
+		*out = new(SipConfig)
+		**out = **in
+	}
 	if in.Nodes != nil {
 		in, out := &in.Nodes, &out.Nodes
 		*out = make(map[VmRoles]NodeSet, len(*in))
@@ -194,6 +199,21 @@ func (in *SIPClusterStatus) DeepCopy() *SIPClusterStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SipConfig) DeepCopyInto(out *SipConfig) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SipConfig.
+func (in *SipConfig) DeepCopy() *SipConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(SipConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VmCount) DeepCopyInto(out *VmCount) {
 	*out = *in

pkg/controllers/sipcluster_controller.go

@@ -66,6 +66,13 @@ func (r *SIPClusterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error)
 
 	err = r.deployInfra(sip, machines)
 	if err != nil {
+		log.Error(err, "unable to deploy infrastructure services")
+		return ctrl.Result{}, err
+	}
+
+	err = r.finish(sip, machines)
+	if err != nil {
+		log.Error(err, "unable to finalize ..")
 		return ctrl.Result{}, err
 	}
 	return ctrl.Result{}, nil
@@ -109,7 +116,7 @@ func (r *SIPClusterReconciler) gatherVBMH(sip airshipv1.SIPCluster) (error, *air
 	// If Not complete schedule , then throw an error.
 	machines := &airshipvms.MachineList{}
 	fmt.Printf("gatherVBMH.Schedule sip:%v machines:%v\n", sip, machines)
-	err := machines.Schedule(sip.Spec.Nodes, r.Client)
+	err := machines.Schedule(sip, r.Client)
 	if err != nil {
 		return err, machines
 	}
@@ -148,3 +155,17 @@ func (r *SIPClusterReconciler) deployInfra(sip airshipv1.SIPCluster, machines *a
 	}
 	return nil
 }
+
+/*
+finish shoulld  take care of any wrpa up tasks..
+*/
+func (r *SIPClusterReconciler) finish(sip airshipv1.SIPCluster, machines *airshipvms.MachineList) error {
+
+	// Label the vBMH's
+	err := machines.ApplyLabels(sip, r.Client)
+	if err != nil {
+		return err
+	}
+	return nil
+
+}

pkg/services/infrastructureservice.go

@@ -15,11 +15,10 @@
 package services
 
 import (
-	"context"
-	helm "github.com/fluxcd/helm-controller/api/v2beta1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	airshipv1 "sipcluster/pkg/api/v1"
 	airshipvms "sipcluster/pkg/vbmh"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // Infrastructure interface should be implemented by each Tenant Required
@@ -47,11 +46,6 @@ func (s *Service) Deploy(machines *airshipvms.MachineList, c client.Client) erro
 	// Take the data from teh appropriate Machines
 	// Prepare the Config
 
-	// Prepare the HelmReleasecd
-	helmRelease := &helm.HelmRelease{}
-	if err := c.Create(context.TODO(), helmRelease); err != nil {
-		return err
-	}
 	return nil
 }
 

pkg/vbmh/errors.go

@@ -26,8 +26,9 @@ type ErrorHostIpNotFound struct {
 	HostName    string
 	ServiceName airshipv1.InfraService
 	IPInterface string
+	Message     string
 }
 
 func (e ErrorHostIpNotFound) Error() string {
-	return fmt.Sprintf("Unable to identify the vBMH Host %v IP address on interface %v required by Infrastructure Service %v ", e.HostName, e.IPInterface, e.ServiceName)
+	return fmt.Sprintf("Unable to identify the vBMH Host %v IP address on interface %v required by Infrastructure Service %v %s ", e.HostName, e.IPInterface, e.ServiceName, e.Message)
 }

pkg/vbmh/machines.go

@@ -20,17 +20,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	airshipv1 "sipcluster/pkg/api/v1"
 	"strings"
 
+	"github.com/PaesslerAG/jsonpath"
 	metal3 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
-
 	corev1 "k8s.io/api/core/v1"
-
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	airshipv1 "sipcluster/pkg/api/v1"
-	//rbacv1 "k8s.io/api/rbac/v1"
-	//"k8s.io/apimachinery/pkg/api/errors"
 )
 
 // ScheduledState
@@ -64,10 +60,13 @@ const (
 	RackLabel   = BaseAirshipSelector + "/rack"
 	ServerLabel = BaseAirshipSelector + "/server"
 
-	// This should be a configurable thing
+	// Thislabekl is associated to group the colletcion of scheduled vBMH's
-	// But probable not in teh CR.. TBD
+	// Will represent the Tenant Cluster or Service Function  Cluster
-	// TODO
+	SipClusterLabelName = "workload-cluster"
-	VBMH_NAMESPACE = "metal3"
+	SipClusterLabel     = BaseAirshipSelector + "/" + SipClusterLabelName
+
+	SipNodeTypeLabelName = "note-type"
+	SipNodeTypeLabel     = BaseAirshipSelector + "/" + SipNodeTypeLabelName
 )
 
 // MAchine represents an individual BMH CR, and teh appropriate
@@ -96,10 +95,10 @@ type MachineList struct {
 	bmhs []*Machine
 }
 
-func (ml *MachineList) Schedule(nodes map[airshipv1.VmRoles]airshipv1.NodeSet, c client.Client) error {
+func (ml *MachineList) Schedule(sip airshipv1.SIPCluster, c client.Client) error {
 
 	// Initialize teh Target list
-	ml.bmhs = ml.init(nodes)
+	ml.bmhs = ml.init(sip.Spec.Nodes)
 
 	// IDentify vBMH's that meet the appropriate selction criteria
 	bmhList, err := ml.getVBMH(c)
@@ -108,7 +107,7 @@ func (ml *MachineList) Schedule(nodes map[airshipv1.VmRoles]airshipv1.NodeSet, c
 	}
 
 	//  Identify and Select the vBMH I actually will use
-	err = ml.identifyNodes(nodes, bmhList)
+	err = ml.identifyNodes(sip, bmhList, c)
 	if err != nil {
 		return err
 	}
@@ -144,27 +143,6 @@ func (ml *MachineList) getVBMH(c client.Client) (*metal3.BareMetalHostList, erro
 	*/
 	scheduleLabels := map[string]string{SipScheduleLabel: "false"}
 
-	/**
-	DELETE SOON, ..
-	labelSelector := metav1.LabelSelector{MatchLabels: scheduleLabels}
-	bmhSelector, err := metav1.LabelSelectorAsSelector(&labelSelector)
-	if err != nil {
-		return bmhList, err
-	}
-
-	fmt.Printf("Schedule.getVBMH bmhSelector:%v\n", bmhSelector)
-	// TODO Namespace where vBMH needs to be found
-	// Might be in THE SIP CR Peerhaps
-	bmListOptions := &client.ListOptions{
-		LabelSelector: bmhSelector,
-		Limit:         100,
-		Namespace:     VBMH_NAMESPACE,
-	}
-
-	fmt.Printf("Schedule.getVBMH bmList context.Background bmhList:%v\n bmListOptions:%v scheduleLabels:%v\n", bmhList, bmListOptions, scheduleLabels)
-	err = c.List(context.Background(), bmhList, bmListOptions, client.InNamespace(VBMH_NAMESPACE))
-	*/
-
 	err := c.List(context.Background(), bmhList, client.MatchingLabels(scheduleLabels))
 	if err != nil {
 		fmt.Printf("Schedule.getVBMH bmhList err:%v\n", err)
@@ -177,7 +155,7 @@ func (ml *MachineList) getVBMH(c client.Client) (*metal3.BareMetalHostList, erro
 	return bmhList, fmt.Errorf("Unable to identify vBMH available for scheduling. Selecting  %v ", scheduleLabels)
 }
 
-func (ml *MachineList) identifyNodes(nodes map[airshipv1.VmRoles]airshipv1.NodeSet, bmhList *metal3.BareMetalHostList) error {
+func (ml *MachineList) identifyNodes(sip airshipv1.SIPCluster, bmhList *metal3.BareMetalHostList, c client.Client) error {
 	// If using the SIP Sheduled label, we now have a list of vBMH;'s
 	// that are not scheduled
 	// Next I need to apply the constraints
@@ -187,12 +165,12 @@ func (ml *MachineList) identifyNodes(nodes map[airshipv1.VmRoles]airshipv1.NodeS
 	// - Racks  : Dont select two machines in the same rack
 	// - Server : Dont select two machines in the same server
 	fmt.Printf("Schedule.identifyNodes bmList size:%d\n", len(bmhList.Items))
-	for nodeRole, nodeCfg := range nodes {
+	for nodeRole, nodeCfg := range sip.Spec.Nodes {
 		scheduleSetMap, err := ml.initScheduleMaps(nodeCfg.Scheduling)
 		if err != nil {
 			return err
 		}
-		err = ml.scheduleIt(nodeRole, nodeCfg, bmhList, scheduleSetMap)
+		err = ml.scheduleIt(nodeRole, nodeCfg, bmhList, scheduleSetMap, c, sip.Spec.Config)
 		if err != nil {
 			return err
 		}
@@ -228,12 +206,30 @@ func (ml *MachineList) initScheduleMaps(constraints []airshipv1.SchedulingOption
 	return setMap, ErrorConstraintNotFound{}
 }
 
-func (ml *MachineList) scheduleIt(nodeRole airshipv1.VmRoles, nodeCfg airshipv1.NodeSet, bmList *metal3.BareMetalHostList, scheduleSetMap map[airshipv1.SchedulingOptions]*ScheduleSet) error {
+func (ml *MachineList) countScheduled(nodeRole airshipv1.VmRoles, c client.Client, sipCfg *airshipv1.SipConfig) int {
+
+	bmhList := &metal3.BareMetalHostList{}
+	scheduleLabels := map[string]string{
+		SipScheduleLabel: "true",
+		SipClusterLabel:  sipCfg.ClusterName,
+		SipNodeTypeLabel: string(nodeRole),
+	}
+
+	err := c.List(context.Background(), bmhList, client.MatchingLabels(scheduleLabels))
+	if err != nil {
+		return 0
+	}
+	return len(bmhList.Items)
+}
+
+func (ml *MachineList) scheduleIt(nodeRole airshipv1.VmRoles, nodeCfg airshipv1.NodeSet, bmList *metal3.BareMetalHostList,
+	scheduleSetMap map[airshipv1.SchedulingOptions]*ScheduleSet, c client.Client, sipCfg *airshipv1.SipConfig) error {
 	validBmh := true
-	nodeTarget := (nodeCfg.Count.Active + nodeCfg.Count.Standby)
+	nodeTarget := (nodeCfg.Count.Active + nodeCfg.Count.Standby) - ml.countScheduled(nodeRole, c, sipCfg)
+
 	fmt.Printf("Schedule.scheduleIt  nodeRole:%v nodeTarget:%d nodeCfg.VmFlavor:%s  ml.bmhs len:%d \n", nodeRole, nodeTarget, nodeCfg.VmFlavor, len(ml.bmhs))
 	for _, bmh := range bmList.Items {
-		fmt.Printf("---------------\n Schedule.scheduleIt  bmh.ObjectMeta.Name:%s \n", bmh.ObjectMeta.Name)
+		//fmt.Printf("---------------\n Schedule.scheduleIt  bmh.ObjectMeta.Name:%s \n", bmh.ObjectMeta.Name)
 		for _, constraint := range nodeCfg.Scheduling {
 			// Do I care about this constraint
 
@@ -255,11 +251,11 @@ func (ml *MachineList) scheduleIt(nodeRole airshipv1.VmRoles, nodeCfg airshipv1.
 						scheduleSetMap[constraint].Add(cLabelValue)
 					}
 				}
-				fmt.Printf("Schedule.scheduleIt cLabelValue:%s, cFlavorMatches:%t scheduleSetMap[%v]:%v\n", cLabelValue, cFlavorMatches, constraint, scheduleSetMap[constraint])
+				//fmt.Printf("Schedule.scheduleIt cLabelValue:%s, cFlavorMatches:%t scheduleSetMap[%v]:%v\n", cLabelValue, cFlavorMatches, constraint, scheduleSetMap[constraint])
 
 			}
 		}
-		fmt.Printf("Schedule.scheduleIt nodeTarget:%d, validBmh:%t  ml.bmhs len:%d\n", nodeTarget, validBmh, len(ml.bmhs))
+		//fmt.Printf("Schedule.scheduleIt nodeTarget:%d, validBmh:%t  ml.bmhs len:%d\n", nodeTarget, validBmh, len(ml.bmhs))
 		// All the constraints have been checked
 		if validBmh {
 			// Lets add it to the list as a schedulable thing
@@ -273,7 +269,8 @@ func (ml *MachineList) scheduleIt(nodeRole airshipv1.VmRoles, nodeCfg airshipv1.
 			}
 			// Probable need to use the nodeRole as a label here
 			ml.bmhs = append(ml.bmhs, m)
-			fmt.Printf("Schedule.scheduleIt ADDED   ml.bmhs len:%d machine:%v \n", len(ml.bmhs), m)
+			fmt.Printf("---------------\nSchedule.scheduleIt ADDED  bmh.ObjectMeta.Name:%s  ml.bmhs len:%d machine:%v \n", bmh.ObjectMeta.Name, len(ml.bmhs), m)
+			// TODO Probablle should remove the bmh from the list so if there are other node targtes they dont even take it into account
 			nodeTarget = nodeTarget - 1
 			if nodeTarget == 0 {
 				break
@@ -305,6 +302,7 @@ func (ml *MachineList) Extrapolate(sip airshipv1.SIPCluster, c client.Client) er
 		// Identify Network Data Secret name
 
 		networkDataSecret := &corev1.Secret{}
+		//fmt.Printf("Schedule.Extrapolate Namespace:%s  Name:%s\n", bmh.Spec.NetworkData.Namespace, bmh.Spec.NetworkData.Name)
 		// c is a created client.
 		err := c.Get(context.Background(), client.ObjectKey{
 			Namespace: bmh.Spec.NetworkData.Namespace,
@@ -313,7 +311,7 @@ func (ml *MachineList) Extrapolate(sip airshipv1.SIPCluster, c client.Client) er
 		if err != nil {
 			return err
 		}
-
+		//fmt.Printf("Schedule.Extrapolate  networkDataSecret:%v\n", networkDataSecret)
 		// Assuming there might be other data
 		// Retrieve IP's for Service defined Network Interfaces
 		err = ml.getIp(machine, networkDataSecret, sip.Spec.InfraServices)
@@ -472,45 +470,40 @@ func (ml *MachineList) Extrapolate(sip airshipv1.SIPCluster, c client.Client) er
     ]
 }
 ***/
+
 func (ml *MachineList) getIp(machine *Machine, networkDataSecret *corev1.Secret, infraServices map[airshipv1.InfraService]airshipv1.InfraConfig) error {
 	var secretData interface{}
 	// Now I have the Secret
 	// Lets find the IP's for all Interfaces defined in Cfg
+	foundIp := false
 	for svcName, svcCfg := range infraServices {
 		// Did I already find teh IP for these interface
 		if machine.Data.IpOnInterface[svcCfg.NodeInterface] == "" {
-			foundInterface := false
+			json.Unmarshal(networkDataSecret.Data["networkData"], &secretData)
-			foundIp := false
+			fmt.Printf("Schedule.Extrapolate.getIp secretData:%v\n", secretData)
-			jsonData := []byte(networkDataSecret.StringData["networkData"])
+
-			json.Unmarshal(jsonData, &secretData)
+			queryFilter := fmt.Sprintf("$..networks[? (@.id==\"%s\")].ip_address", svcCfg.NodeInterface)
-			networkData := secretData.(map[string]interface{})
+			fmt.Printf("Schedule.Extrapolate.getIp queryFilter:%v\n", queryFilter)
-			for ndName, ndData := range networkData {
+			ip_address, err := jsonpath.Get(queryFilter, secretData)
-				if ndName == "networks" {
+
-					switch ndData := ndData.(type) {
+			if err == nil {
-					case []interface{}:
+				foundIp = true
-						for _, ndInfo := range ndData {
+				for _, value := range ip_address.([]interface{}) {
-							ndInfoSlice := strings.Split(fmt.Sprintf("%v", ndInfo), ":")
+					machine.Data.IpOnInterface[svcCfg.NodeInterface] = value.(string)
-							if ndInfoSlice[0] == "id" && ndInfoSlice[0] == svcCfg.NodeInterface {
-								foundInterface = true
-							}
-							if ndInfoSlice[0] == "ip_address" && foundInterface {
-								machine.Data.IpOnInterface[svcCfg.NodeInterface] = ndInfoSlice[1]
-								foundIp = true
-								break
-							}
-						}
-						if foundIp {
-							break
-						}
-					}
-				}
-				if !foundIp {
-					return &ErrorHostIpNotFound{
-						HostName:    machine.Bmh.ObjectMeta.Name,
-						ServiceName: svcName,
-						IPInterface: svcCfg.NodeInterface,
-					}
 				}
+
+			}
+			// Skip if error
+			// Should signal that I need to exclude this machine
+			// Which also means I am now short potentially.
+			fmt.Printf("Schedule.Extrapolate.getIp machine.Data.IpOnInterface[%s]:%v\n", svcCfg.NodeInterface, machine.Data.IpOnInterface[svcCfg.NodeInterface])
+		}
+
+		if !foundIp {
+			return &ErrorHostIpNotFound{
+				HostName:    machine.Bmh.ObjectMeta.Name,
+				ServiceName: svcName,
+				IPInterface: svcCfg.NodeInterface,
 			}
 		}
 	}
@@ -544,7 +537,7 @@ func (ss *ScheduleSet) Add(labelValue string) {
 	ss.set[labelValue] = true
 }
 func (ss *ScheduleSet) GetLabels(labels map[string]string, flavorLabel string) (string, bool) {
-	fmt.Printf("Schedule.scheduleIt.GetLabels  labels:%v, flavorLabel:%s\n", labels, flavorLabel)
+	//fmt.Printf("Schedule.scheduleIt.GetLabels  labels:%v, flavorLabel:%s\n", labels, flavorLabel)
 	if labels == nil {
 		return "", false
 	}
@@ -554,3 +547,28 @@ func (ss *ScheduleSet) GetLabels(labels map[string]string, flavorLabel string) (
 	}
 	return labels[ss.labelName], false
 }
+
+/*
+ApplyLabel : marks the appropriate machine labels to teh vBMH's that
+have benn selected by the scheduling.
+This is done only after the Infrastcuture Services have been  deployed
+*/
+func (ml *MachineList) ApplyLabels(sip airshipv1.SIPCluster, c client.Client) error {
+	for _, machine := range ml.bmhs {
+		fmt.Printf("Apply  machine:%v\n", machine)
+		bmh := machine.Bmh
+		bmh.Labels[SipClusterLabel] = sip.Spec.Config.ClusterName
+		bmh.Labels[SipScheduleLabel] = "true"
+		bmh.Labels[SipNodeTypeLabel] = string(machine.VmRole)
+
+		// This is bombing when it find 1 error
+		// Might be better to acculumalte the errors, and
+		// Allow it  to continue.
+		err := c.Update(context.Background(), bmh)
+		if err != nil {
+			fmt.Printf("ApplyLabel bmh:%s err:%v\n", bmh.ObjectMeta.Name, err)
+			return err
+		}
+	}
+	return nil
+}