package main
import (
"fmt"
"os/exec"
"path"
"runtime"
"strings"
"github.com/metalkast/metalkast/cmd/kast/bootstrap"
"github.com/metalkast/metalkast/cmd/kast/log"
"github.com/spf13/cobra"
clusterctllog "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
)
// bootstrapCmd represents the bootstrap command
var (
bootstrapCmd = &cobra.Command{
Use: "bootstrap",
Short: "Bootstraps a metalkast cluster from the provided manifests",
Long: `Bootstraps a metalkast cluster from the provided manifests.
On a high level the bootstrap process performs the following steps:
1. Boot the bootstrap node from a live CD and start and initiate a Kubernetes cluster
2. Provision target cluster from the bootstrap node cluster
3. Pivot: Move the target cluster's manifests to itself
4. Join the rest of the nodes to the target cluster
Usage:
kast bootstrap MANIFESTS...
`,
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
runDirectory, err := CreateRunDirectory()
if err != nil {
log.Log.Error(err, "Failed to create run directory")
return err
}
cliLogger, err := log.NewLogger(log.LoggerOptions{
OutputPath: path.Join(runDirectory, "run.log"),
})
if err != nil {
return err
}
defer (cliLogger.GetSink()).(*log.TeaLogSink).Close()
log.SetLogger(cliLogger)
clusterctllog.SetLogger(cliLogger.V(1).WithName("clusterctl"))
err = checkIpmitoolInstalled()
if err != nil {
switch runtime.GOOS {
case "linux":
log.Log.V(1).Info("To install ipmitool on Ubuntu run:\n\n\tapt-get install -y ipmitool")
case "darwin":
log.Log.V(1).Info("To install ipmitool on MacOS run:\n\n\tbrew install ipmitool")
default:
log.Log.Error(fmt.Errorf("platform %s is not suported", runtime.GOOS), "kast is not supported on your platform")
}
return fmt.Errorf("failed to detect ipmitool installation: %w", err)
}
b, err := bootstrap.FromManifests(args)
if err != nil {
return fmt.Errorf("failed to configure bootstrap from manifests: %w", err)
}
if err := b.Run(bootstrap.BootstrapOptions{
BootstrapNodeOptions: bootstrap.BootstrapNodeOptions{
KubeCfgDestPath: path.Join(runDirectory, "bootstrap.kubeconfig"),
SSHKeyDestPath: path.Join(runDirectory, "ssh.key"),
},
}); err != nil {
log.Log.Error(err, "Bootstrap failed")
return err
}
return nil
},
}
)
func init() {
rootCmd.AddCommand(bootstrapCmd)
}
func checkIpmitoolInstalled() error {
cmd := exec.Command("ipmitool", "-V")
cmd.Stderr = nil
cmd.Stdout = nil
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to run ipmitool: %w", err)
}
const expectedVersionPrefix = "ipmitool version 1."
if !strings.HasPrefix(string(out), expectedVersionPrefix) {
return fmt.Errorf(
"failed to detect ipmitool version: version reported '%s', but expected '1.0+'",
strings.TrimSpace(strings.TrimPrefix(string(out), expectedVersionPrefix)),
)
}
return nil
}
package bootstrap
import (
"bytes"
"context"
"fmt"
"net"
"path"
"reflect"
"regexp"
"strings"
"time"
"github.com/manifestival/manifestival"
bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
metal3v1beta1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1"
"github.com/metalkast/metalkast/cmd/kast/log"
"github.com/metalkast/metalkast/pkg/cluster"
"github.com/metalkast/metalkast/pkg/kustomize"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/clientcmd"
clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
bootstrapv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
kubeadmv1beta1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
type Bootstrap struct {
bootstrapClusterConfig bootstrapClusterConfig
manifests manifestival.Manifest
targetClusterPreMoveManifests manifestival.Manifest
pollInterval time.Duration
}
func FromManifests(manifestsPaths []string) (*Bootstrap, error) {
manifests := manifestival.Manifest{}
for _, p := range manifestsPaths {
manifestsYaml, err := kustomize.Build(p)
if err != nil {
return nil, err
}
m, err := manifestival.ManifestFrom(manifestival.Reader(bytes.NewReader(manifestsYaml)))
if err != nil {
return nil, fmt.Errorf("failed to convert kustomize layer (%s) to in-memory manifests: %w", p, err)
}
manifests = manifests.Append(m)
}
bcc, err := bootstrapClusterConfigFromManifests(manifests)
if err != nil {
return nil, fmt.Errorf("failed to parse bootstrap node BMC config from manifests: %w", err)
}
return &Bootstrap{
manifests: manifests,
bootstrapClusterConfig: *bcc,
targetClusterPreMoveManifests: targetClusterPreMoveManifests(manifests),
pollInterval: time.Second,
}, nil
}
type bootstrapClusterConfig struct {
bootstrapNodeOptions BootstrapNodeOptions
bootstrapClusterManifests manifestival.Manifest
clusterNamespace string
}
func bootstrapClusterConfigFromManifests(manifests manifestival.Manifest) (*bootstrapClusterConfig, error) {
var setup bootstrapClusterConfig
bmhManifests := manifests.Filter(manifestival.ByGVK(bmov1alpha1.GroupVersion.WithKind("BareMetalHost")))
bmhResources := bmhManifests.Resources()
if len(bmhResources) == 0 {
return nil, fmt.Errorf("failed to find any BareMetalHosts")
}
bmhManifest, err := manifestival.ManifestFrom(manifestival.Slice(append(bmhManifests.Resources()[:1], bmhManifests.Resources()[2:]...)))
if err != nil {
panic(fmt.Errorf("failed to create manifests subset: %w", err))
}
kubeadmControlPlane := &kubeadmv1beta1.KubeadmControlPlane{}
setup.bootstrapClusterManifests, err = manifests.Filter(
manifestival.Not(manifestival.In(bmhManifest)),
manifestival.Not(manifestival.ByAnnotation(bootstrapClusterApplyAnnotation, "false")),
manifestival.Not(manifestival.ByKind(reflect.TypeOf(clusterapiv1beta1.MachineDeployment{}).Name())),
).Transform(func(u *unstructured.Unstructured) error {
if u.GroupVersionKind() == kubeadmv1beta1.GroupVersion.WithKind("KubeadmControlPlane") {
unstructured.SetNestedField(u.Object, int64(1), "spec", "replicas")
// ClusterAPI requires maxSurge to be non-zero when replicas < 3
unstructured.SetNestedField(u.Object, int64(1), "spec", "rolloutStrategy", "rollingUpdate", "maxSurge")
setup.clusterNamespace = u.GetNamespace()
err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, kubeadmControlPlane)
if err != nil {
return err
}
}
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to transform some manifests: %w", err)
}
bmh := &bmov1alpha1.BareMetalHost{}
err = runtime.DefaultUnstructuredConverter.FromUnstructured(bmhResources[0].Object, bmh)
if err != nil {
return nil, fmt.Errorf("failed to parse BareMetalHost (%s): %w", bmhResources[0].GetName(), err)
}
redfishUrlParser := regexp.MustCompile(`http(s)?:\/\/[^\/]*`)
setup.bootstrapNodeOptions.RedfishUrl = redfishUrlParser.FindString(bmh.Spec.BMC.Address)
if setup.bootstrapNodeOptions.RedfishUrl == "" {
return nil, fmt.Errorf("failed to find a BareMetalHost with a valid redfish address")
}
secretManifest := manifests.Filter(manifestival.All(
manifestival.ByGVK(schema.FromAPIVersionAndKind("v1", "Secret")),
manifestival.ByName(bmh.Spec.BMC.CredentialsName),
func(u *unstructured.Unstructured) bool {
return u.GetNamespace() == bmh.Namespace
},
)).Resources()
if len(secretManifest) != 1 {
return nil, fmt.Errorf("failed to find credentials for BareMetalHost (%s): %w", bmh.GetName(), err)
}
secret := corev1.Secret{}
err = runtime.DefaultUnstructuredConverter.FromUnstructured(secretManifest[0].Object, &secret)
if err != nil {
return nil, fmt.Errorf("failed to parse Secret (%s): %w", secretManifest[0].GetName(), err)
}
var ok bool
setup.bootstrapNodeOptions.RedfishUsername, ok = secret.StringData["username"]
if !ok {
setup.bootstrapNodeOptions.RedfishUsername = string(secret.Data["username"])
}
setup.bootstrapNodeOptions.RedfishPassword, ok = secret.StringData["password"]
if !ok {
setup.bootstrapNodeOptions.RedfishPassword = string(secret.Data["password"])
}
kubeadmControlPlaneMachineTemplateManifests := manifests.Filter(manifestival.All(
manifestival.ByGVK(kubeadmControlPlane.Spec.MachineTemplate.InfrastructureRef.GroupVersionKind()),
manifestival.ByName(kubeadmControlPlane.Spec.MachineTemplate.InfrastructureRef.Name),
func(u *unstructured.Unstructured) bool {
return u.GetNamespace() == kubeadmControlPlane.GetNamespace()
},
))
if l := len(kubeadmControlPlaneMachineTemplateManifests.Resources()); l != 1 {
return nil, fmt.Errorf("want exactly one machine template for KubeadmControlPlane, but found %d", l)
}
kubeadmControlPlaneMachineTemplate := &metal3v1beta1.Metal3MachineTemplate{}
err = runtime.DefaultUnstructuredConverter.FromUnstructured(
kubeadmControlPlaneMachineTemplateManifests.Resources()[0].Object,
kubeadmControlPlaneMachineTemplate,
)
if err != nil {
return nil, fmt.Errorf("failed to parse cluster's Metal3MachineTemplate: %w", err)
}
const nodeImgSuffix = ".img"
if !strings.HasSuffix(kubeadmControlPlaneMachineTemplate.Spec.Template.Spec.Image.URL, nodeImgSuffix) {
return nil, fmt.Errorf("cluster's node image URL needs to have .img extension")
}
setup.bootstrapNodeOptions.LiveIsoUrl = strings.TrimSuffix(
kubeadmControlPlaneMachineTemplate.Spec.Template.Spec.Image.URL, nodeImgSuffix,
) + "-netboot-live.iso"
return &setup, nil
}
func targetClusterPreMoveManifests(manifests manifestival.Manifest) manifestival.Manifest {
return manifests.Filter(func(u *unstructured.Unstructured) bool {
excludedGroups := []string{
kubeadmv1beta1.GroupVersion.Group,
clusterapiv1beta1.GroupVersion.Group,
bootstrapv1beta1.GroupVersion.Group,
bmov1alpha1.GroupVersion.Group,
metal3v1beta1.GroupVersion.Group,
}
return !slices.Contains(excludedGroups, u.GroupVersionKind().Group)
})
}
type BootstrapOptions struct {
BootstrapNodeOptions
}
func (b *Bootstrap) Run(options BootstrapOptions) error {
if kubecfgDestPath := options.BootstrapNodeOptions.KubeCfgDestPath; kubecfgDestPath != "" {
b.bootstrapClusterConfig.bootstrapNodeOptions.KubeCfgDestPath = kubecfgDestPath
}
bootstrapNode, err := NewBootstrapNode(b.bootstrapClusterConfig.bootstrapNodeOptions)
if err != nil {
return fmt.Errorf("failed to init bootstrap node: %w", err)
}
log.Log.Info("Provisioning bootstrap cluster")
bootstrapCluster, err := bootstrapNode.BootstrapCluster()
if err != nil {
return fmt.Errorf("failed to provision target cluster: %w", err)
}
log.Log.Info("Applying manifests to bootstrap cluster")
if err := bootstrapCluster.ApplyManifest(b.bootstrapClusterConfig.bootstrapClusterManifests); err != nil {
return fmt.Errorf("failed to apply manifests: %w", err)
}
log.Log.Info("Creating target cluster")
var lastEventTimestamp time.Time
var targetClusterInitialNodeIP string
err = wait.PollUntilContextTimeout(context.TODO(), time.Second*1, time.Hour, true, func(ctx context.Context) (bool, error) {
events := &corev1.EventList{}
if err := bootstrapCluster.List(ctx, events, &client.ListOptions{Namespace: b.bootstrapClusterConfig.clusterNamespace}); err != nil {
log.Log.V(1).Error(err, "failed to list events")
return false, nil
}
slices.SortStableFunc(events.Items, func(a, b corev1.Event) int {
return a.LastTimestamp.Compare(b.LastTimestamp.Time)
})
for _, e := range events.Items {
if e.LastTimestamp.After(lastEventTimestamp) {
log.Log.V(1).Info(e.Message, "name", e.InvolvedObject.Name, "kind", e.InvolvedObject.Kind)
}
}
if eventCount := len(events.Items); eventCount > 0 {
lastEventTimestamp = events.Items[eventCount-1].LastTimestamp.Time
}
bareMetalHostsLists := &bmov1alpha1.BareMetalHostList{}
if err := bootstrapCluster.List(ctx, bareMetalHostsLists); err != nil {
log.Log.V(1).Error(err, "failed to list bareMetalHostsLists")
return false, nil
}
if len(bareMetalHostsLists.Items) > 1 {
return false, fmt.Errorf("expected only single BareMetalHost")
} else if len(bareMetalHostsLists.Items) != 1 {
return false, nil
}
if bareMetalHostsLists.Items[0].Status.HardwareDetails != nil {
for _, nic := range bareMetalHostsLists.Items[0].Status.HardwareDetails.NIC {
ip := net.ParseIP(nic.IP)
if ip == nil || ip.To4() == nil {
continue
}
conn, err := net.DialTimeout("tcp", net.JoinHostPort(nic.IP, "6443"), time.Second*2)
if err != nil {
continue
}
conn.Close()
targetClusterInitialNodeIP = nic.IP
return true, nil
}
}
return false, nil
})
if err != nil {
return fmt.Errorf("timed out waiting for target cluster to be created: %w", err)
}
targetClusterKubeconfig, err := b.getTargetClusterKubeconfig(bootstrapCluster)
if err != nil {
return fmt.Errorf("failed to get target cluster kubeconfig: %w", err)
}
temporaryTargetClusterKubeconfig, err := kubeconfigWithReplacedHost(targetClusterKubeconfig, targetClusterInitialNodeIP)
if err != nil {
return fmt.Errorf("failed to create temporary target cluster kubeconfig: %w", err)
}
targetCluster, err := cluster.NewCluster(
temporaryTargetClusterKubeconfig,
path.Join(path.Dir(bootstrapNode.kubeCfgDest), "target.kubeconfig"),
log.Log.V(1).WithName("target cluster"),
)
if err != nil {
return fmt.Errorf("failed to initialize target cluster client: %w", err)
}
log.Log.Info("Applying initial manifests to target cluster")
if err := targetCluster.Applier.ApplyManifest(b.targetClusterPreMoveManifests); err != nil {
return fmt.Errorf("failed to apply pre-move manifests to target cluster: %w", err)
}
log.Log.Info("Waiting for target cluster to finish provisioning")
if err := wait.PollUntilContextTimeout(context.TODO(), time.Second*1, time.Minute*5, true, func(ctx context.Context) (bool, error) {
clusterList := &clusterv1.ClusterList{}
if err := bootstrapCluster.List(ctx, clusterList); err != nil {
log.Log.V(1).Error(err, "failed to list clusters")
return false, nil
}
if len(clusterList.Items) > 1 {
return false, fmt.Errorf("expected only single clusters")
} else if len(clusterList.Items) != 1 {
return false, nil
}
return clusterList.Items[0].Status.ControlPlaneReady, nil
}); err != nil {
return fmt.Errorf("timed out waiting for target cluster to be provisioned: %w", err)
}
targetCluster, err = cluster.NewCluster(
targetClusterKubeconfig,
path.Join(path.Dir(bootstrapNode.kubeCfgDest), "target.kubeconfig"),
log.Log.V(1).WithName("target cluster"),
)
if err != nil {
return fmt.Errorf("failed to initialize target cluster client: %w", err)
}
log.Log.Info("Moving the cluster")
wait.PollUntilContextCancel(context.TODO(), b.pollInterval, true, func(ctx context.Context) (done bool, err error) {
err = bootstrapCluster.Move(targetCluster, b.bootstrapClusterConfig.clusterNamespace)
if err != nil {
log.Log.V(1).Error(err, "Failed to move the cluster")
log.Log.V(1).Info("Retrying to move the cluster")
}
return err == nil, nil
})
log.Log.Info("Applying all manifests to the target cluster")
if err := targetCluster.Applier.ApplyManifest(b.manifests); err != nil {
return fmt.Errorf("failed to apply all manifests after cluster pivoting: %w", err)
}
log.Log.Info(fmt.Sprintf(
`Your Kubernetes target cluster has initialized successfully!
To start using your cluster, you need to run the following:
export KUBECONFIG=%s
You should now commit all the source files to the git repository.`,
targetCluster.KubeCfgPath()))
return nil
}
func (b *Bootstrap) getTargetClusterKubeconfig(bootstrapCluster *cluster.Cluster) ([]byte, error) {
kubeadmControlPlaneResources := b.manifests.Filter(
manifestival.ByGVK(kubeadmv1beta1.GroupVersion.WithKind("KubeadmControlPlane")),
).Resources()
if l := len(kubeadmControlPlaneResources); l != 1 {
return nil, fmt.Errorf("want exactly one KubeadmControlPlane in manifests but found: %d", l)
}
kubeadmControlPlane := kubeadmControlPlaneResources[0]
kubeconfigSecret := corev1.Secret{}
err := wait.PollUntilContextTimeout(context.TODO(), b.pollInterval, time.Minute*10, true, func(ctx context.Context) (done bool, err error) {
return bootstrapCluster.Client.Get(context.TODO(), types.NamespacedName{
Name: fmt.Sprintf("%s-kubeconfig", kubeadmControlPlane.GetName()),
Namespace: kubeadmControlPlane.GetNamespace(),
}, &kubeconfigSecret) == nil, nil
})
if err != nil {
return nil, fmt.Errorf("failed to get target cluster kubeconfig from secret: %w", err)
}
return kubeconfigSecret.Data["value"], nil
}
func kubeconfigWithReplacedHost(kubeconfigContentInput []byte, newHost string) ([]byte, error) {
kubeconfig, err := clientcmd.Load(kubeconfigContentInput)
if err != nil {
return nil, err
}
clusters := maps.Keys(kubeconfig.Clusters)
if len(clusters) != 1 {
return nil, fmt.Errorf("expected single cluster in kubeconfig, got %v", len(clusters))
}
kubeconfig.Clusters[clusters[0]].Server = fmt.Sprintf("https://%s:6443", newHost)
kubeconfigContentResult, err := clientcmd.Write(*kubeconfig)
if err != nil {
return nil, fmt.Errorf("failed to render kubeconfig: %w", err)
}
return kubeconfigContentResult, nil
}
package bootstrap
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"encoding/pem"
"fmt"
"os"
"strings"
"text/template"
"time"
"github.com/Netflix/go-expect"
"github.com/metalkast/metalkast/cmd/kast/log"
"github.com/metalkast/metalkast/pkg/bmc"
"github.com/metalkast/metalkast/pkg/cluster"
"github.com/metalkast/metalkast/pkg/logr"
"golang.org/x/crypto/ssh"
"k8s.io/apimachinery/pkg/util/wait"
)
const (
BootstrapNodeLiveCdUsername = "bootstrap"
BootstrapNodeLiveCdPassword = "bootstrap"
)
type BootstrapNode struct {
bmc *bmc.BMC
liveIsoUrl string
kubeCfgDest string
sshKeyDest string
}
type BootstrapNodeOptions struct {
RedfishUrl string
RedfishUsername string
RedfishPassword string
LiveIsoUrl string
KubeCfgDestPath string
SSHKeyDestPath string
}
func NewBootstrapNode(options BootstrapNodeOptions) (*BootstrapNode, error) {
bmc, err := bmc.NewBMC(options.RedfishUrl, options.RedfishUsername, options.RedfishPassword, log.Log.V(1).WithName("bmc"))
if err != nil {
return nil, fmt.Errorf("failed to init BMC for bootstrap node: %w", err)
}
kubeCfgDest := options.KubeCfgDestPath
if kubeCfgDest == "" {
kubeCfgDest = "bootstrap.kubeconfig"
}
sshKeyDest := options.SSHKeyDestPath
if sshKeyDest == "" {
sshKeyDest = "ssh.key"
}
return &BootstrapNode{
bmc: bmc,
liveIsoUrl: options.LiveIsoUrl,
kubeCfgDest: kubeCfgDest,
sshKeyDest: sshKeyDest,
}, nil
}
func (n *BootstrapNode) start() error {
if err := n.bmc.RedfishClient.InsertMedia(n.liveIsoUrl); err != nil {
return err
}
if err := n.bmc.RedfishClient.SetBootMedia(); err != nil {
return err
}
if err := n.bmc.RedfishClient.Boot(); err != nil {
return err
}
return nil
}
type sshConfig struct {
user string
userAuthKey ecdsa.PrivateKey
hostIP string
hostPublicKey ssh.PublicKey
}
func (c *sshConfig) sshClient() (*ssh.Client, error) {
signer, err := ssh.NewSignerFromKey(&c.userAuthKey)
if err != nil {
return nil, fmt.Errorf("failed to create signer from a private key: %w", err)
}
config := &ssh.ClientConfig{
User: c.user,
Auth: []ssh.AuthMethod{
ssh.PublicKeys(signer),
},
HostKeyCallback: ssh.FixedHostKey(c.hostPublicKey),
}
sshConn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", c.hostIP, 22), config)
if err != nil {
return nil, fmt.Errorf("failed to dial: %s", err)
}
return sshConn, nil
}
func (n *BootstrapNode) configureSSH(privateKey ecdsa.PrivateKey) (*sshConfig, error) {
var (
hostPublicKey ssh.PublicKey
hostIp string
)
err := wait.PollUntilContextTimeout(context.TODO(), time.Second, 20*time.Minute, true, func(ctx context.Context) (bool, error) {
err := n.bmc.IpmiClient.Run(ctx, func(c *expect.Console) error {
if _, err := c.Write([]byte("\000")); err != nil {
return err
}
if _, err := c.SendLine("\n"); err != nil {
return err
}
if _, err := c.ExpectString("login:"); err != nil {
return err
}
if _, err := c.SendLine(BootstrapNodeLiveCdUsername); err != nil {
return err
}
if _, err := c.ExpectString("Password:"); err != nil {
return err
}
if _, err := c.SendLine(BootstrapNodeLiveCdPassword); err != nil {
return err
}
const prompt = "$ "
if _, err := c.ExpectString(prompt); err != nil {
return err
}
if _, err := c.SendLine("sudo ssh-keygen -A && sudo systemctl enable --now ssh"); err != nil {
return err
}
if _, err := c.ExpectString(prompt); err != nil {
return err
}
publicKey, err := ssh.NewPublicKey(privateKey.Public())
if err != nil {
return err
}
if _, err := c.SendLine(fmt.Sprintf("mkdir -p ~/.ssh && echo '%s' > ~/.ssh/authorized_keys", ssh.MarshalAuthorizedKey(publicKey))); err != nil {
return err
}
if _, err := c.ExpectString(prompt); err != nil {
return err
}
const printHostPublicKeyCmd = "cat /etc/ssh/ssh_host_ecdsa_key.pub"
if _, err := c.SendLine(printHostPublicKeyCmd); err != nil {
return err
}
hostPublicKeyOutput, err := c.ExpectString(prompt)
if err != nil {
return err
}
hostPublicKey, _, _, _, err = ssh.ParseAuthorizedKey([]byte(
strings.TrimSpace(strings.TrimPrefix(
strings.TrimSuffix(hostPublicKeyOutput, prompt),
printHostPublicKeyCmd,
)),
))
if err != nil {
return err
}
// https://unix.stackexchange.com/a/167040
const printHostIpCmd = "ip route get 1.1.1.1 | grep -oP 'src \\K\\S+'"
if _, err := c.SendLine(printHostIpCmd); err != nil {
return err
}
hostIpOutput, err := c.ExpectString(prompt)
if err != nil {
return err
}
hostIp = strings.TrimSpace(strings.TrimPrefix(
strings.TrimSuffix(hostIpOutput, prompt),
printHostIpCmd,
))
if _, err := c.SendLine("exit"); err != nil {
return err
}
if _, err := c.Send("~."); err != nil {
return err
}
return nil
})
if err != nil {
log.Log.V(1).Error(err, "failed to configure ssh via IPMI")
}
return err == nil, nil
})
if err != nil {
return nil, fmt.Errorf("failed to configure ssh access to bootstrap node: %w", err)
}
return &sshConfig{
user: BootstrapNodeLiveCdUsername,
userAuthKey: privateKey,
hostIP: hostIp,
hostPublicKey: hostPublicKey,
}, nil
}
func initKubeadm(c sshConfig) error {
sshClient, err := c.sshClient()
if err != nil {
return fmt.Errorf("failed to init ssh: %s", err)
}
initSession, err := sshClient.NewSession()
if err != nil {
return fmt.Errorf("failed to create session: %s", err)
}
defer initSession.Close()
kubeadmInitCommandTemplate := `sudo bash -c '
set -x
set -eEuo pipefail
disk=$(lsblk | awk '"'"'/disk/ {print $1; exit}'"'"')
mkfs.ext4 /dev/$disk -F
mkdir /tmp/containerd
mount /dev/$disk /tmp/containerd
systemctl stop containerd
cp -r /var/lib/containerd/* /tmp/containerd/
umount /tmp/containerd
mount /dev/$disk /var/lib/containerd
systemctl start containerd
kubeVersion=$(ctr -n k8s.io i ls | grep -o -P "(?<=kube-apiserver:)v1\.[0-9]+\.[0-9]+")
# TODO: parse skip phases and pod network cidr from manifests
kubeadm init --kubernetes-version $kubeVersion --pod-network-cidr 10.244.0.0/16 --control-plane-endpoint {{ .hostname }} --skip-phases=addon/kube-proxy
export KUBECONFIG=/etc/kubernetes/admin.conf
kubectl taint nodes --all node-role.kubernetes.io/control-plane:NoSchedule-
kubectl -n kube-system create configmap cilium-apiserver-endpoint --from-literal=KUBERNETES_SERVICE_HOST={{ .hostname }}
'`
tmpl := template.Must(template.New("notImportant").Parse(kubeadmInitCommandTemplate))
kubeadmInitCommandBuilder := strings.Builder{}
if err = tmpl.Execute(&kubeadmInitCommandBuilder, map[string]interface{}{
"hostname": c.hostIP,
}); err != nil {
return fmt.Errorf("failed to execute template: %s", err)
}
logger := log.Log.V(1).WithName("ssh init cluster")
initSession.Stderr = logr.NewLogWriter(logger)
initSession.Stdout = logr.NewLogWriter(logger)
return initSession.Run(kubeadmInitCommandBuilder.String())
}
func getBootstrapClusterKubeconfig(c sshConfig) ([]byte, error) {
sshClient, err := c.sshClient()
if err != nil {
return nil, fmt.Errorf("failed to init ssh: %s", err)
}
readKubeconfigSession, err := sshClient.NewSession()
if err != nil {
return nil, fmt.Errorf("failed to create session: %s", err)
}
defer readKubeconfigSession.Close()
kubeconfig, err := readKubeconfigSession.Output("sudo cat /etc/kubernetes/admin.conf")
if err != nil {
return nil, fmt.Errorf("failed to read kubeconfig: %s", err)
}
return kubeconfig, nil
}
func (n *BootstrapNode) BootstrapCluster() (*cluster.Cluster, error) {
var err error
if err = n.start(); err != nil {
return nil, fmt.Errorf("failed to start bootstrap node: %w", err)
}
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, fmt.Errorf("failed to generate temporary SSH key to use for bootstrap node: %w", err)
}
sshKeyFileContent, err := ssh.MarshalPrivateKey(privateKey, "")
if err != nil {
return nil, fmt.Errorf("failed to serialize ssh private key: %w", err)
}
if err = os.WriteFile(n.sshKeyDest, pem.EncodeToMemory(sshKeyFileContent), 0600); err != nil {
return nil, fmt.Errorf("failed to save ssh private key: %w", err)
}
sshConfig, err := n.configureSSH(*privateKey)
if err != nil {
return nil, fmt.Errorf("failed to configure bootstrap node ssh via IPMI: %w", err)
}
if err = initKubeadm(*sshConfig); err != nil {
return nil, fmt.Errorf("failed to initialize kubeadm on bootstrap node: %w", err)
}
bootstrapClusterKubeconfig, err := getBootstrapClusterKubeconfig(*sshConfig)
if err != nil {
return nil, fmt.Errorf("failed to fetch kubeconfig from bootstrap node cluster: %w", err)
}
bootstrapCluster, err := cluster.NewCluster(
bootstrapClusterKubeconfig,
n.kubeCfgDest,
log.Log.V(1).WithName("bootstrap cluster"),
)
if err != nil {
return nil, fmt.Errorf("failed to initialize cluster setup: %w", err)
}
return bootstrapCluster, nil
}
package main
import (
"bytes"
"context"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/getsops/sops/v3/decrypt"
"github.com/hashicorp/go-retryablehttp"
"github.com/manifestival/manifestival"
"github.com/metalkast/metalkast/cmd/kast/log"
"github.com/spf13/cobra"
"github.com/stmcginnis/gofish"
"golang.org/x/sync/errgroup"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// generateCmd represents the generate command
var (
generateCmd = &cobra.Command{
Use: "generate",
Short: "Generates BareMetalHosts manifests from set of Secret credentials",
Long: `Generates BareMetalHosts manifests from set of Secret credentials.
User should provide a source .yaml file containing set of Secrets, each containing
credentials for a given set of nodes configured in metalkast.io/redfish-urls annotation.
Example:
apiVersion: v1
kind: Secret
metadata:
name: k8s-nodes
annotations:
metalkast.io/redfish-urls: |-
https://192.168.122.101
https://192.168.122.102
https://192.168.122.103
stringData:
username: admin
password: password
type: Opaque
Usage:
kast generate SRC DEST
`,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
cliLogger, err := log.NewLogger(log.LoggerOptions{})
if err != nil {
return fmt.Errorf("failed to init logger: %w", err)
}
defer (cliLogger.GetSink()).(*log.TeaLogSink).Close()
log.SetLogger(cliLogger)
return generateBareMetalHosts(args[0], args[1], generateOptions{})
},
}
)
func init() {
rootCmd.AddCommand(generateCmd)
}
const (
redfishUrlsAnnotation = "metalkast.io/redfish-urls"
bareMetalHostSecretUsernameField = "username"
bareMetalHostSecretPasswordField = "password"
)
type generateOptions struct {
HTTPClient *http.Client
}
func generateBareMetalHosts(inputPath, outputPath string, options generateOptions) error {
manifests, err := manifestival.ManifestFrom(manifestival.Path(inputPath))
if len(manifests.Filter(func(u *unstructured.Unstructured) bool {
_, isSopsEncrypted := u.Object["sops"]
return isSopsEncrypted
}).Resources()) > 0 {
decrypted, err := decrypt.File(inputPath, "yaml")
if err != nil {
return fmt.Errorf("failed to decrypt input: %w", err)
}
manifests, err = manifestival.ManifestFrom(manifestival.Reader(bytes.NewReader(decrypted)))
if err != nil {
return err
}
}
if err != nil {
return fmt.Errorf("failed to decrypt input: %w", err)
}
if err != nil {
return fmt.Errorf("failed to read source manifests (%v): %w", inputPath, err)
}
secrets := manifests.Filter(manifestival.All(
manifestival.ByGVK(corev1.SchemeGroupVersion.WithKind("Secret")),
func(u *unstructured.Unstructured) bool {
_, found := u.GetAnnotations()[redfishUrlsAnnotation]
return found
},
))
ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
defer cancel()
g, ctx := errgroup.WithContext(ctx)
resultsChannel := make(chan *yaml.RNode)
var bmhCount int
for _, s := range secrets.Resources() {
secret := &corev1.Secret{}
err := runtime.DefaultUnstructuredConverter.FromUnstructured(s.Object, secret)
if err != nil {
return err
}
for i, redfishUrl := range strings.Split(secret.GetAnnotations()[redfishUrlsAnnotation], "\n") {
if strings.TrimSpace(redfishUrl) == "" {
continue
}
bmhCount++
outputIndex := bmhCount
redfishUrl := strings.TrimSuffix(redfishUrl, "/")
suffix := fmt.Sprintf("-%d", i+1)
g.Go(func() error {
bmhRNode, err := generateBareMetalHost(*secret, redfishUrl, suffix, outputIndex, options)
if err != nil {
return err
}
resultsChannel <- bmhRNode
return nil
})
}
}
var results []*yaml.RNode
outer:
for {
select {
case res := <-resultsChannel:
results = append(results, res)
if len(results) == bmhCount {
err := g.Wait()
if err != nil {
return err
}
}
case <-ctx.Done():
if len(results) != bmhCount {
return fmt.Errorf("some nodes timed out")
}
break outer
}
}
f, err := os.OpenFile(outputPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer f.Close()
writer := kio.ByteWriter{
Writer: f,
Sort: true,
}
return writer.Write(results)
}
func generateBareMetalHost(secret corev1.Secret, redfishUrl, suffix string, outputIndex int, options generateOptions) (*yaml.RNode, error) {
username, ok := secret.StringData[bareMetalHostSecretUsernameField]
if !ok {
username = string(secret.Data[bareMetalHostSecretUsernameField])
}
password, ok := secret.StringData[bareMetalHostSecretPasswordField]
if !ok {
password = string(secret.Data[bareMetalHostSecretPasswordField])
}
config := gofish.ClientConfig{
Endpoint: redfishUrl,
// TODO: (GAL-311) Parametrize
Insecure: true,
Username: username,
Password: password,
BasicAuth: true,
}
if options.HTTPClient != nil {
config.HTTPClient = options.HTTPClient
}
client, err := gofish.Connect(config)
if err != nil {
return nil, fmt.Errorf("failed to create redfish client: %w", err)
}
defer client.Logout()
retryClient := retryablehttp.NewClient()
retryClient.Logger = nil
retryClient.HTTPClient = client.HTTPClient
client.HTTPClient = retryClient.StandardClient()
systems, err := client.Service.Systems()
if err != nil {
return nil, fmt.Errorf("failed to list systems: %w", err)
}
if len(systems) == 0 {
return nil, fmt.Errorf("no systems found")
}
provider := "redfish"
if systems[0].Manufacturer == "Dell Inc." {
provider = "idrac"
}
// TODO: add support for HPE iLO 5
ethernetInterfaces, err := systems[0].EthernetInterfaces()
if err != nil {
return nil, fmt.Errorf("failed to list ethernet interfaces: %w", err)
}
if len(ethernetInterfaces) == 0 {
return nil, fmt.Errorf("no ethernet interfaces found")
}
redfishUrlParsed, err := url.Parse(redfishUrl)
if err != nil {
return nil, fmt.Errorf("failed to parse redfish url (%v): %w", redfishUrl, err)
}
bmh := map[string]interface{}{
"apiVersion": "metal3.io/v1alpha1",
"kind": "BareMetalHost",
"metadata": map[string]interface{}{
"name": fmt.Sprintf("node-%d", outputIndex),
"annotations": map[string]interface{}{
kioutil.IndexAnnotation: fmt.Sprint(outputIndex),
},
},
"spec": map[string]interface{}{
"automatedCleaningMode": "disabled",
"bmc": map[string]interface{}{
"address": fmt.Sprintf("%s-virtualmedia+%s", provider, redfishUrlParsed.JoinPath(systems[0].ODataID)),
"credentialsName": secret.Name,
},
"bootMACAddress": ethernetInterfaces[0].MACAddress,
"online": true,
"rootDeviceHints": map[string]interface{}{
"minSizeGigabytes": 10,
},
},
}
bmhRNode, err := yaml.FromMap(bmh)
if err != nil {
return nil, err
}
if secret.Namespace != "" {
bmhRNode.SetNamespace(secret.Namespace)
}
return bmhRNode, nil
}
package log
import (
"github.com/go-logr/logr"
"sigs.k8s.io/controller-runtime/pkg/log"
)
// SetLogger sets a concrete logging implementation for all deferred Loggers.
func SetLogger(l logr.Logger) {
Log = l
}
// Log is the base logger used by kubebuilder. It delegates
// to another logr.Logger. You *must* call SetLogger to
// get any actual logging.
var Log = logr.New(log.NullLogSink{})
package log
import (
"fmt"
"os"
"strings"
"github.com/charmbracelet/bubbles/spinner"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/go-logr/logr"
"github.com/metalkast/metalkast/cmd/kast/options"
"github.com/muesli/termenv"
)
type model struct {
logs chan logEntry
spinner spinner.Model
activeTopLevelLog *logEntry
subLevelLogs []logEntry
subLevelLogsCap int
terminalWidth int
logFile *os.File
}
type quit struct{}
var (
// Color code to name reference: https://github.com/muesli/termenv#color-chart
spinnerStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("4"))
checkmarkStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("2"))
errorLogStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("1"))
subLevelLogStyle = lipgloss.NewStyle().Faint(true)
appStyle = lipgloss.NewStyle()
logLineStyle = lipgloss.NewStyle()
)
// A command that waits for the activity on a channel.
func waitForActivity(sub chan logEntry) tea.Cmd {
return func() tea.Msg {
for s := range sub {
return s
}
return quit{}
}
}
func (m model) Init() tea.Cmd {
return tea.Batch(
m.spinner.Tick,
waitForActivity(m.logs),
)
}
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case logEntry:
if m.logFile != nil {
fileLogLine := msg.String()
if msg.err != nil {
fileLogLine = fmt.Sprintf("[ERROR]%s: %s", fileLogLine, msg.err)
}
m.logFile.WriteString(fileLogLine + "\n")
}
var permanentLogs []tea.Cmd
if msg.level == 0 && msg.err == nil {
m.subLevelLogs = []logEntry{}
if m.activeTopLevelLog != nil {
m.activeTopLevelLog.success = true
renderLine := logLineStyle.Width(m.terminalWidth).Render(checkmarkStyle.Render("✓ ") + m.activeTopLevelLog.String())
for _, l := range strings.Split(renderLine, "\n") {
permanentLogs = append(permanentLogs, tea.Println(l))
}
}
if msg.success {
msg.success = true
renderLine := logLineStyle.Width(m.terminalWidth).Render(checkmarkStyle.Render("✓ ") + msg.String())
for _, l := range strings.Split(renderLine, "\n") {
permanentLogs = append(permanentLogs, tea.Println(l))
}
m.activeTopLevelLog = nil
} else {
m.activeTopLevelLog = &msg
}
} else if m.activeTopLevelLog != nil {
m.subLevelLogs = append(m.subLevelLogs, msg)
} else {
renderLine := msg.String()
if msg.err != nil {
renderLine = errorLogStyle.Render(msg.String())
permanentLogs = append(permanentLogs, tea.Println(logLineStyle.Width(m.terminalWidth).Render(msg.String())))
}
permanentLogs = append(permanentLogs, tea.Println(logLineStyle.Width(m.terminalWidth).Render(renderLine)))
}
return m, tea.Batch(waitForActivity(m.logs), tea.Sequence(permanentLogs...))
case spinner.TickMsg:
var cmd tea.Cmd
m.spinner, cmd = m.spinner.Update(msg)
return m, cmd
case tea.WindowSizeMsg:
m.terminalWidth = msg.Width
return m, nil
case quit:
var commands []tea.Cmd
if m.activeTopLevelLog != nil {
m.activeTopLevelLog.success = true
renderLine := logLineStyle.Width(m.terminalWidth).Render(checkmarkStyle.Render("✓ ") + m.activeTopLevelLog.String())
for _, l := range strings.Split(renderLine, "\n") {
commands = append(commands, tea.Println(l))
}
}
printSubLevelLogs := m.activeTopLevelLog == nil
for _, l := range m.subLevelLogs {
if l.err != nil {
printSubLevelLogs = true
break
}
}
if printSubLevelLogs {
for _, l := range m.subLevelLogs {
renderLine := l.String()
if l.err != nil {
renderLine = errorLogStyle.Render(l.String())
commands = append(commands, tea.Println(logLineStyle.Width(m.terminalWidth).Render(l.String())))
}
commands = append(commands, tea.Println(logLineStyle.Width(m.terminalWidth).Render(renderLine)))
}
}
m.activeTopLevelLog = nil
m.subLevelLogs = nil
commands = append(commands, tea.Quit)
return m, tea.Sequence(commands...)
default:
return m, nil
}
}
func (m model) View() string {
builder := strings.Builder{}
if m.activeTopLevelLog != nil {
builder.WriteString(
appStyle.Width(m.terminalWidth).Render(m.spinner.View()+m.activeTopLevelLog.String()) + "\n",
)
}
subLevelLogDisplay := []logEntry{}
subLevelLogStartIndex := 0
for i := len(m.subLevelLogs) - 1; i >= 0; i-- {
if m.subLevelLogs[i].refreshable {
subLevelLogDisplay = append(subLevelLogDisplay, m.subLevelLogs[i])
subLevelLogStartIndex = i + 1
break
}
}
if capEntriesIndexStart := len(m.subLevelLogs) - m.subLevelLogsCap + len(subLevelLogDisplay); capEntriesIndexStart > subLevelLogStartIndex {
subLevelLogStartIndex = capEntriesIndexStart
}
subLevelLogDisplay = append(subLevelLogDisplay, m.subLevelLogs[subLevelLogStartIndex:]...)
for _, logLine := range subLevelLogDisplay {
line := logLine.String()
if logLine.err != nil {
line = errorLogStyle.Render(line)
}
builder.WriteString(subLevelLogStyle.Width(m.terminalWidth).Render(line) + "\n")
}
s := builder.String()
return appStyle.Width(m.terminalWidth).Render(s)
}
type logKeyPair struct {
key interface{}
value interface{}
}
type logEntry struct {
msg string
keyPairs []logKeyPair
refreshable bool
level int
success bool
err error
}
func newLogEntry(level int, msg string, keysAndValues ...interface{}) logEntry {
keysAndValues = keysAndValues[:len(keysAndValues)/2*2]
res := logEntry{
msg: msg,
level: level,
}
for i := 0; i < len(keysAndValues); i += 2 {
key := keysAndValues[i]
value := keysAndValues[i+1]
if key == "refreshable" {
if refreshable, ok := value.(bool); ok {
res.refreshable = refreshable
}
continue
}
if key == "success" {
if success, ok := value.(bool); ok {
res.success = success
}
continue
}
res.keyPairs = append(res.keyPairs, logKeyPair{
key: key,
value: value,
})
}
return res
}
func (e logEntry) keyPairsString() string {
builder := strings.Builder{}
for _, kp := range e.keyPairs {
builder.Write([]byte(fmt.Sprintf("%v=%v ", kp.key, kp.value)))
}
return strings.TrimSpace(builder.String())
}
func (e logEntry) String() string {
s := fmt.Sprintf("%s %s\n", e.msg, e.keyPairsString())
if e.err != nil {
s = fmt.Sprintf("%s: %s %s\n", e.msg, e.err, e.keyPairsString())
}
return strings.TrimSpace(s)
}
type TeaLogSink struct {
logs chan logEntry
prefixes []string
teaShutdown chan error
levelLimit int
}
var _ logr.LogSink = &TeaLogSink{}
type LoggerOptions struct {
OutputPath string
}
func NewLogger(opts LoggerOptions) (logr.Logger, error) {
s := spinner.New(spinner.WithSpinner(spinner.Dot))
s.Style = spinnerStyle
lipgloss.SetColorProfile(termenv.ANSI)
var logFile *os.File
if opts.OutputPath != "" {
var err error
logFile, err = tea.LogToFile(opts.OutputPath, "")
if err != nil {
return logr.Logger{}, err
}
}
m := model{
logs: make(chan logEntry),
spinner: s,
subLevelLogsCap: 10,
// Arbitrary initlal terminal width
terminalWidth: 80,
logFile: logFile,
}
l := &TeaLogSink{
logs: m.logs,
teaShutdown: make(chan error),
levelLimit: 2 + options.Verbosity,
}
p := tea.NewProgram(m, tea.WithoutCatchPanics(), tea.WithoutSignalHandler(), tea.WithInput(nil))
go func() {
_, err := p.Run()
l.teaShutdown <- err
}()
return logr.New(l), nil
}
func (l *TeaLogSink) Close() error {
close(l.logs)
err := <-l.teaShutdown
return err
}
// Enabled implements logr.LogSink.
func (l *TeaLogSink) Enabled(level int) bool {
return l.levelLimit > level
}
// Error implements logr.LogSink.
func (l *TeaLogSink) Error(err error, msg string, keysAndValues ...interface{}) {
if prefix := l.prefix(); prefix != "" {
msg = prefix + " " + msg
}
entry := newLogEntry(0, msg, keysAndValues...)
entry.err = err
l.logs <- entry
}
// Info implements logr.LogSink.
func (l *TeaLogSink) Info(level int, msg string, keysAndValues ...interface{}) {
if prefix := l.prefix(); prefix != "" {
msg = prefix + " " + msg
}
entry := newLogEntry(level, msg, keysAndValues...)
l.logs <- entry
}
// Init implements logr.LogSink.
func (*TeaLogSink) Init(info logr.RuntimeInfo) {
// Not important
}
// WithName implements logr.LogSink.
func (l *TeaLogSink) WithName(name string) logr.LogSink {
newLogger := &TeaLogSink{
prefixes: l.prefixes,
logs: l.logs,
levelLimit: l.levelLimit,
}
newLogger.prefixes = append(newLogger.prefixes, name)
return newLogger
}
// WithValues implements logr.LogSink.
func (l *TeaLogSink) WithValues(keysAndValues ...interface{}) logr.LogSink {
newLogger := &TeaLogSink{
prefixes: l.prefixes,
logs: l.logs,
levelLimit: l.levelLimit,
}
for i, key := range keysAndValues {
if vIndex := i + 1; vIndex < len(keysAndValues) {
newLogger.prefixes = append(newLogger.prefixes, fmt.Sprintf("%v=%v", key, keysAndValues[vIndex]))
}
}
return newLogger
}
func (l *TeaLogSink) prefix() string {
builder := strings.Builder{}
for _, p := range l.prefixes {
builder.WriteString(fmt.Sprintf("[%s]", p))
}
return builder.String()
}
package main
func main() {
Execute()
}
package options
import "github.com/spf13/cobra"
var Verbosity int
const maxVerbosity = 5
func Add(cmd *cobra.Command) {
cmd.PersistentFlags().CountVar(&Verbosity, "v", "number for the log level verbosity")
if Verbosity > maxVerbosity {
Verbosity = maxVerbosity
}
}
package main
import (
"os"
"github.com/go-logr/logr"
"github.com/metalkast/metalkast/cmd/kast/options"
"github.com/spf13/cobra"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "kast",
Short: "Quickly provision Kubernetes bare metal clusters",
Long: `Kast is a tool for quickly provisioning Kubernetes bare metal clusters`,
SilenceUsage: true,
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
err := rootCmd.Execute()
if err != nil {
os.Exit(1)
}
}
func init() {
initLoggers()
options.Add(rootCmd)
}
func initLoggers() {
// Inspired from https://github.com/fluxcd/flux2/pull/3932
ctrllog.SetLogger(logr.New(ctrllog.NullLogSink{}))
}
package main
import (
"os"
"path"
)
func CreateRunDirectory() (string, error) {
wd, err := os.Getwd()
if err != nil {
return "", err
}
dir, err := os.MkdirTemp(wd, "metalkast-bootstrap-*")
if err != nil {
return "", err
}
if err := os.WriteFile(path.Join(dir, ".gitignore"), []byte("*\n"), 0644); err != nil {
return "", err
}
return dir, nil
}
package fake
import (
"context"
expect "github.com/Netflix/go-expect"
"github.com/metalkast/metalkast/pkg/bmc"
)
type FakeIpmiTool struct {
c *expect.Console
}
func NewFakeIpmiTool(c *expect.Console) FakeIpmiTool {
return FakeIpmiTool{
c: c,
}
}
var _ bmc.IpmiSolClient = &FakeIpmiTool{}
func (t *FakeIpmiTool) Run(ctx context.Context, f func(c *expect.Console) error) error {
return f(t.c)
}
package bmc
import (
"context"
"fmt"
"net/url"
"os/exec"
"strings"
"time"
"github.com/Netflix/go-expect"
"github.com/go-logr/logr"
kastlogr "github.com/metalkast/metalkast/pkg/logr"
"golang.org/x/sync/errgroup"
"k8s.io/apimachinery/pkg/util/wait"
)
type IpmiSolClient interface {
Run(context.Context, func(c *expect.Console) error) error
}
type ipmiTool struct {
logger logr.Logger
ipmiAddress string
ipmiUsername string
ipmiPassword string
}
var _ IpmiSolClient = &ipmiTool{}
func (t *ipmiTool) Run(ctx context.Context, f func(c *expect.Console) error) error {
c, err := expect.NewConsole(expect.WithStdout(kastlogr.NewLogWriter(t.logger)), expect.WithDefaultTimeout(10*time.Second))
if err != nil {
return fmt.Errorf("failed to configure console: %w", err)
}
activateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
out, err := exec.CommandContext(activateCtx, "ipmitool", "-I", "lanplus", "-H", t.ipmiAddress, "-U", t.ipmiUsername, "-P", t.ipmiPassword, "sol", "deactivate").CombinedOutput()
cancel()
if err != nil && !strings.Contains(string(out), "already de-activated") {
t.logger.Error(err, "failed to deactivate previous IPMI SOL Session")
}
g, ctx := errgroup.WithContext(ctx)
ctx, cancel = context.WithCancel(ctx)
cmd := exec.CommandContext(ctx, "ipmitool", "-I", "lanplus", "-H", t.ipmiAddress, "-U", t.ipmiUsername, "-P", t.ipmiPassword, "sol", "activate")
cmd.Stdin = c.Tty()
cmd.Stdout = c.Tty()
cmd.Stderr = c.Tty()
err = cmd.Start()
if err != nil {
cancel()
return fmt.Errorf("failed to start IPMI SOL Session: %w", err)
}
g.Go(func() error {
return cmd.Wait()
})
if _, err := c.ExpectString("SOL Session operational"); err != nil {
cancel()
return fmt.Errorf("could not establish SOL Session")
}
err = wait.PollUntilContextTimeout(ctx, time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
if _, err := c.SendLine("\004"); err != nil {
return false, nil
}
if _, err := c.Expect(expect.String("login:"), expect.WithTimeout(time.Second*5)); err != nil {
return false, nil
}
return true, nil
})
if err != nil {
cancel()
g.Wait()
return fmt.Errorf("timed-out waiting for console")
}
g.Go(func() error {
return f(c)
})
defer cancel()
return g.Wait()
}
func newIpmiTool(ipmiAddress, ipmiUsername, ipmiPassword string, logger logr.Logger) (*ipmiTool, error) {
return &ipmiTool{
logger: logger,
ipmiAddress: ipmiAddress,
ipmiUsername: ipmiUsername,
ipmiPassword: ipmiPassword,
}, nil
}
type BMC struct {
IpmiClient IpmiSolClient
RedfishClient *RedFish
}
func NewBMC(redfishUrl, username, password string, logger logr.Logger) (*BMC, error) {
redfishUrlParsed, err := url.Parse(redfishUrl)
if err != nil {
return nil, fmt.Errorf("failed to get ipmi host from redfish url: %w", err)
}
ipmiClient, err := newIpmiTool(redfishUrlParsed.Host, username, password, logger.WithName("ipmi console"))
if err != nil {
return nil, fmt.Errorf("failed to initialize IPMI client: %w", err)
}
redfishClient, err := NewRedFish(redfishUrl, username, password)
if err != nil {
return nil, fmt.Errorf("failed to initialize Redfish client: %w", err)
}
return &BMC{
IpmiClient: ipmiClient,
RedfishClient: redfishClient,
}, nil
}
package bmc
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/stmcginnis/gofish"
"github.com/stmcginnis/gofish/redfish"
"k8s.io/apimachinery/pkg/util/wait"
)
func getVirtualMediaCD(vms []*redfish.VirtualMedia) (*redfish.VirtualMedia, error) {
var vMedia *redfish.VirtualMedia
for _, vm := range vms {
if strings.ToLower(vm.ID) == "cd" {
vMedia = vm
break
}
}
if vMedia == nil {
return nil, fmt.Errorf("CD virtual media not found")
}
return vMedia, nil
}
type RedFish struct {
system *redfish.ComputerSystem
cd *redfish.VirtualMedia
config *gofish.ClientConfig
client *gofish.APIClient
}
func (rf *RedFish) Boot() error {
if err := rf.initSystem(); err != nil {
return fmt.Errorf("failed to init system: %w", err)
}
if rf.system.PowerState == redfish.OffPowerState {
if err := rf.system.Reset(redfish.OnResetType); err != nil {
return fmt.Errorf("failed to boot system: %v", err)
}
} else {
if err := rf.system.Reset(redfish.ForceRestartResetType); err != nil {
return fmt.Errorf("failed to boot system: %v", err)
}
}
return nil
}
type dellTask struct {
Oem struct {
Dell struct {
JobState string `json:"JobState"`
} `json:"Dell"`
} `json:"Oem"`
}
func (rf *RedFish) SetBootMedia() error {
if err := rf.initSystem(); err != nil {
return fmt.Errorf("failed to init system: %w", err)
}
switch rf.system.Manufacturer {
case "Dell Inc.":
// https://github.com/dell/iDRAC-Redfish-Scripting/blob/7f1836308754d0e9d9fb98ec6ce7e7afff10b487/Redfish%20Python/SetNextOneTimeBootVirtualMediaDeviceOemREDFISH.py#L68
payload := map[string]interface{}{
"ShareParameters": map[string]interface{}{
"Target": "ALL",
},
"ImportBuffer": "<SystemConfiguration><Component FQDD=\"iDRAC.Embedded.1\"><Attribute Name=\"ServerBoot.1#BootOnce\">Enabled</Attribute><Attribute Name=\"ServerBoot.1#FirstBootDevice\">VCD-DVD</Attribute></Component></SystemConfiguration>",
}
// https://github.com/dell/iDRAC-Redfish-Scripting/blob/7f1836308754d0e9d9fb98ec6ce7e7afff10b487/Redfish%20Python/SetNextOneTimeBootVirtualMediaDeviceOemREDFISH.py#L66
resp, err := rf.client.Post("/redfish/v1/Managers/iDRAC.Embedded.1/Actions/Oem/EID_674_Manager.ImportSystemConfiguration", payload)
if err != nil {
return fmt.Errorf("failed to set boot media: %w", err)
}
task_uri := resp.Header.Get("Location")
err = wait.PollUntilContextTimeout(context.TODO(), time.Second, time.Minute, true, func(ctx context.Context) (bool, error) {
resp, err := rf.client.Get(task_uri)
if err != nil {
return false, err
}
decoder := json.NewDecoder(resp.Body)
task := &dellTask{}
if err = decoder.Decode(&task); err != nil {
return false, err
}
return task.Oem.Dell.JobState == "Completed", nil
})
if err != nil {
return err
}
default:
boot := redfish.Boot{
BootSourceOverrideTarget: redfish.CdBootSourceOverrideTarget,
}
if err := rf.system.SetBoot(boot); err != nil {
return fmt.Errorf("failed to set boot media: %v", err)
}
}
return nil
}
func (rf *RedFish) InsertMedia(url string) error {
if err := rf.initCD(); err != nil {
return err
}
if rf.cd.Inserted {
if err := rf.cd.EjectMedia(); err != nil {
return fmt.Errorf("failed to eject existing inserted media: %w", err)
}
}
err := rf.cd.InsertMediaConfig(redfish.VirtualMediaConfig{
Image: url,
Inserted: true,
})
if err != nil {
return fmt.Errorf("failed to insert media: %v", err)
}
return nil
}
func (rf *RedFish) initClient() error {
if rf.client != nil {
return nil
}
var err error
if rf.client, err = gofish.Connect(*rf.config); err != nil {
return fmt.Errorf("failed to initialize the RedFish client: %v", err)
}
return nil
}
func (rf *RedFish) initConfig(url, username, password string) {
rf.config = &gofish.ClientConfig{
Endpoint: url,
// TODO: (GAL-311) Parametrize
Insecure: true,
Username: username,
Password: password,
BasicAuth: true,
}
}
func (rf *RedFish) initSystem() error {
if rf.system != nil {
return nil
}
if rf.client == nil {
return fmt.Errorf("client not initialized")
}
systems, err := rf.client.Service.Systems()
if err != nil {
return fmt.Errorf("failed to get the RedFish systems: %v", err)
}
if len(systems) == 0 {
return fmt.Errorf("no systems found")
}
rf.system = systems[0]
return nil
}
func (rf *RedFish) initCD() error {
if rf.cd != nil {
return nil
}
if err := rf.initSystem(); err != nil {
return fmt.Errorf("failed to init system: %w", err)
}
managerNames := rf.system.ManagedBy
if len(managerNames) != 1 {
return fmt.Errorf("only 1 manager is expected for each system")
}
managers, err := rf.client.Service.Managers()
if err != nil {
return fmt.Errorf("failed to get the RedFish managers: %v", err)
}
var manager *redfish.Manager
for _, m := range managers {
if m.ODataID == managerNames[0] {
manager = m
break
}
}
if manager == nil {
return fmt.Errorf("manager for the system %s not found", rf.system.Name)
}
vMedia, err := manager.VirtualMedia()
if err != nil {
return fmt.Errorf("failed to get the RedFish virtual media: %v", err)
}
if rf.cd, err = getVirtualMediaCD(vMedia); err != nil {
return fmt.Errorf("failed to get the RedFish virtual media CD: %v", err)
}
return nil
}
func (rf *RedFish) Close() {
rf.client.Logout()
}
func NewRedFish(url, username, password string) (*RedFish, error) {
rf := &RedFish{}
rf.initConfig(url, username, password)
if err := rf.initClient(); err != nil {
return nil, err
}
return rf, nil
}
package cluster
import (
"context"
"fmt"
"strings"
"time"
"github.com/go-logr/logr"
"github.com/manifestival/manifestival"
"github.com/metalkast/metalkast/pkg/kustomize"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
)
type Applier struct {
client manifestival.Client
logger logr.Logger
}
func NewApplier(client manifestival.Client, logger logr.Logger) *Applier {
return &Applier{
client: client,
logger: logger,
}
}
func (a *Applier) ApplyManifest(manifest manifestival.Manifest) error {
manifestCount := len(manifest.Resources())
successCount := 0
applyList := manifest.Resources()
if err := wait.PollUntilContextTimeout(context.TODO(), time.Second*1, time.Minute*10, true, func(ctx context.Context) (bool, error) {
retryList := []unstructured.Unstructured{}
for _, r := range applyList {
m, err := manifestival.ManifestFrom(
manifestival.Slice([]unstructured.Unstructured{r}),
manifestival.UseClient(a.client),
)
if err != nil {
return false, err
}
a.logger.Info(fmt.Sprintf(
"[%d/%d] Applying manifest %s",
successCount+1,
manifestCount,
strings.TrimPrefix(types.NamespacedName{Name: r.GetName(), Namespace: r.GetNamespace()}.String(), "/"),
), "refreshable", true)
if err = m.Apply(); err != nil {
a.logger.Error(err, fmt.Sprintf("failed to apply manifest %s, will retry later...", strings.TrimPrefix(types.NamespacedName{Name: r.GetName(), Namespace: r.GetNamespace()}.String(), "/")))
retryList = append(retryList, r)
} else {
successCount++
}
}
applyList = retryList
return len(applyList) == 0, nil
}); err != nil {
return fmt.Errorf("failed to apply all manifests: %w", err)
}
a.logger.Info(fmt.Sprintf("Applied all %d manifests", manifestCount), "refreshable", true)
return nil
}
func (a *Applier) Apply(manifests string) error {
m, err := manifestival.ManifestFrom(manifestival.Reader(strings.NewReader(manifests)))
if err != nil {
return fmt.Errorf("failed to instantiate manifests: %w", err)
}
return a.ApplyManifest(m)
}
func (a *Applier) ApplyKustomize(path string) error {
manifests, err := kustomize.Build(path)
if err != nil {
return err
}
if err = a.Apply(string(manifests)); err != nil {
return fmt.Errorf("failed to apply kustomize layer (%s): %w", path, err)
}
return nil
}
package cluster
import (
"context"
"fmt"
"os"
"github.com/go-logr/logr"
"github.com/hashicorp/go-retryablehttp"
mfc "github.com/manifestival/controller-runtime-client"
bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
kastlogr "github.com/metalkast/metalkast/pkg/logr"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func init() {
utilruntime.Must(bmov1alpha1.AddToScheme(scheme.Scheme))
utilruntime.Must(clusterapiv1beta1.AddToScheme(scheme.Scheme))
}
type Cluster struct {
kubeCfgPath string
*Applier
client.Client
logger logr.Logger
}
func NewCluster(kubeCfgData []byte, kubeCfgDest string, logger logr.Logger) (*Cluster, error) {
if err := os.WriteFile(kubeCfgDest, kubeCfgData, 0644); err != nil {
return nil, fmt.Errorf("failed to write cluster kubeconfig (to destination path %v): %w", kubeCfgDest, err)
}
config, err := clientcmd.BuildConfigFromFlags("", kubeCfgDest)
if err != nil {
return nil, fmt.Errorf("failed to initialize cluster config: %w", err)
}
kubeClient, err := rest.HTTPClientFor(config)
if err != nil {
return nil, fmt.Errorf("failed to create cluster client: %w", err)
}
retryClient := retryablehttp.NewClient()
retryClient.Logger = kastlogr.NewFromLogger(logger.V(1).WithName("retry-client"))
retryClient.HTTPClient = kubeClient
kubeControllerClient, err := client.New(config, client.Options{
HTTPClient: retryClient.StandardClient(),
})
if err != nil {
return nil, fmt.Errorf("failed to create cluster client: %w", err)
}
mc := mfc.NewClient(kubeControllerClient)
return &Cluster{
kubeCfgPath: kubeCfgDest,
Applier: NewApplier(mc, logger),
Client: kubeControllerClient,
logger: logger,
}, nil
}
func (c Cluster) ApplyPaths(paths ...string) error {
for _, p := range paths {
if err := c.Applier.ApplyKustomize(p); err != nil {
return fmt.Errorf("failed to deploy %s: %s", p, err)
}
}
return nil
}
func (c *Cluster) Move(target *Cluster, namespace string) error {
clusterctlClient, err := clusterctlclient.New(context.TODO(), "")
if err != nil {
return fmt.Errorf("failed to init Cluster API client: %w", err)
}
err = clusterctlClient.Move(
context.TODO(),
clusterctlclient.MoveOptions{
FromKubeconfig: clusterctlclient.Kubeconfig{Path: c.kubeCfgPath},
ToKubeconfig: clusterctlclient.Kubeconfig{Path: target.kubeCfgPath},
Namespace: namespace,
})
if err != nil {
return fmt.Errorf("failed to move Cluster API objects: %w", err)
}
return nil
}
func (c *Cluster) KubeCfgPath() string {
return c.kubeCfgPath
}
package kustomize
import (
"fmt"
"github.com/getsops/sops/v3/cmd/sops/formats"
"github.com/getsops/sops/v3/decrypt"
"github.com/spf13/cobra"
"sigs.k8s.io/kustomize/api/krusty"
"sigs.k8s.io/kustomize/kustomize/v5/commands/build"
"sigs.k8s.io/kustomize/kyaml/filesys"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
func Build(path string) ([]byte, error) {
options := build.HonorKustomizeFlags(krusty.MakeDefaultOptions(), (&cobra.Command{}).Flags())
options.PluginConfig.HelmConfig.Enabled = true
options.PluginConfig.HelmConfig.Command = "helm"
k := krusty.MakeKustomizer(
options,
)
if path == "" {
path = "."
}
m, err := k.Run(sopsDecryptingFs{
filesys.MakeFsOnDisk(),
}, path)
if err != nil {
return nil, fmt.Errorf("failed to build kustomize layer (%s): %w", path, err)
}
yamlManifests, err := m.AsYaml()
if err != nil {
return nil, fmt.Errorf("failed converting kustomize build (%s) to yaml: %w", path, err)
}
return yamlManifests, nil
}
var _ filesys.FileSystem = sopsDecryptingFs{}
type sopsDecryptingFs struct {
filesys.FileSystem
}
// ReadFile implements filesys.FileSystem.
func (fs sopsDecryptingFs) ReadFile(path string) ([]byte, error) {
data, err := fs.FileSystem.ReadFile(path)
if err != nil {
return nil, err
}
rNode, err := yaml.Parse(string(data))
if err != nil {
return data, nil
}
rNodeMap, err := rNode.Map()
if err != nil {
// Yaml might successfully parse some files as string or similar
// if it cannot be converted to map (i.e. is not nested), it's not sops encrypted YAML
return data, nil
}
if _, isSopsEncrypted := rNodeMap["sops"]; isSopsEncrypted {
decrypted, err := decrypt.DataWithFormat(data, formats.Yaml)
if err != nil {
return nil, fmt.Errorf("failed to decrypt file (%v): %w", path, err)
}
return decrypted, nil
}
return data, nil
}
package logr
import (
"bytes"
"io"
"regexp"
"strings"
"github.com/go-logr/logr"
)
var _ io.Writer = &LogWriter{}
type LogWriter struct {
log logr.Logger
buffer bytes.Buffer
lineBuffer bytes.Buffer
}
func NewLogWriter(log logr.Logger) *LogWriter {
return &LogWriter{
log: log,
}
}
// Write implements io.Writer.
func (w *LogWriter) Write(p []byte) (n int, err error) {
n, err = w.buffer.Write(p)
if err != nil {
return
}
for {
b, err := w.buffer.ReadBytes(byte('\n'))
if _, err := w.lineBuffer.Write(b); err != nil {
return n, err
}
if err != nil {
return n, nil
}
writeLine := strings.TrimSpace(StripAnsi(w.lineBuffer.String()))
w.log.Info(writeLine)
w.lineBuffer.Reset()
}
}
const ansi = "[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))"
var re = regexp.MustCompile(ansi)
func StripAnsi(str string) string {
return re.ReplaceAllString(str, "")
}
package logr
import (
"github.com/go-logr/logr"
"github.com/hashicorp/go-retryablehttp"
)
var _ retryablehttp.LeveledLogger = &RetryableLogger{}
type RetryableLogger struct {
log logr.Logger
}
// Debug implements retryablehttp.LeveledLogger.
func (l *RetryableLogger) Debug(msg string, keysAndValues ...interface{}) {
l.log.V(4).Info(msg, keysAndValues...)
}
// Error implements retryablehttp.LeveledLogger.
func (l *RetryableLogger) Error(msg string, keysAndValues ...interface{}) {
var err error
for i, k := range keysAndValues {
if k == "error" && len(keysAndValues) > i+1 {
err, _ = keysAndValues[i+1].(error)
}
}
l.log.Error(err, msg, keysAndValues...)
}
// Info implements retryablehttp.LeveledLogger.
func (l *RetryableLogger) Info(msg string, keysAndValues ...interface{}) {
l.log.Info(msg, keysAndValues...)
}
// Warn implements retryablehttp.LeveledLogger.
func (l *RetryableLogger) Warn(msg string, keysAndValues ...interface{}) {
l.log.V(1).Info(msg, keysAndValues...)
}
func NewFromLogger(log logr.Logger) *RetryableLogger {
return &RetryableLogger{
log: log,
}
}
package testutil
import (
"testing"
"github.com/manifestival/manifestival"
"github.com/stretchr/testify/assert"
)
func TestManifests(t *testing.T, source manifestival.Source) manifestival.Manifest {
m, err := manifestival.ManifestFrom(source)
assert.NoError(t, err)
return m
}