From 340cb8e895a088dc1117e85e78b692153cc5f2f5 Mon Sep 17 00:00:00 2001 From: John Bowdre Date: Sat, 10 Dec 2022 17:02:12 -0600 Subject: [PATCH 1/5] script cleanup, shift to using dynamically-generated SSH keys for Packer --- packer/data/user-data.pkrtpl.hcl | 6 ++++-- .../ssh_private_key_packer.example.pem | 7 ------- packer/scripts/generalize.sh | 3 +++ packer/ubuntu-k8s.example.pkrvars.hcl | 7 +++++-- packer/ubuntu-k8s.pkr.hcl | 18 ++++++++--------- packer/variables.pkr.hcl | 20 ++++--------------- 6 files changed, 25 insertions(+), 36 deletions(-) delete mode 100644 packer/packer_cache/ssh_private_key_packer.example.pem diff --git a/packer/data/user-data.pkrtpl.hcl b/packer/data/user-data.pkrtpl.hcl index b41f336..6306582 100644 --- a/packer/data/user-data.pkrtpl.hcl +++ b/packer/data/user-data.pkrtpl.hcl @@ -183,7 +183,9 @@ autoinstall: lock-passwd: false sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash -%{ if length( build_key ) > 0 ~} +%{ if length( ssh_keys ) > 0 ~} ssh_authorized_keys: - - ${ build_key } +%{ for ssh_key in ssh_keys ~} + - ${ ssh_key } +%{ endfor ~} %{ endif ~} diff --git a/packer/packer_cache/ssh_private_key_packer.example.pem b/packer/packer_cache/ssh_private_key_packer.example.pem deleted file mode 100644 index de9806a..0000000 --- a/packer/packer_cache/ssh_private_key_packer.example.pem +++ /dev/null @@ -1,7 +0,0 @@ ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW -QyNTUxOQAAACDqS76cYpT46QmoQEMbOEUIL/i2+cTm1C17cBKIphAaQAAAAJC/ririv64q -4gAAAAtzc2gtZWQyNTUxOQAAACDqS76cYpT46QmoQEMbOEUIL/i2+cTm1C17cBKIphAaQA -AAAECBctr1BYu+QL8D8IFHQ8uN/Us4X9xkj9HYf1hQjqrXCupLvpxilPjpCahAQxs4RQgv -+Lb5xObULXtwEoimEBpAAAAADGpvaG5AcGVuZ3VpbgE= ------END OPENSSH PRIVATE KEY----- diff --git a/packer/scripts/generalize.sh b/packer/scripts/generalize.sh index 9be2b3a..a0e6862 100644 --- a/packer/scripts/generalize.sh +++ b/packer/scripts/generalize.sh @@ -30,6 +30,9 @@ sudo rm -rf /var/tmp/* echo '>> Clearing host keys...' sudo rm -f /etc/ssh/ssh_host_* +echo '>> Removing Packer SSH key...' +sed -i '/packer_key/d' ~/.ssh/authorized_keys + echo '>> Clearing machine-id...' sudo truncate -s 0 /etc/machine-id if [ -f /var/lib/dbus/machine-id ]; then diff --git a/packer/ubuntu-k8s.example.pkrvars.hcl b/packer/ubuntu-k8s.example.pkrvars.hcl index d16b7a0..8c57b45 100644 --- a/packer/ubuntu-k8s.example.pkrvars.hcl +++ b/packer/ubuntu-k8s.example.pkrvars.hcl @@ -77,10 +77,13 @@ communicator_port = 22 communicator_timeout = "20m" common_ip_wait_timeout = "20m" common_shutdown_timeout = "15m" -build_remove_keys = false +vm_shutdown_command = "sudo /usr/sbin/shutdown -P now" +build_remove_keys = true build_username = "admin" build_password = "VMware1!" -build_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOpLvpxilPjpCahAQxs4RQgv+Lb5xObULXtwEoimEBpA builder" +ssh_keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOpLvpxilPjpCahAQxs4RQgv+Lb5xObULXtwEoimEBpA builder" +] // Provisioner Settings post_install_scripts = [ diff --git a/packer/ubuntu-k8s.pkr.hcl b/packer/ubuntu-k8s.pkr.hcl index a9023f8..8e908af 100644 --- a/packer/ubuntu-k8s.pkr.hcl +++ b/packer/ubuntu-k8s.pkr.hcl @@ -5,7 +5,6 @@ // BLOCK: packer // The Packer configuration. - packer { required_version = ">= 1.8.2" required_plugins { @@ -20,18 +19,21 @@ packer { } } -// BLOCK: locals -// Defines the local variables. +// BLOCK: data +// Defines data sources. data "sshkey" "install" { + type = "ed25519" + name = "packer_key" } +// BLOCK: locals +// Defines local variables. locals { ssh_public_key = data.sshkey.install.public_key ssh_private_key_file = data.sshkey.install.private_key_path build_tool = "HashiCorp Packer ${packer.version}" build_date = formatdate("YYYY-MM-DD hh:mm ZZZ", timestamp()) build_description = "Kubernetes Ubuntu 20.04 Node template\nBuild date: ${local.build_date}\nBuild tool: ${local.build_tool}" - shutdown_command = "sudo -S -E shutdown -P now" iso_paths = ["[${var.common_iso_datastore}] ${var.iso_path}/${var.iso_file}"] iso_checksum = "${var.iso_checksum_type}:${var.iso_checksum_value}" data_source_content = { @@ -39,7 +41,7 @@ locals { "/user-data" = templatefile("data/user-data.pkrtpl.hcl", { build_username = var.build_username build_password = bcrypt(var.build_password) - build_key = var.build_key + ssh_keys = concat([local.ssh_public_key], var.ssh_keys) vm_guest_os_language = var.vm_guest_os_language vm_guest_os_keyboard = var.vm_guest_os_keyboard vm_guest_os_timezone = var.vm_guest_os_timezone @@ -52,7 +54,6 @@ locals { // BLOCK: source // Defines the builder configuration blocks. - source "vsphere-iso" "ubuntu-k8s" { // vCenter Server Endpoint Settings and Credentials @@ -106,13 +107,12 @@ source "vsphere-iso" "ubuntu-k8s" { boot_wait = var.vm_boot_wait boot_command = var.vm_boot_command ip_wait_timeout = var.common_ip_wait_timeout - shutdown_command = local.shutdown_command + shutdown_command = var.vm_shutdown_command shutdown_timeout = var.common_shutdown_timeout // Communicator Settings and Credentials communicator = "ssh" ssh_username = var.build_username - ssh_password = var.build_password ssh_private_key_file = local.ssh_private_key_file ssh_clear_authorized_keys = var.build_remove_keys ssh_port = var.communicator_port @@ -151,7 +151,6 @@ source "vsphere-iso" "ubuntu-k8s" { // BLOCK: build // Defines the builders to run, provisioners, and post-processors. - build { sources = [ "source.vsphere-iso.ubuntu-k8s" @@ -173,6 +172,7 @@ build { provisioner "shell" { execute_command = "bash {{ .Path }}" + expect_disconnect = true scripts = var.pre_final_scripts } } \ No newline at end of file diff --git a/packer/variables.pkr.hcl b/packer/variables.pkr.hcl index 96bdc5c..673bf82 100644 --- a/packer/variables.pkr.hcl +++ b/packer/variables.pkr.hcl @@ -7,7 +7,6 @@ // Defines the input variables. // vSphere Credentials - variable "vsphere_endpoint" { type = string description = "The fully qualified domain name or IP address of the vCenter Server instance. ('vcenter.lab.local')" @@ -32,7 +31,6 @@ variable "vsphere_insecure_connection" { } // vSphere Settings - variable "vsphere_datacenter" { type = string description = "The name of the target vSphere datacenter. ('Lab Datacenter')" @@ -59,7 +57,6 @@ variable "vsphere_folder" { } // Virtual Machine Settings - variable "vm_name" { type = string description = "Name of the new VM to create." @@ -175,7 +172,6 @@ variable "common_remove_cdrom" { } // Template and Content Library Settings - variable "common_template_conversion" { type = bool description = "Convert the virtual machine to template. Must be 'false' for content library." @@ -207,7 +203,6 @@ variable "common_content_library_skip_export" { } // Snapshot Settings - variable "common_snapshot_creation" { type = bool description = "Create a snapshot for Linked Clones." @@ -221,7 +216,6 @@ variable "common_snapshot_name" { } // OVF Export Settings - variable "common_ovf_export_enabled" { type = bool description = "Enable OVF artifact export." @@ -240,7 +234,6 @@ variable "common_ovf_export_path" { } // Removable Media Settings - variable "common_iso_datastore" { type = string description = "The name of the source vSphere datastore for ISO images. ('datastore-iso-01')" @@ -278,7 +271,6 @@ variable "cd_label" { } // Boot Settings - variable "vm_boot_order" { type = string description = "The boot order for virtual machines devices. ('disk,cdrom')" @@ -313,11 +305,9 @@ variable "common_shutdown_timeout" { } // Communicator Settings and Credentials - variable "build_username" { type = string description = "The username to login to the guest operating system. ('admin')" - sensitive = true } variable "build_password" { @@ -333,10 +323,11 @@ variable "build_password_encrypted" { default = null } -variable "build_key" { - type = string - description = "The public key to login to the guest operating system." +variable "ssh_keys" { + type = list(string) + description = "List of public keys to be added to ~/.ssh/authorized_keys." sensitive = true + default = [] } variable "build_remove_keys" { @@ -346,7 +337,6 @@ variable "build_remove_keys" { } // Communicator Settings - variable "communicator_port" { type = string description = "The port for the communicator protocol." @@ -370,7 +360,6 @@ variable "communicator_ssl" { } // Provisioner Settings - variable "cloud_init_apt_packages" { type = list(string) description = "A list of apt packages to install during the subiquity cloud-init installer." @@ -396,7 +385,6 @@ variable "pre_final_scripts" { } // Kubernetes Settings - variable "k8s_version" { type = string description = "Kubernetes version to be installed. Latest stable is listed at https://dl.k8s.io/release/stable.txt" From 87b627ebcb939c3ecc6def3f79b3688471428ec1 Mon Sep 17 00:00:00 2001 From: John Bowdre Date: Tue, 13 Dec 2022 15:25:05 -0600 Subject: [PATCH 2/5] script cleanup --- packer/scripts/enable-vmware-customization.sh | 6 +++++- packer/scripts/persist-cloud-init-net.sh | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packer/scripts/enable-vmware-customization.sh b/packer/scripts/enable-vmware-customization.sh index 4b6f2c0..fba2c01 100644 --- a/packer/scripts/enable-vmware-customization.sh +++ b/packer/scripts/enable-vmware-customization.sh @@ -1,4 +1,8 @@ #!/bin/bash -eu echo '>> Enabling legacy VMware Guest Customization...' -echo 'disable_vmware_customization: true' | sudo tee -a /etc/cloud/cloud.cfg +if grep -q 'disable_vmware_customization' /etc/cloud/cloud.cfg; then + sudo sed -i 's/^disable_vmware_customization:.*$/disable_vmware_customization: True/' /etc/cloud/cloud.cfg +else + echo 'disable_vmware_customization: true' | sudo tee -a /etc/cloud/cloud.cfg +fi sudo vmware-toolbox-cmd config set deployPkg enable-custom-scripts true diff --git a/packer/scripts/persist-cloud-init-net.sh b/packer/scripts/persist-cloud-init-net.sh index 005a6ad..9a50d4b 100644 --- a/packer/scripts/persist-cloud-init-net.sh +++ b/packer/scripts/persist-cloud-init-net.sh @@ -1,3 +1,7 @@ #!/bin/sh -eu echo '>> Preserving network settings...' -echo 'manual_cache_clean: True' | sudo tee -a /etc/cloud/cloud.cfg \ No newline at end of file +if grep -q 'manual_cache_clean' /etc/cloud/cloud.cfg; then + sudo sed -i 's/^manual_cache_clean.*$/manual_cache_clean: True/' /etc/cloud/cloud.cfg +else + echo 'manual_cache_clean: True' | sudo tee -a /etc/cloud/cloud.cfg +fi \ No newline at end of file From 17a27d0570f79cb8caaac0228721bad12a8c6191 Mon Sep 17 00:00:00 2001 From: John Bowdre Date: Tue, 20 Dec 2022 08:26:06 -0600 Subject: [PATCH 3/5] update script logic --- terraform/scripts/initialize-controlplane.sh | 44 +++++++++++++------- terraform/scripts/join-workers.sh | 4 +- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/terraform/scripts/initialize-controlplane.sh b/terraform/scripts/initialize-controlplane.sh index 8dbeee8..a3451cb 100644 --- a/terraform/scripts/initialize-controlplane.sh +++ b/terraform/scripts/initialize-controlplane.sh @@ -1,18 +1,17 @@ #!/bin/bash -eu source ./env.txt -if [ ! -f /etc/kubernetes/manifests/kube-vip.yaml ]; then - echo ">> Configuring kube-vip..." - sudo ctr image pull ghcr.io/kube-vip/kube-vip:"${KUBEVIP_VER}" - sudo ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:"${KUBEVIP_VER}" vip /kube-vip manifest pod \ - --interface ens192 \ - --vip "${K8S_CONTROLPLANE_VIP}" \ - --controlplane \ - --arp \ - --leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml -fi - if [ "${HOSTNAME}" == "${K8S_INITIAL_NODE}" ]; then + if [ ! -f /etc/kubernetes/manifests/kube-vip.yaml ]; then + echo ">> Configuring kube-vip..." + sudo ctr image pull ghcr.io/kube-vip/kube-vip:"${KUBEVIP_VER}" + sudo ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:"${KUBEVIP_VER}" vip /kube-vip manifest pod \ + --interface ens192 \ + --vip "${K8S_CONTROLPLANE_VIP}" \ + --controlplane \ + --arp \ + --leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml + fi if ! kubectl get nodes 2>/dev/null; then echo ">> Bootstrapping first controlplane node..." cat << EOF > kubeadminit.yaml @@ -57,14 +56,14 @@ EOF echo ">> Creating discovery config..." kubectl -n kube-public get configmap cluster-info -o jsonpath='{.data.kubeconfig}' > discovery.yaml - sudo install -o root -g root -m 600 discovery.yaml /etc/kubernetes/discovery.yaml + sudo install -m 600 discovery.yaml /etc/kubernetes/discovery.yaml else echo ">> [ERROR] Cluster initialization unsuccessful on ${HOSTNAME}! <<" exit 1 fi fi echo ">> Waiting up to 10 minutes for all control-plane nodes to be Ready..." - python3 -m http.server & + python3 -m http.server 2>/dev/null & PROC_ID=$! attempts_max=60 attempt=0 @@ -388,8 +387,11 @@ else attempt=0 until [ -f /etc/kubernetes/discovery.yaml ]; do wget "http://${K8S_CONTROLPLANE_VIP}:8000/discovery.yaml" 2>/dev/null - sudo install -o root -g root -m 600 discovery.yaml /etc/kubernetes/discovery.yaml 2>/dev/null - if [ ! -f /etc/kubernetes/discovery.yaml ]; then + if ! sudo install -m 600 discovery.yaml /etc/kubernetes/discovery.yaml 2>/dev/null; then + if [ ${attempt} -eq ${attempts_max} ]; then + echo ">> [ERROR] Timeout waiting for discovery.yaml! <<" + exit 1 + fi attempt=$((attempt+1)) sleep 10 fi @@ -409,7 +411,17 @@ nodeRegistration: controlPlane: certificateKey: ${KUBEADM_CERTKEY} EOF - if sudo kubeadm join "${K8S_CONTROLPLANE_VIP}":6443 --config kubeadmjoin.yaml; then + if sudo kubeadm join "${K8S_CONTROLPLANE_VIP}:6443" --config kubeadmjoin.yaml; then + if [ ! -f /etc/kubernetes/manifests/kube-vip.yaml ]; then + echo ">> Configuring kube-vip..." + sudo ctr image pull ghcr.io/kube-vip/kube-vip:"${KUBEVIP_VER}" + sudo ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:"${KUBEVIP_VER}" vip /kube-vip manifest pod \ + --interface ens192 \ + --vip "${K8S_CONTROLPLANE_VIP}" \ + --controlplane \ + --arp \ + --leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml + fi echo ">> Node ${HOSTNAME} successfully initialized!" touch .k8s-node-success mkdir -p "${HOME}"/.kube diff --git a/terraform/scripts/join-workers.sh b/terraform/scripts/join-workers.sh index 8e10b7e..c776f86 100644 --- a/terraform/scripts/join-workers.sh +++ b/terraform/scripts/join-workers.sh @@ -17,7 +17,7 @@ echo ">> Continuing after $((attempt*10)) seconds." echo ">> Waiting up to 10 minutes for all control-plane nodes..." attempts_max=60 attempt=0 -until "$(wget http://${K8S_CONTROLPLANE_VIP}:8000/.k8s-controlplane-success)" 2>/dev/null; do +until wget "http://${K8S_CONTROLPLANE_VIP}:8000/.k8s-controlplane-success" 2>/dev/null; do if [ ${attempt} -eq ${attempts_max} ]; then echo ">> [ERROR] Timeout waiting for control-plane nodes! <<" exit 1 @@ -31,7 +31,7 @@ attempts_max=6 attempt=0 until [ -f /etc/kubernetes/discovery.yaml ]; do wget "http://${K8S_CONTROLPLANE_VIP}:8000/discovery.yaml" 2>/dev/null - sudo install -o root -g root -m 600 discovery.yaml /etc/kubernetes/discovery.yaml 2>/dev/null + sudo install -m 600 discovery.yaml /etc/kubernetes/discovery.yaml 2>/dev/null if [ ! -f /etc/kubernetes/discovery.yaml ]; then attempt=$((attempt+1)) sleep 10 From 8d228f9ba06d54f3af3b83983725520f3ff6f929 Mon Sep 17 00:00:00 2001 From: John Bowdre Date: Tue, 20 Dec 2022 08:26:48 -0600 Subject: [PATCH 4/5] wait for network timeout --- terraform/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/main.tf b/terraform/main.tf index 23579e0..267a727 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -79,6 +79,7 @@ resource "vsphere_virtual_machine" "control" { firmware = data.vsphere_virtual_machine.template.firmware hardware_version = data.vsphere_virtual_machine.template.hardware_version scsi_type = data.vsphere_virtual_machine.template.scsi_type + wait_for_guest_net_timeout = 10 extra_config = { "disk.EnableUUID" = "TRUE" @@ -163,6 +164,7 @@ resource "vsphere_virtual_machine" "worker" { firmware = data.vsphere_virtual_machine.template.firmware hardware_version = data.vsphere_virtual_machine.template.hardware_version scsi_type = data.vsphere_virtual_machine.template.scsi_type + wait_for_guest_net_timeout = 10 network_interface { network_id = data.vsphere_network.network.id From 2d14363967050033fbf067f718c4e2d076719411 Mon Sep 17 00:00:00 2001 From: John Bowdre Date: Tue, 20 Dec 2022 08:27:12 -0600 Subject: [PATCH 5/5] bump VM specs --- terraform/terraform.example.tfvars | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/terraform.example.tfvars b/terraform/terraform.example.tfvars index 0711f6a..0c9a2a1 100644 --- a/terraform/terraform.example.tfvars +++ b/terraform/terraform.example.tfvars @@ -20,15 +20,15 @@ vm-domain = "lab.local" # Control plane specs vm-control-count = "3" vm-control-cpu = "2" -vm-control-ram = "2048" +vm-control-ram = "4096" vm-control-disk-size = "30" vm-control-ip-address-start = "60" vm-control-name = "k8s-control" # Worker specs vm-worker-count = "3" -vm-worker-cpu = "1" -vm-worker-ram = "1024" +vm-worker-cpu = "4" +vm-worker-ram = "8192" vm-worker-disk-size = "30" vm-worker-ip-address-start = "64" vm-worker-name = "k8s-worker"