Home Lab 2
Now that the base servers are ready for an OS, I need to setup the Network stack for my lab. While I wanted to stay inexpensive, I also wanted everything to be rack-mounted so I ended up buying a Cisco 3750X from Ebay.
I slotted in the two fans, and PSU and bolted on the front-mounting bracket.
And finally bolted it into the rack:
Initial Switch Configuration
First, I reset the switch by following This guide. Then I connected via USB Console and preformed initial setup of the switch which the boot CLI walks you through. My next step is to setup a management interface so I can access the switch over telnet from my LAN, but first I need to setup a server to access it from on the LAN!
Installing Proxmox on PowerEdge Servers
Before I fully configure the network, I installed Proxmox onto each of my hosts using initial networking on WAN so I could install updates.
With that all setup, I want to control my network configuration mostly with pfSense, so I wrote a terraform module to deploy pfSense onto the R610 running Proxmox.
pfSense via Terraform and Packer
Terraform
providers.tf
terraform {
required_providers {
// https://github.com/Telmate/terraform-provider-proxmox
proxmox = {
source = "Telmate/proxmox"
version = "2.6.8"
}
}
}
provider "proxmox" {
pm_api_url = "https://192.168.2.25:8006/api2/json"
# Password supplied via envvar PM_PASS
pm_user = "root@pam"
pm_tls_insecure = true
}
variables.tf
/* Base image URL and checksum */
variable "PFSENSE_IMG_URL" {
default = "https://nyifiles.netgate.com/mirror/downloads/pfSense-CE-2.5.1-RELEASE-amd64.iso.gz"
}
variable "PFSENSE_IMG_SUM" {
default = "be79df534558e6a73f7be2e8643c6ed01580e40b79b255f9bd8e8cca6471fee7"
}
/* Pass Proxmox Variables into Packer */
variable "PM_PASS" {
default = ""
description = "Password for Proxmox user"
}
variable "PM_NODE" {
default = ""
description = "Node to deploy resources onto"
}
variable "PM_URL" {
default = ""
description = "API Url for Proxmox"
}
variable "PM_USER" {
default = ""
description = "User for Proxmox"
}
variable "PM_HOST" {
}
packer.tf
// Upload pfsense base image to Proxmox server
resource "null_resource" "pfsense_base_image" {
provisioner "remote-exec" {
inline = [
"cd /var/lib/vz/template/iso",
"wget '${var.PFSENSE_IMG_URL}'",
"gunzip -c pfSense-CE-2.5.1-RELEASE-amd64.iso.gz > pfSense-CE-2.5.1-RELEASE-amd64.iso",
"rm pfSense-CE-2.5.1-RELEASE-amd64.iso.gz"
]
connection {
type = "ssh"
user = "root"
password = "${var.PM_PASS}"
host = "192.168.2.25"
}
}
}
// Execute Packer to build our pfsense image on Proxmox server
resource "null_resource" "packer_pfsense_build" {
provisioner "local-exec" {
environment = {
PKR_VAR_PM_URL="${var.PM_URL}"
PKR_VAR_PM_USER="${var.PM_USER}"
PKR_VAR_PM_PASS="${var.PM_PASS}"
PKR_VAR_PM_NODE="${var.PM_NODE}"
PKR_VAR_PFSENSE_IMG_PATH="local:iso/pfSense-CE-2.5.1-RELEASE-amd64.iso"
PKR_VAR_PFSENSE_IMG_CHECKSUM="${var.PFSENSE_IMG_SUM}"
PKR_VAR_PFSENSE_VERSION="2.5.1"
}
command = <<EOT
"cd ${path.module}/packer && packer build ."
EOT
}
depends_on = [
null_resource.pfsense_base_image
]
}
proxmox.tf
resource "proxmox_vm_qemu" "pfsense_test" {
# Clone and metadata config
name = "firewall"
target_node = var.PM_NODE
desc = "pfSense node placed infront of whole network"
clone = "pfSense_"
full_clone = false
# PXE boot makes the chain fail, make sure we boot from disk!!
boot = "dcn"
bootdisk = "scsi0"
# System
memory = 4096
cores = 2
sockets = 2
cpu = "host"
qemu_os = "l26"
# WAN
network {
model = "virtio"
bridge = "vmbr0"
#vlan_tag = "10"
#firewall = true
# I setup a static DHCP lease on my router for this MAC
macaddr = "16:fb:80:e2:ac:a7"
}
# WLAN
network {
model = "virtio"
bridge = "vmbr1"
#tag = "10"
#firewall = true
}
# Management Network
network {
model = "virtio"
bridge = "vmbr2"
#tag = "20"
#firewall = true
}
# Worker Pool Network
network {
model = "virtio"
bridge = "vmbr3"
#tag = "30"
#firewall = true
}
define_connection_info = false
depends_on = [
null_resource.packer_pfsense_build,
]
lifecycle {
ignore_changes = [
# After a reboot, this swaps as it begins owning it's own disk ontop of the template
full_clone,
define_connection_info,
disk,
]
}
}
resource "proxmox_vm_qemu" "pfsense" {
# Clone and metadata config
name = "firewall"
target_node = var.PM_NODE
desc = "pfSense node placed infront of whole network"
clone = "pfSense"
full_clone = false
# PXE boot makes the chain fail, make sure we boot from disk!!
boot = "dcn"
bootdisk = "scsi0"
# System
memory = 4096
cores = 2
sockets = 2
cpu = "host"
qemu_os = "l26"
# WAN
network {
model = "virtio"
bridge = "vmbr0"
#vlan_tag = "10"
#firewall = true
# I setup a static DHCP lease on my router for this MAC
macaddr = "16:fb:80:e2:ac:a7"
}
# WLAN
network {
model = "virtio"
bridge = "vmbr1"
#tag = "10"
#firewall = true
}
# Management Network
network {
model = "virtio"
bridge = "vmbr2"
#tag = "20"
#firewall = true
}
# Worker Pool Network
network {
model = "virtio"
bridge = "vmbr3"
#tag = "30"
#firewall = true
}
define_connection_info = false
depends_on = [
null_resource.packer_pfsense_build,
]
lifecycle {
ignore_changes = [
# After a reboot, this swaps as it begins owning it's own disk ontop of the template
full_clone,
define_connection_info,
disk,
]
}
}
Packer
pfsense.pkr.hcl
source "proxmox-iso" "pfsense" {
# ID
template_name = "pfSense_${var.PFSENSE_VERSION}"
template_description = "pfSense firewall for home lab network"
# Proxmox Access configuration
proxmox_url = var.PM_URL
username = var.PM_USER
password = var.PM_PASS
node = var.PM_NODE
insecure_skip_tls_verify = true
# Base ISO File configuration
iso_file = var.PFSENSE_IMG_PATH
iso_storage_pool = "local-lvm"
iso_checksum = var.PFSENSE_IMG_CHECKSUM
# System
memory = 2048
cores = 2
sockets = 1
cpu_type = "host"
os = "l26"
# Storage
disks {
type="scsi"
disk_size="16G"
storage_pool="local-lvm"
storage_pool_type="lvm"
}
# Network
network_adapters {
model = "virtio"
bridge = "vmbr0"
#vlan_tag = "10"
#firewall = true
}
# Remove installation media
unmount_iso = true
# Start this on system boot
onboot = true
# Boot commands
boot_wait = "2m"
boot_command = [
# Accept copyright
"<enter><wait2s>",
# Options: Install, Rescue Shell, Recover config.xml
# Press enter to install
"<enter><wait2s>",
# Select keyboard layout, default US
"<enter><wait2s>",
# Options: Auto (UFS) Bios, Auto (UFS) UEFI, Manual, Shell, Auto (ZFS)
# Enter for Auto UFS Bios
"<enter><wait6m>",
# Manual Configuration menu
# Default No, Left for Yes
"<left><wait><enter><wait10s>",
# Dropped to shell...
#"echo 'Port 22' >> /etc/ssh/sshd_config"
#"echo 'ListenAddress 0.0.0.0 >> /etc/ssh/sshd_config'"
# Allow password auth
#"echo 'PasswordAuthentication yes' >> /etc/ssh/sshd_config",
#"<enter><wait>",
# Enable sshd on reboot
#"echo 'sshd_enable=\"YES\"' >> /etc/rc.conf",
#"<enter><wait>",
# Create a user for ourselves
#"pw user add -n packer -c 'Packer' -G wheel -s /usr/local/bin/bash",
#"<enter><wait>",
#"passwd packer",
#"<enter><wait>",
#"testing"
#"<enter><wait>",
#"testing",
#"<enter><wait>",
"reboot",
#"<enter><wait2m>",
#Chain the rest or it fails to get through due to reboot...
# Enter to reboot
# n to setting up vlan tags
# vtnet0 for WAN interface name
# blank for LAN interface
# y to confirm and start configuration
# 14 -> y to enable sshd
"<enter><wait3m>n<enter><wait1s>vtnet0<enter><wait1s><enter><wait1s>y<enter><wait2m>14<enter><wait>y<enter><wait>"
]
# We don't need to do anything further in packer for now
# If we did, we would have to install qemu utils to discover IP & configure ssh communicator
communicator = "none"
}
build {
// Load iso configuration
sources = ["source.proxmox-iso.pfsense"]
}
variables.pkr.hcl
# Override with PKR_VAR_foo=bar
#variable "" {
# type = string
# description = ""
#}
variable "PM_URL" {
type = string
description = "Url to JSON API for Proxmox"
}
variable "PM_USER" {
type = string
description = "Username for JSON API for Proxmox"
}
variable "PM_PASS" {
type = string
description = "Password for JSON API for Proxmox"
}
variable "PM_NODE" {
type = string
description = "Node on Proxmox to use"
}
variable "PFSENSE_IMG_PATH" {
type = string
description = "Path to Pfsense image on Proxmox system"
}
variable "PFSENSE_IMG_CHECKSUM" {
type = string
description = "sha256 checksum of pfsense image"
}
variable "PKR_VAR_PFSENSE_VERSION" {
type = string
description = "Version of pfsense image"
}
The end result is a nice atomic pfsense instance running on my R610 proxmox host! I can now use pfsense and proxmox to declare my network stack:
Cisco Switch Configuration
Now I had to learn how to really configure the cisco switch. They have their own little CLI language that you need to use in order to configure the switch, so this process was almost entirely reading documentation and figuring out how to actually set all of this up! It took me a lot of trial and error to finally get something that works, and at the end of the day I have a lot of changes I want to make that I'll describe in later blogs. What I ended up configuring:
- MGMT network port
- All other interfaces Trunked, with VLAN's as layed out in my pfsense network declarations
Because I did all this quite long ago from the time of writing, I'm not going to go into much detail. I'll save the nitty-gritty for when I start really locking down my network policies and getting fun monitoring/alerts in place.
Now that all of this is complete, I can start working on the pieces I actually care about, THE SOFTWARE!
