Contents

Home Lab 2

Now that the base servers are ready for an OS, I need to setup the Network stack for my lab. While I wanted to stay inexpensive, I also wanted everything to be rack-mounted so I ended up buying a Cisco 3750X from Ebay.
1
2
3

I slotted in the two fans, and PSU and bolted on the front-mounting bracket.
4

And finally bolted it into the rack:
5

Initial Switch Configuration

First, I reset the switch by following This guide.

Then I connected via USB Console and preformed initial setup of the switch which the boot CLI walks you through.

My next step is to setup a management interface so I can access the switch over telnet from my LAN, but first I need to setup a server to access it from on the LAN!

Installing Proxmox on PowerEdge Servers

Before I fully configure the network, I installed Proxmox onto each of my hosts using initial networking on WAN so I could install updates.
6

With that all setup, I want to control my network configuration mostly with pfSense, so I wrote a terraform module to deploy pfSense onto the R610 running Proxmox.

pfSense via Terraform and Packer

Terraform

providers.tf

terraform {
  required_providers {
    // https://github.com/Telmate/terraform-provider-proxmox
    proxmox = {
      source = "Telmate/proxmox"
      version = "2.6.8"
    }
  }
}

provider "proxmox" {
    pm_api_url = "https://192.168.2.25:8006/api2/json"
    # Password supplied via envvar PM_PASS
    pm_user = "[email protected]" 
    pm_tls_insecure = true   
}

variables.tf

/* Base image URL and checksum */
variable "PFSENSE_IMG_URL" {
    default = "https://nyifiles.netgate.com/mirror/downloads/pfSense-CE-2.5.1-RELEASE-amd64.iso.gz"
}
variable "PFSENSE_IMG_SUM" {
    default = "be79df534558e6a73f7be2e8643c6ed01580e40b79b255f9bd8e8cca6471fee7"
}
/* Pass Proxmox Variables into Packer */
variable "PM_PASS" {
    default = ""
    description = "Password for Proxmox user"
}
variable "PM_NODE" {
    default = ""
    description = "Node to deploy resources onto"
}
variable "PM_URL" {
    default = ""
    description = "API Url for Proxmox"
}
variable "PM_USER" {
    default = ""
    description = "User for Proxmox"
}
variable "PM_HOST" {
  
}

packer.tf

// Upload pfsense base image to Proxmox server
resource "null_resource" "pfsense_base_image" {
    provisioner "remote-exec" {
        inline = [
            "cd /var/lib/vz/template/iso",
            "wget '${var.PFSENSE_IMG_URL}'",
            "gunzip -c pfSense-CE-2.5.1-RELEASE-amd64.iso.gz > pfSense-CE-2.5.1-RELEASE-amd64.iso",
            "rm pfSense-CE-2.5.1-RELEASE-amd64.iso.gz"
        ]
        connection {
            type     = "ssh"
            user     = "root"
            password = "${var.PM_PASS}"
            host     = "192.168.2.25"
        }
    }
}

// Execute Packer to build our pfsense image on Proxmox server
resource "null_resource" "packer_pfsense_build" {
    provisioner "local-exec" {
        environment = {
            PKR_VAR_PM_URL="${var.PM_URL}"
            PKR_VAR_PM_USER="${var.PM_USER}"
            PKR_VAR_PM_PASS="${var.PM_PASS}"
            PKR_VAR_PM_NODE="${var.PM_NODE}"
            PKR_VAR_PFSENSE_IMG_PATH="local:iso/pfSense-CE-2.5.1-RELEASE-amd64.iso"
            PKR_VAR_PFSENSE_IMG_CHECKSUM="${var.PFSENSE_IMG_SUM}"
            PKR_VAR_PFSENSE_VERSION="2.5.1"
        }
        command = <<EOT
            "cd ${path.module}/packer && packer build ."
        EOT
    }
    
    depends_on = [
      null_resource.pfsense_base_image
    ]
}

proxmox.tf

resource "proxmox_vm_qemu" "pfsense_test" {
    # Clone and metadata config
    name = "firewall"
    target_node = var.PM_NODE
    desc = "pfSense node placed infront of whole network"
    clone = "pfSense_"
    full_clone = false
    # PXE boot makes the chain fail, make sure we boot from disk!!
    boot = "dcn"
    bootdisk = "scsi0"

    # System
    memory = 4096
    cores = 2
    sockets = 2
    cpu = "host"
    qemu_os = "l26"

    # WAN
    network {
      model = "virtio"
      bridge = "vmbr0"
      #vlan_tag = "10"
      #firewall = true
      # I setup a static DHCP lease on my router for this MAC
      macaddr = "16:fb:80:e2:ac:a7"
     }
     # WLAN
     network {
      model = "virtio"
      bridge = "vmbr1"
      #tag = "10"
      #firewall = true
     }
     # Management Network
     network {
      model = "virtio"
      bridge = "vmbr2"
      #tag = "20"
      #firewall = true
     }
     # Worker Pool Network
    network {
      model = "virtio"
      bridge = "vmbr3"
      #tag = "30"
      #firewall = true
     }

    define_connection_info = false

    depends_on = [
      null_resource.packer_pfsense_build,
    ]

    lifecycle {
      ignore_changes = [
        # After a reboot, this swaps as it begins owning it's own disk ontop of the template
        full_clone,
        define_connection_info,
        disk,
      ]
    }
}

resource "proxmox_vm_qemu" "pfsense" {
    # Clone and metadata config
    name = "firewall"
    target_node = var.PM_NODE
    desc = "pfSense node placed infront of whole network"
    clone = "pfSense"
    full_clone = false
    # PXE boot makes the chain fail, make sure we boot from disk!!
    boot = "dcn"
    bootdisk = "scsi0"

    # System
    memory = 4096
    cores = 2
    sockets = 2
    cpu = "host"
    qemu_os = "l26"

    # WAN
    network {
      model = "virtio"
      bridge = "vmbr0"
      #vlan_tag = "10"
      #firewall = true
      # I setup a static DHCP lease on my router for this MAC
      macaddr = "16:fb:80:e2:ac:a7"
     }
     # WLAN
     network {
      model = "virtio"
      bridge = "vmbr1"
      #tag = "10"
      #firewall = true
     }
     # Management Network
     network {
      model = "virtio"
      bridge = "vmbr2"
      #tag = "20"
      #firewall = true
     }
     # Worker Pool Network
    network {
      model = "virtio"
      bridge = "vmbr3"
      #tag = "30"
      #firewall = true
     }

    define_connection_info = false

    depends_on = [
      null_resource.packer_pfsense_build,
    ]

    lifecycle {
      ignore_changes = [
        # After a reboot, this swaps as it begins owning it's own disk ontop of the template
        full_clone,
        define_connection_info,
        disk,
      ]
    }
}

Packer

pfsense.pkr.hcl

source "proxmox-iso" "pfsense" {
    # ID
    template_name = "pfSense_${var.PFSENSE_VERSION}"
    template_description = "pfSense firewall for home lab network"
    # Proxmox Access configuration
    proxmox_url = var.PM_URL
    username = var.PM_USER
    password = var.PM_PASS
    node = var.PM_NODE
    insecure_skip_tls_verify = true
    # Base ISO File configuration
    iso_file = var.PFSENSE_IMG_PATH
    iso_storage_pool = "local-lvm"
    iso_checksum = var.PFSENSE_IMG_CHECKSUM

    # System
    memory = 2048
    cores = 2
    sockets = 1
    cpu_type = "host"
    os = "l26"
    
    # Storage
    disks {
        type="scsi"
        disk_size="16G"
        storage_pool="local-lvm"
        storage_pool_type="lvm"
    }
    

    # Network
    network_adapters {
            model = "virtio"
            bridge = "vmbr0"
            #vlan_tag = "10"
            #firewall = true
    }
    
    
    # Remove installation media
    unmount_iso = true
    # Start this on system boot
    onboot = true

    # Boot commands
    boot_wait = "2m"
    boot_command = [
        # Accept copyright
        "<enter><wait2s>",
        # Options: Install, Rescue Shell, Recover config.xml
        # Press enter to install
        "<enter><wait2s>",
        # Select keyboard layout, default US
        "<enter><wait2s>",
        # Options: Auto (UFS) Bios, Auto (UFS) UEFI, Manual, Shell, Auto (ZFS)
        # Enter for Auto UFS Bios
        "<enter><wait6m>",
        # Manual Configuration menu
        # Default No, Left for Yes
        "<left><wait><enter><wait10s>",

        # Dropped to shell...
        #"echo 'Port 22' >> /etc/ssh/sshd_config"
        #"echo 'ListenAddress 0.0.0.0 >> /etc/ssh/sshd_config'"
        # Allow password auth
        #"echo 'PasswordAuthentication yes' >> /etc/ssh/sshd_config",
        #"<enter><wait>",
        # Enable sshd on reboot
        #"echo 'sshd_enable=\"YES\"' >> /etc/rc.conf",
        #"<enter><wait>",
        # Create a user for ourselves
        #"pw user add -n packer -c 'Packer'  -G wheel -s /usr/local/bin/bash",
        #"<enter><wait>",
        #"passwd packer",
        #"<enter><wait>",
        #"testing"
        #"<enter><wait>",
        #"testing",
        #"<enter><wait>",
        "reboot",
        #"<enter><wait2m>",

        #Chain the rest or it fails to get through due to reboot...
        # Enter to reboot
        # n to setting up vlan tags
        # vtnet0 for WAN interface name
        # blank for LAN interface
        # y to confirm and start configuration
        # 14 -> y to enable sshd
        "<enter><wait3m>n<enter><wait1s>vtnet0<enter><wait1s><enter><wait1s>y<enter><wait2m>14<enter><wait>y<enter><wait>"
    ]
    # We don't need to do anything further in packer for now
    # If we did, we would have to install qemu utils to discover IP & configure ssh communicator
    communicator = "none"
}

build {
    // Load iso configuration
   sources = ["source.proxmox-iso.pfsense"]
}

variables.pkr.hcl

# Override with PKR_VAR_foo=bar
#variable "" {
#    type = string
#    description = ""
#}

variable "PM_URL" {
    type = string
    description = "Url to JSON API for Proxmox"
}
variable "PM_USER" {
    type = string
    description = "Username for JSON API for Proxmox"
}
variable "PM_PASS" {
    type = string
    description = "Password for JSON API for Proxmox"
}
variable "PM_NODE" {
    type = string
    description = "Node on Proxmox to use"
}
variable "PFSENSE_IMG_PATH" {
    type = string
    description = "Path to Pfsense image on Proxmox system"
}

variable "PFSENSE_IMG_CHECKSUM" {
    type = string
    description = "sha256 checksum of pfsense image"
}
variable "PKR_VAR_PFSENSE_VERSION" {
    type = string
    description = "Version of pfsense image"
}

The end result is a nice atomic pfsense instance running on my R610 proxmox host! I can now use pfsense and proxmox to declare my network stack:

7
8

Cisco Switch Configuration

Now I had to learn how to really configure the cisco switch. They have their own little CLI language that you need to use in order to configure the switch, so this process was almost entirely reading documentation and figuring out how to actually set all of this up! It took me a lot of trial and error to finally get something that works, and at the end of the day I have a lot of changes I want to make that I’ll describe in later blogs.

What I ended up configuring:
- MGMT network port
- All other interfaces Trunked, with VLAN’s as layed out in my pfsense network declarations

Because I did all this quite long ago from the time of writing, I’m not going to go into much detail. I’ll save the nitty-gritty for when I start really locking down my network policies and getting fun monitoring/alerts in place.

Now that all of this is complete, I can start working on the pieces I actually care about, THE SOFTWARE!

9

Directory
$ cd content && tree