Since all of Netsoc's services run as VMs on Proxmox, we can define our entire infrastructure using Infrastructure-as-Code.
All of our IaC is fully open source and available on GitHub
While you read this wiki page, you may find it helpful to check out the files as they're mentioned
proxmox-cloudinit-vm
:
create-infra-databases.yml
): groups:
- vm
- prometheus_base
- prometheus_docker
- fluentd
- ipaclients
host_vars:
ansible_ssh_private_key_file: "./keys/infra/databases/id_rsa"
#cloud-config
)create-infra-databases.yml
):cloudinit:
pool: local
force: yes
userdata: |
#cloud-config
preserve_hostname: false
manage_etc_hosts: true
fqdn: databases.infra.netsoc.co
packages:
- qemu-guest-agent
runcmd:
- [ systemctl, enable, qemu-guest-agent ]
- [ systemctl, start, qemu-guest-agent, --no-block ]
users:
- name: netsoc
gecos: Netsoc Management User
primary_group: netsoc
groups: netsoc
shell: /bin/bash
sudo: ALL=(ALL) NOPASSWD:ALL
ssh_authorized_keys:
- "{{ lookup('file', './keys/infra/databases/id_rsa.pub') }}"
apt:
preserve_sources_list: true
primary:
- arches: [default]
uri: http://ie.archive.ubuntu.com/ubuntu/
security:
- uri: http://security.ubuntu.com/ubuntu
disk_setup:
/dev/vdb:
table_type: 'gpt'
layout:
- [100,83] # 100% Linux fs
overwrite: false
fs_setup:
- label: data-disk
filesystem: ext4
device: '/dev/vdb1'
overwrite: false
mounts:
- [ vdb, /netsoc, auto ]
networkconfig:
version: 2
ethernets: "{{ interfaces.infra.databases }}"
qemu-guest-agent
from APT reposqemu-guest-agent
servicenetsoc
user with sudo
permissions and a public key so we can ssh in to provision the VMcreate-infra-auth.yml
for more infoqemu-guest-agent
will tell us what IPs the machines are runningproxmox_inventory.py
./start-dev.sh
scripthost_vars
from the VM's descriptionansible -i proxmox_inventory.py all -m ping
and it should be able to ping all the VMsansible-playbook -i proxmox_inventory.py ...
will use that inventoryansible-playbook -i hosts -i proxmox_inventory ...
./proxmox_inventory.py --list
provision-*.yml
databases.vm.netsoc.co
provision-router.yml
setup-external-dns.yml
playbookprovision-infra-auth-internal-dns.yml
vars/network.yml
to set IP allocations for VMscreate-*.yml
to create a VM that uses that IPvars/network.yml
excerpt
vlan:
wan: 10
proxmox: 20
infra: 30
k8s: 40
cloud: 50
extern: 60 # VRRP stuff
router: 70 # Used for vyos clustering + failover
oob: 80 # OOB, connected to CIX OOBnet and OOB ports on machines
mgmt: 90 #
subnets:
wan: "10.0.{{ vlan.wan }}.0/24"
proxmox: "10.0.{{ vlan.proxmox }}.0/24"
infra: "10.0.{{ vlan.infra }}.0/24"
k8s: "10.{{ vlan.k8s }}.0.0/16"
cloud: "10.{{ vlan.cloud }}.0.0/16"
gateway4:
infra: "10.0.{{ vlan.infra }}.1"
k8s: "10.{{ vlan.k8s }}.0.1"
nameservers:
infra:
search:
- "infra.netsoc.co"
addresses:
- "{{ subnets.infra | ipmath(15) }}" # 10.0.30.15!
public:
search: []
addresses:
- 1.1.1.1
vmbr: "vmbr0"
interfaces:
infra:
databases:
net0:
match:
macaddress: "b2:eb:14:0a:54:a5"
addresses:
- "{{ subnets.infra | ipaddr(25) }}" # 10.0.30.25!
gateway4: "{{ gateway4.infra }}"
nameservers: "{{ nameservers.infra }}"
optional: true
link-local: []
/netsoc
create-infra-databases.yml
- name: "INFRA - Ensure databases server"
hosts: lovelace
roles:
- role: proxmox-cloudinit-vm
vars:
vm:
name: "databases.infra.netsoc.co"
clone:
pool: local
qcow2_url: "https://cloud-images.ubuntu.com/releases/focal/release-20200921.1/ubuntu-20.04-server-cloudimg-amd64.img"
qcow2_hash: sha256:d18a9d2b890d3c1401e70a281cae44b61816aa669c4936f7a99c168e572ec8cb
cores: 4
memory: 4096
description:
groups:
- vm
- prometheus_base
- prometheus_docker
- fluentd
- ipaclients
host_vars:
ansible_ssh_private_key_file: "./keys/infra/databases/id_rsa"
net:
net0: "virtio={{ interfaces.infra.databases.net0.match.macaddress }},tag={{ vlan.infra }},bridge={{ vmbr }}"
disks:
pool: local-lvm
boot:
size: 30G
format: raw
extra:
- size: 45G
format: raw
efi:
pool: local
cloudinit:
pool: local
force: yes
userdata: |
#cloud-config
preserve_hostname: false
manage_etc_hosts: true
fqdn: databases.infra.netsoc.co
packages:
- qemu-guest-agent
runcmd:
- [ systemctl, enable, qemu-guest-agent ]
- [ systemctl, start, qemu-guest-agent, --no-block ]
users:
- name: netsoc
gecos: Netsoc Management User
primary_group: netsoc
groups: netsoc
shell: /bin/bash
sudo: ALL=(ALL) NOPASSWD:ALL
ssh_authorized_keys:
- "{{ lookup('file', './keys/infra/databases/id_rsa.pub') }}"
apt:
preserve_sources_list: true
primary:
- arches: [default]
uri: http://ie.archive.ubuntu.com/ubuntu/
security:
- uri: http://security.ubuntu.com/ubuntu
disk_setup:
/dev/vdb:
table_type: 'gpt'
layout:
- [100,83] # 100% Linux fs
overwrite: false
fs_setup:
- label: data-disk
filesystem: ext4
device: '/dev/vdb1'
overwrite: false
mounts:
- [ vdb, /netsoc, auto ]
networkconfig:
version: 2
ethernets: "{{ interfaces.infra.databases }}"
wait_for_ssh_ip: "{{ interfaces.infra.databases.net0.addresses[0] }}"
vars_files:
- vars/network.yml
- vars/secrets_mapping.yml
- vars/secrets.yml
- name: "Reload inventory to pull new VMs"
hosts: 127.0.0.1
connection: local
tasks:
- meta: refresh_inventory
Then run ./start-dev.sh
to get into dev mode followed by ./run.sh create-infra-databases.sh
run.sh
is a handy script that hides all the ansible-playbook
flags you need
provision-infra-databases.yml
- name: "INITIAL SETUP"
hosts: databases.infra.netsoc.co
become: yes
roles:
- ansible-requirements
- utf8-locale
- docker
tasks:
- name: "Make /netsoc only readable by root"
file:
path: "/netsoc"
owner: root
group: root
mode: '1770'
tags:
- initial
- name: "DATABASES - MySQL/PG"
hosts: databases.infra.netsoc.co
become: yes
tasks:
- name: MYSQL/PG/PROM - Ensure directories exist
file:
path: "/netsoc/{{ item }}"
owner: root
group: root
# Needs to be 775 as this folder is accessed internally by a dirsrv user
# https://github.com/freeipa/freeipa-container/issues/281
mode: '0755'
state: directory
with_items:
- mysql
- pg
- name: PG - Install psycopg2 (needed for ansible pg driver)
apt:
name: python3-psycopg2
state: present
- name: MYSQL/PG - Remember Compose definition
set_fact:
definition:
version: "3.5"
services:
mysql:
hostname: "mysql.databases.infra.netsoc.co"
image: mysql:8.0.20
restart: on-failure
environment:
MYSQL_ROOT_PASSWORD: "{{ mysql.infra.root_password }}"
volumes:
- "/netsoc/mysql:/var/lib/mysql"
ports:
- "3306:3306"
pg:
hostname: "pg.databases.netsoc.co"
image: postgres:12.3
restart: on-failure
environment:
POSTGRES_PASSWORD: "{{ postgres.infra.postgres_password }}"
volumes:
- "/netsoc/pg:/var/lib/postgresql/data"
ports:
- "5432:5432"
- name: MYSQL/PG - Setup services
docker_compose:
project_name: databases
recreate: smart
remove_orphans: yes
state: present
definition: "{{ definition }}"
vars_files:
- "vars/network.yml"
- "vars/secrets_mapping.yml"
- "vars/secrets.yml"
tags:
- provision
Then run ./run.sh provision-infra-databases.sh
If you wanted to run this again but recreate the database images you can run ./run.sh provision-infra-databases.sh --tags provision
to only select the tasks with provision
tags
* The data already stored in the dbs won't be destroyed because they mount the folders already on the data disk),