Production 3-Node HashiCorp Vault Cluster on AWS Ubuntu

sudo apt-get update
sudo apt-get install -y gpg wget lsb-release

wget -O- https://apt.releases.hashicorp.com/gpg | \
  sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg

echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \
  https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \
  sudo tee /etc/apt/sources.list.d/hashicorp.list

sudo apt-get update
sudo apt-get install -y vault

vault --version

# On each node — create TLS directory

sudo mkdir -p /opt/vault/tls
sudo chown vault:vault /opt/vault/tls

# Option A: Let's Encrypt (if nodes have public DNS)

sudo apt-get install -y certbot

sudo certbot certonly --standalone \
  -d vault-1.yourdomain.com \
  --non-interactive --agree-tos -m admin@yourdomain.com

sudo cp /etc/letsencrypt/live/vault-1.yourdomain.com/fullchain.pem /opt/vault/tls/cert.pem
sudo cp /etc/letsencrypt/live/vault-1.yourdomain.com/privkey.pem /opt/vault/tls/key.pem
sudo chown vault:vault /opt/vault/tls/*.pem
sudo chmod 600 /opt/vault/tls/key.pem

# Option B: Self-signed (internal-only cluster)

# Generate CA (do this once, then copy to all nodes)
openssl genrsa -out /opt/vault/tls/ca-key.pem 4096
openssl req -x509 -new -nodes -key /opt/vault/tls/ca-key.pem \
  -sha256 -days 3650 -out /opt/vault/tls/ca.pem \
  -subj "/CN=Vault CA"

# Generate node cert (run on each node with correct IP/DNS)
openssl genrsa -out /opt/vault/tls/key.pem 4096
openssl req -new -key /opt/vault/tls/key.pem \
  -out /opt/vault/tls/vault.csr \
  -subj "/CN=vault-1.yourdomain.com"

# Sign with CA
openssl x509 -req -in /opt/vault/tls/vault.csr \
  -CA /opt/vault/tls/ca.pem -CAkey /opt/vault/tls/ca-key.pem \
  -CAcreateserial -out /opt/vault/tls/cert.pem -days 365 -sha256 \
  -extfile <(printf "subjectAltName=IP:10.0.1.10,DNS:vault-1.yourdomain.com")

sudo chown vault:vault /opt/vault/tls/*.pem
sudo chmod 600 /opt/vault/tls/key.pem

# Run on ALL 3 nodes

sudo mkdir -p /opt/vault/data
sudo chown -R vault:vault /opt/vault

sudo systemctl enable vault
sudo systemctl start vault
sudo systemctl status vault

# On vault-1 only

export VAULT_ADDR="https://10.0.1.10:8200"
export VAULT_CACERT="/opt/vault/tls/ca.pem"

vault operator init -recovery-shares=5 -recovery-threshold=3

# Expected output

Recovery Key 1: xxxxx...
Recovery Key 2: xxxxx...
Recovery Key 3: xxxxx...
Recovery Key 4: xxxxx...
Recovery Key 5: xxxxx...

Initial Root Token: hvs.xxxxxxxxxxxx

Vault initialized with 5 key shares and a key threshold of 3.

# Login

vault login hvs.xxxxxxxxxxxx

# Check Vault status on any node

vault status

# List Raft peers (shows all 3 nodes)

vault operator raft list-peers

# Expected output

Node       Address            State       Voter
----       -------            -----       -----
vault-1    10.0.1.10:8201     leader      true
vault-2    10.0.2.10:8201     follower    true
vault-3    10.0.3.10:8201     follower    true

# AWS CLI — Create target group with health check

aws elbv2 create-target-group \
  --name vault-tg \
  --protocol TCP \
  --port 8200 \
  --vpc-id vpc-xxxxxxxx \
  --target-type instance \
  --health-check-protocol HTTPS \
  --health-check-path "/v1/sys/health?standbyok=true" \
  --health-check-port 8200

# Register all 3 nodes

aws elbv2 register-targets \
  --target-group-arn arn:aws:elasticloadbalancing:...:targetgroup/vault-tg/xxx \
  --targets Id=i-node1 Id=i-node2 Id=i-node3

# /opt/vault/scripts/backup.sh

#!/bin/bash
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
SNAPSHOT_FILE="/tmp/vault-snapshot-${TIMESTAMP}.snap"
S3_BUCKET="s3://my-vault-backups/snapshots"

export VAULT_ADDR="https://127.0.0.1:8200"
export VAULT_CACERT="/opt/vault/tls/ca.pem"
export VAULT_TOKEN=$(cat /opt/vault/.backup-token)

vault operator raft snapshot save "$SNAPSHOT_FILE"

aws s3 cp "$SNAPSHOT_FILE" "$S3_BUCKET/${TIMESTAMP}.snap" --sse AES256

rm -f "$SNAPSHOT_FILE"
echo "Snapshot saved: $TIMESTAMP"

# Cron job (run on leader node)

sudo chmod +x /opt/vault/scripts/backup.sh

# Daily at 2 AM
echo "0 2 * * * vault /opt/vault/scripts/backup.sh" | sudo tee /etc/cron.d/vault-backup

# Check current leader

vault operator raft list-peers

# Stop the leader node

# On the leader node (e.g., vault-1):
sudo systemctl stop vault

# Verify new leader elected (from another node)

export VAULT_ADDR="https://10.0.2.10:8200"
vault operator raft list-peers

# Expected: vault-2 or vault-3 is now "leader"
# Vault API continues to work via NLB

# Bring the node back

# On vault-1:
sudo systemctl start vault

# It auto-unseals via KMS and re-joins as follower
vault operator raft list-peers

# Add to /etc/vault.d/vault.hcl on all nodes

telemetry {
  prometheus_retention_time = "24h"
  disable_hostname         = true
}

# Prometheus scrape config

- job_name: 'vault'
  metrics_path: '/v1/sys/metrics'
  params:
    format: ['prometheus']
  scheme: https
  tls_config:
    ca_file: /etc/prometheus/vault-ca.pem
  bearer_token_file: /etc/prometheus/vault-token
  static_configs:
    - targets:
        - '10.0.1.10:8200'
        - '10.0.2.10:8200'
        - '10.0.3.10:8200'

# Remove a dead node from the cluster

vault operator raft remove-peer vault-3

# Add a replacement node (new node joins via retry_join)

# Just start Vault on the new node with retry_join config
# It auto-joins and syncs data from the leader

# Step down the leader (graceful)

vault operator step-down

# Restore from snapshot

vault operator raft snapshot restore /path/to/snapshot.snap