Managing TLS Certificates Across 100+ Domains Without Expiration Outages

TL;DR — Quick Fix

# Check expiring certificates across all domains
echo | openssl s_client -connect example.com:443 -servername example.com 2>/dev/null | \
  openssl x509 -noout -dates

# Install cert-manager for automated renewal
helm install cert-manager jetstack/cert-manager \
  --namespace cert-manager --create-namespace \
  --set installCRDs=true

---

Why Certificates Expire in Production

Certificate expiration is one of the most common causes of production outages and one of the most preventable.

Cert-Manager: Automated Certificate Lifecycle

Installation

helm repo add jetstack https://charts.jetstack.io
helm repo update

helm install cert-manager jetstack/cert-manager \
  --namespace cert-manager \
  --create-namespace \
  --version v1.15.0 \
  --set installCRDs=true \
  --set prometheus.enabled=true

ClusterIssuer for Let's Encrypt

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: devops@yourcompany.com
    privateKeySecretRef:
      name: letsencrypt-prod-account-key
    solvers:
      - http01:
          ingress:
            class: nginx
      - dns01:
          route53:
            region: us-east-1
            hostedZoneID: Z1234567890
        selector:
          dnsZones:
            - "yourcompany.com"

Certificate Resource

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: api-tls
  namespace: production
spec:
  secretName: api-tls-secret
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  dnsNames:
    - api.yourcompany.com
    - api-v2.yourcompany.com
  renewBefore: 720h
  duration: 2160h

Wildcard Certificate

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: wildcard-tls
  namespace: production
spec:
  secretName: wildcard-tls-secret
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  dnsNames:
    - "*.yourcompany.com"
    - "yourcompany.com"

Ingress Annotation (Simplest Method)

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: api-ingress
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
  tls:
    - hosts:
        - api.yourcompany.com
      secretName: api-tls-auto
  rules:
    - host: api.yourcompany.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: api-service
                port:
                  number: 80

Monitoring Certificate Expiry

Prometheus Alerts

groups:
  - name: certificate-alerts
    rules:
      - alert: CertificateExpiringSoon
        expr: certmanager_certificate_expiration_timestamp_seconds - time() < 2592000
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "Certificate {{ $labels.name }} expires in less than 30 days"

      - alert: CertificateExpiringCritical
        expr: certmanager_certificate_expiration_timestamp_seconds - time() < 604800
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Certificate {{ $labels.name }} expires in less than 7 days!"

      - alert: CertificateRenewalFailed
        expr: certmanager_certificate_ready_status{condition="False"} == 1
        for: 30m
        labels:
          severity: critical
        annotations:
          summary: "Certificate {{ $labels.name }} is not ready — renewal may have failed"

External Certificate Monitoring Script

#!/bin/bash
# check-certs.sh — Run daily via cron

DOMAINS=(
  "api.yourcompany.com"
  "app.yourcompany.com"
  "admin.yourcompany.com"
  "cdn.yourcompany.com"
)

WARN_DAYS=30
CRITICAL_DAYS=7

for domain in "${DOMAINS[@]}"; do
  EXPIRY=$(echo | openssl s_client -connect "$domain:443" -servername "$domain" 2>/dev/null | \
    openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)

  if [ -z "$EXPIRY" ]; then
    echo "ERROR: Cannot check $domain"
    continue
  fi

  EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s)
  NOW_EPOCH=$(date +%s)
  DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))

  if [ "$DAYS_LEFT" -lt "$CRITICAL_DAYS" ]; then
    echo "CRITICAL: $domain expires in $DAYS_LEFT days ($EXPIRY)"
  elif [ "$DAYS_LEFT" -lt "$WARN_DAYS" ]; then
    echo "WARNING: $domain expires in $DAYS_LEFT days ($EXPIRY)"
  else
    echo "OK: $domain expires in $DAYS_LEFT days"
  fi
done

AWS Certificate Manager (ACM)

resource "aws_acm_certificate" "api" {
  domain_name               = "api.yourcompany.com"
  subject_alternative_names = ["api-v2.yourcompany.com"]
  validation_method         = "DNS"

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_route53_record" "cert_validation" {
  for_each = {
    for dvo in aws_acm_certificate.api.domain_validation_options : dvo.domain_name => {
      name   = dvo.resource_record_name
      record = dvo.resource_record_value
      type   = dvo.resource_record_type
    }
  }

  zone_id = data.aws_route53_zone.main.zone_id
  name    = each.value.name
  type    = each.value.type
  records = [each.value.record]
  ttl     = 60
}

resource "aws_acm_certificate_validation" "api" {
  certificate_arn         = aws_acm_certificate.api.arn
  validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn]
}

ACM certificates auto-renew as long as DNS validation records remain in place.

Troubleshooting Cert-Manager

Certificate Stuck in "Not Ready"

# Check certificate status
kubectl get certificates -A
kubectl describe certificate <name> -n <namespace>

# Check ACME orders and challenges
kubectl get orders -A
kubectl get challenges -A
kubectl describe challenge <name> -n <namespace>

Common Failures

---

Frequently Asked Questions

How does cert-manager handle renewal automatically?

Cert-manager checks the renewBefore field (default: 2/3 of certificate lifetime). For a 90-day Let's Encrypt cert, it renews at 60 days. It re-solves the ACME challenge, gets a new cert, and updates the Kubernetes Secret automatically.

Can I use cert-manager with non-Kubernetes services?

Yes, cert-manager stores certificates as Kubernetes Secrets. Sync these to external systems using external-secrets-operator or CSI secret store drivers. For non-Kubernetes infrastructure, use Terraform's ACME provider.

What happens if Let's Encrypt is down during renewal?

Cert-manager retries with exponential backoff. Since renewal starts 30 days before expiry, temporary CA outages won't cause issues. The certificate remains valid until its actual expiry date.

Should I use wildcard certificates or individual certs?

Wildcards are simpler (one cert for all subdomains) but have a larger blast radius if compromised. Use wildcards for internal services and individual certs for public-facing critical services.

How do I handle certificates for internal services?

Use a private CA (cert-manager self-signed or CA issuer) for internal services. This avoids rate limits and doesn't require public DNS validation. Distribute the CA certificate to all services that need to trust internal TLS.

---

Related Resources

• Kubernetes Ingress NGINX Setup — Ingress configuration with TLS
• Secrets Management in DevOps — Managing sensitive credentials
• AWS IAM Least Privilege Guide — Permissions for cert-manager