TL;DR — Quick Fix
Rotate API keys with zero downtime using the dual-key overlap pattern:
# Step 1: Generate new key (both old and new are valid)
NEW_KEY=$(openssl rand -hex 32)
aws secretsmanager update-secret --secret-id my-api-key \
--secret-string "{\"current\":\"$NEW_KEY\",\"previous\":\"$OLD_KEY\"}"
# Step 2: Update consumers to use new key
kubectl rollout restart deployment/api-consumer
# Step 3: Wait for all consumers to pick up new key
sleep 300
# Step 4: Invalidate old key (remove from previous slot)
aws secretsmanager update-secret --secret-id my-api-key \
--secret-string "{\"current\":\"$NEW_KEY\",\"previous\":\"\"}"
# Application code — accept both current and previous keys
def validate_api_key(request_key: str) -> bool:
secret = get_secret("my-api-key")
valid_keys = [secret["current"], secret.get("previous", "")]
return request_key in [k for k in valid_keys if k]
---
Architecture — Zero-Downtime Key Rotation Flow
---
Step 1 — Dual-Key Validation Pattern
The provider (API server) must accept both old and new keys during the overlap window:
# api_auth.py — dual-key validation
import json
import boto3
from functools import lru_cache
from datetime import datetime
secrets_client = boto3.client('secretsmanager')
@lru_cache(maxsize=1)
def _get_secret_cached(secret_id: str, cache_bust: int):
"""Cache secret for 60 seconds to reduce API calls."""
response = secrets_client.get_secret_value(SecretId=secret_id)
return json.loads(response['SecretString'])
def get_valid_keys(secret_id: str) -> list:
"""Get all currently valid keys (current + previous)."""
cache_bust = int(datetime.now().timestamp()) // 60
secret = _get_secret_cached(secret_id, cache_bust)
keys = [secret.get("current", "")]
if secret.get("previous"):
keys.append(secret["previous"])
return [k for k in keys if k]
def validate_api_key(request_key: str, secret_id: str = "my-api-key") -> bool:
"""Validate incoming API key against current and previous keys."""
valid_keys = get_valid_keys(secret_id)
return request_key in valid_keys
---
Step 2 — HashiCorp Vault Dynamic Secrets
Vault can generate short-lived credentials that auto-expire, eliminating manual rotation:
# Enable the database secrets engine
resource "vault_database_secrets_mount" "db" {
path = "database"
postgresql {
name = "production"
username = "vault_admin"
password = var.db_admin_password
connection_url = "postgresql://{{username}}:{{password}}@db.internal:5432/app"
allowed_roles = ["app-role"]
}
}
# Create a role that generates short-lived credentials
resource "vault_database_secret_backend_role" "app" {
backend = vault_database_secrets_mount.db.path
name = "app-role"
db_name = "production"
default_ttl = 3600 # 1 hour
max_ttl = 86400 # 24 hours
creation_statements = [
"CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';",
"GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA public TO \"{{name}}\";"
]
revocation_statements = [
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM \"{{name}}\";",
"DROP ROLE IF EXISTS \"{{name}}\";"
]
}
# vault_client.py — auto-renewing Vault credentials
import hvac
import threading
import time
class VaultCredentialManager:
def __init__(self, vault_addr, role):
self.client = hvac.Client(url=vault_addr)
self.role = role
self._credentials = None
self._lock = threading.Lock()
self._refresh_thread = threading.Thread(
target=self._auto_refresh, daemon=True
)
self._refresh_thread.start()
def get_credentials(self):
with self._lock:
return self._credentials
def _auto_refresh(self):
while True:
try:
response = self.client.secrets.database.generate_credentials(
name=self.role
)
with self._lock:
self._credentials = {
"username": response["data"]["username"],
"password": response["data"]["password"],
}
ttl = response["lease_duration"]
time.sleep(ttl * 0.75) # Refresh at 75% of TTL
except Exception:
time.sleep(30) # Retry on failure
---
Step 3 — AWS Secrets Manager Rotation Lambda
# rotation_lambda.py — AWS Secrets Manager rotation function
import boto3
import json
import secrets
import string
secrets_client = boto3.client('secretsmanager')
def lambda_handler(event, context):
"""Handles the four steps of secret rotation."""
step = event['Step']
secret_id = event['SecretId']
token = event['ClientRequestToken']
if step == "createSecret":
create_secret(secret_id, token)
elif step == "setSecret":
set_secret(secret_id, token)
elif step == "testSecret":
test_secret(secret_id, token)
elif step == "finishSecret":
finish_secret(secret_id, token)
def create_secret(secret_id, token):
"""Generate a new API key."""
new_key = ''.join(secrets.choice(
string.ascii_letters + string.digits
) for _ in range(64))
secrets_client.put_secret_value(
SecretId=secret_id,
ClientRequestToken=token,
SecretString=json.dumps({"api_key": new_key}),
VersionStages=['AWSPENDING']
)
def set_secret(secret_id, token):
"""Apply the new key to the target service."""
pending = secrets_client.get_secret_value(
SecretId=secret_id, VersionStage='AWSPENDING'
)
new_key = json.loads(pending['SecretString'])['api_key']
register_api_key(new_key)
def test_secret(secret_id, token):
"""Verify the new key works."""
pending = secrets_client.get_secret_value(
SecretId=secret_id, VersionStage='AWSPENDING'
)
new_key = json.loads(pending['SecretString'])['api_key']
if not verify_api_key_works(new_key):
raise ValueError("New API key validation failed")
def finish_secret(secret_id, token):
"""Mark the new version as current."""
secrets_client.update_secret_version_stage(
SecretId=secret_id,
VersionStage='AWSCURRENT',
MoveToVersionId=token,
RemoveFromVersionId=get_current_version(secret_id)
)
# Terraform — schedule automatic rotation every 30 days
resource "aws_secretsmanager_secret_rotation" "api_key" {
secret_id = aws_secretsmanager_secret.api_key.id
rotation_lambda_arn = aws_lambda_function.rotation.arn
rotation_rules {
automatically_after_days = 30
}
}
---
Step 4 — Application-Level Key Refresh
# Kubernetes — External Secrets Operator for auto-refresh
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: api-credentials
spec:
refreshInterval: 5m
secretStoreRef:
name: aws-secrets-manager
kind: ClusterSecretStore
target:
name: api-credentials
creationPolicy: Owner
data:
- secretKey: api-key
remoteRef:
key: production/api-key
property: current
# Hot-reload secrets without restart (file watcher pattern)
import os
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
class SecretReloader(FileSystemEventHandler):
"""Watch mounted secret files for changes and reload."""
def __init__(self, secret_path, callback):
self.secret_path = secret_path
self.callback = callback
def on_modified(self, event):
if event.src_path == self.secret_path:
with open(self.secret_path) as f:
new_value = f.read().strip()
self.callback(new_value)
# Usage with Kubernetes mounted secrets
secret_path = "/var/run/secrets/api-key"
observer = Observer()
observer.schedule(
SecretReloader(secret_path, update_api_key),
path=os.path.dirname(secret_path)
)
observer.start()
---
Step 5 — Rotation Monitoring and Alerting
# Prometheus alerting rules
groups:
- name: secret-rotation-alerts
rules:
- alert: SecretRotationOverdue
expr: |
time() - secret_last_rotated_timestamp > 86400 * 45
for: 1h
labels:
severity: warning
annotations:
summary: "Secret not rotated in 45+ days"
- alert: OldKeyStillInUse
expr: |
sum(rate(api_auth_total{key_version="previous"}[5m])) > 0
for: 30m
labels:
severity: warning
annotations:
summary: "Previous API key still receiving traffic"
- alert: RotationLambdaFailed
expr: |
aws_lambda_errors_total{function="secret-rotation"} > 0
for: 5m
labels:
severity: critical
annotations:
summary: "Secret rotation Lambda is failing"
#!/bin/bash
# rotation-audit.sh — Check rotation status across all secrets
echo "=== Secrets Rotation Audit ==="
aws secretsmanager list-secrets --query 'SecretList[].{
Name:Name,
LastRotated:LastRotatedDate,
RotationEnabled:RotationEnabled,
NextRotation:NextRotationDate
}' --output table
---
Frequently Asked Questions
How long should the overlap window be between old and new keys?
At minimum, long enough for all consumers to pick up the new key. For Kubernetes with External Secrets Operator (5-min refresh), allow 15-30 minutes. For manual deployments, allow 1-2 hours. Track usage of old vs new keys in metrics to confirm rollover.
What if a consumer misses the rotation window?
This is why monitoring is critical. Alert when the old key is still being used after the expected overlap window. The provider should log which key version authenticated each request, making it easy to identify lagging consumers before revoking the old key.
Should I use Vault or AWS Secrets Manager?
Use AWS Secrets Manager if you are AWS-native and need simple rotation for AWS service credentials (RDS, Redshift). Use HashiCorp Vault for multi-cloud, on-prem, or when you need dynamic short-lived credentials. Vault has more complexity but more flexibility.
How do I rotate keys for third-party APIs (Stripe, Twilio)?
Most third-party APIs support multiple active keys. Create the new key in their dashboard, update your secrets store, verify the new key works, then delete the old key from the third-party dashboard. Automate this with their APIs where available.
What is the recommended rotation frequency?
For API keys: every 30-90 days depending on risk level. For database credentials with Vault: use dynamic secrets with 1-24 hour TTLs (no manual rotation needed). For service-to-service mTLS: certificates with 24-72 hour lifetimes via cert-manager.
---