TL;DR Quick Fix
Find and fix your biggest storage costs immediately:
# Find the top 10 most expensive S3 buckets
aws s3api list-buckets --query 'Buckets[].Name' --output text | \
tr '\t' '\n' | while read bucket; do
SIZE=$(aws s3 ls s3://$bucket --recursive --summarize | tail -1 | awk '{print $3}')
echo "$SIZE $bucket"
done | sort -rn | head -10
# Apply immediate lifecycle rule to expire old artifacts
aws s3api put-bucket-lifecycle-configuration \
--bucket ci-artifacts-prod \
--lifecycle-configuration file://lifecycle-policy.json
{
"Rules": [
{
"ID": "expire-old-artifacts",
"Status": "Enabled",
"Filter": { "Prefix": "builds/" },
"Expiration": { "Days": 30 },
"NoncurrentVersionExpiration": { "NoncurrentDays": 7 }
},
{
"ID": "transition-to-glacier",
"Status": "Enabled",
"Filter": { "Prefix": "releases/" },
"Transitions": [
{ "Days": 90, "StorageClass": "GLACIER" }
]
}
]
}
---
Architecture Overview
---
Identifying Storage Bloat
S3 Storage Analysis Script
#!/bin/bash
# s3-storage-audit.sh
set -euo pipefail
echo "=== S3 Storage Audit Report ==="
echo "Date: $(date -u +%Y-%m-%d)"
for bucket in $(aws s3api list-buckets --query 'Buckets[].Name' --output text); do
SIZE_BYTES=$(aws cloudwatch get-metric-statistics \
--namespace AWS/S3 \
--metric-name BucketSizeBytes \
--dimensions Name=BucketName,Value=$bucket Name=StorageType,Value=StandardStorage \
--start-time $(date -u -v-1d +%Y-%m-%dT%H:%M:%S) \
--end-time $(date -u +%Y-%m-%dT%H:%M:%S) \
--period 86400 \
--statistics Average \
--query 'Datapoints[0].Average' --output text 2>/dev/null)
if [ "$SIZE_BYTES" != "None" ] && [ -n "$SIZE_BYTES" ]; then
SIZE_GB=$(echo "scale=2; $SIZE_BYTES / 1073741824" | bc)
COST=$(echo "scale=2; $SIZE_GB * 0.023" | bc)
echo "$SIZE_GB GB | \$$COST/mo | $bucket"
fi
done | sort -rn | head -20
Find Old Artifacts by Prefix
# Find artifacts older than 60 days
aws s3api list-objects-v2 \
--bucket ci-artifacts-prod \
--prefix "builds/" \
--query "Contents[?LastModified<='$(date -u -v-60d +%Y-%m-%d)'].{Key:Key,Size:Size,Modified:LastModified}" \
--output table
# Count objects and total size by prefix
aws s3 ls s3://ci-artifacts-prod/builds/ --recursive --summarize
---
S3 Lifecycle Rules for Auto-Deletion
# terraform/s3-lifecycle.tf
resource "aws_s3_bucket_lifecycle_configuration" "artifacts" {
bucket = aws_s3_bucket.ci_artifacts.id
rule {
id = "expire-ci-builds"
status = "Enabled"
filter {
prefix = "builds/"
}
expiration {
days = 14
}
noncurrent_version_expiration {
noncurrent_days = 3
}
}
rule {
id = "archive-releases"
status = "Enabled"
filter {
prefix = "releases/"
}
transition {
days = 30
storage_class = "STANDARD_IA"
}
transition {
days = 90
storage_class = "GLACIER"
}
expiration {
days = 365
}
}
rule {
id = "cleanup-multipart"
status = "Enabled"
filter {
prefix = ""
}
abort_incomplete_multipart_upload {
days_after_initiation = 7
}
}
}
---
ECR Image Lifecycle Policies
# Apply ECR lifecycle policy to keep only recent images
aws ecr put-lifecycle-policy \
--repository-name my-app \
--lifecycle-policy-text file://ecr-lifecycle.json
{
"rules": [
{
"rulePriority": 1,
"description": "Keep last 5 production-tagged images",
"selection": {
"tagStatus": "tagged",
"tagPrefixList": ["prod-", "release-"],
"countType": "imageCountMoreThan",
"countNumber": 5
},
"action": { "type": "expire" }
},
{
"rulePriority": 2,
"description": "Expire untagged images after 3 days",
"selection": {
"tagStatus": "untagged",
"countType": "sinceImagePushed",
"countUnit": "days",
"countNumber": 3
},
"action": { "type": "expire" }
},
{
"rulePriority": 3,
"description": "Keep last 20 dev images",
"selection": {
"tagStatus": "tagged",
"tagPrefixList": ["dev-", "feature-"],
"countType": "imageCountMoreThan",
"countNumber": 20
},
"action": { "type": "expire" }
}
]
}
---
Nexus/JFrog Cleanup Policies
JFrog Artifactory Cleanup
# artifactory-cleanup.yml
apiVersion: cleanup/v1
kind: CleanupPolicy
metadata:
name: ci-snapshot-cleanup
spec:
repos:
- libs-snapshot-local
- docker-local
criteria:
lastDownloadedBefore: 30d
lastCreatedBefore: 14d
actions:
delete: true
dryRun: false
schedule: "0 2 0" # Every Sunday at 2 AM
# JFrog CLI cleanup for Docker repositories
jfrog rt delete "docker-local//builds/" \
--spec-vars "created-before=30d" \
--quiet
# Nexus cleanup via REST API
curl -X POST "https://nexus.company.com/service/rest/v1/tasks/run" \
-H "Content-Type: application/json" \
-d '{
"id": "cleanup-maven-snapshots",
"type": "repository.maven2.remove-snapshots",
"properties": {
"repositoryName": "maven-snapshots",
"minimumRetained": "3",
"snapshotRetentionDays": "14"
}
}'
---
GitHub Actions Artifact Retention
# .github/workflows/build.yml
- name: Upload build artifact
uses: actions/upload-artifact@v4
with:
name: build-output-${{ github.sha }}
path: dist/
retention-days: 5 # Default is 90 days!
compression-level: 9
# Delete old workflow runs and their artifacts
gh run list --limit 100 --status completed \
--json databaseId,createdAt \
--jq '.[] | select(.createdAt < "2024-01-01") | .databaseId' | \
xargs -I {} gh run delete {}
---
Storage Cost Monitoring
#!/usr/bin/env python3
# storage_cost_alert.py - Monitor and alert on storage cost anomalies
import boto3
from datetime import datetime, timedelta
def check_storage_costs():
ce = boto3.client('ce')
end_date = datetime.now().strftime('%Y-%m-%d')
start_date = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')
response = ce.get_cost_and_usage(
TimePeriod={'Start': start_date, 'End': end_date},
Granularity='DAILY',
Metrics=['UnblendedCost'],
Filter={
'Dimensions': {
'Key': 'SERVICE',
'Values': [
'Amazon Simple Storage Service',
'Amazon Elastic Container Registry Public'
]
}
},
GroupBy=[{'Type': 'DIMENSION', 'Key': 'SERVICE'}]
)
total_cost = sum(
float(day['Total']['UnblendedCost']['Amount'])
for day in response['ResultsByTime']
)
daily_avg = total_cost / 7
monthly_projected = daily_avg * 30
if monthly_projected > 500:
print(f"ALERT: projected ${monthly_projected:.2f}/mo")
return monthly_projected
if __name__ == "__main__":
projected = check_storage_costs()
print(f"Projected monthly storage cost: ${projected:.2f}")
---
FAQ
How much can I realistically save with lifecycle rules?
Most teams see 60-85% reduction in artifact storage costs after implementing lifecycle rules. The biggest wins come from CI build artifacts (which are rarely accessed after 7 days) and untagged container images that accumulate silently.
Will deleting old artifacts break anything?
Not if you separate build artifacts from release artifacts. Build outputs (test results, intermediate files) are safe to expire after 7-14 days. Release artifacts should be archived to Glacier, not deleted, in case you need to reproduce a production build.
How do I prevent storage bloat from recurring?
Set retention policies at the source: GitHub Actions retention-days, ECR lifecycle policies, and S3 lifecycle rules should be part of your infrastructure-as-code. Add a storage cost alert that fires when monthly costs exceed a threshold.
Should I use S3 Intelligent-Tiering instead of lifecycle rules?
Intelligent-Tiering adds a per-object monitoring fee ($0.0025/1000 objects). For CI artifacts with predictable access patterns (hot for 1-2 days, then never), explicit lifecycle rules are cheaper. Use Intelligent-Tiering for release artifacts where access patterns are unpredictable.
How do I clean up Terraform state file bloat?
Terraform state files grow with version history in S3. Enable versioning lifecycle rules to keep only the last 10 versions and expire older ones after 30 days. Also run terraform state rm for resources that were manually deleted.
---