Skip to content

Commit 2cd4a5d

Browse files
allan-thatchallan-thatch
authored
Allow CMKs and bucket keys (#1)
Add support for using customer-managed KMS keys and bucket keys for encryption. This is a fork version of render-examples#2.
1 parent 4fbaf01 commit 2cd4a5d

File tree

10 files changed

+245
-28
lines changed

10 files changed

+245
-28
lines changed

‎README.md‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -67,18 +67,21 @@ terraform apply \
6767

6868
## Terraform Variables
6969

70-
| Variable | Required | Default | Description |
71-
| ------------------------- | -------- | ---------------------------- | ------------------------------------------------------ |
72-
| `aws_s3_bucket_name` | Yes | - | Name of the S3 bucket to create for storing audit logs |
73-
| `render_api_key` | Yes | - | Render API key for accessing audit logs |
74-
| `render_workspace_ids` | No | `[]` | List of workspace IDs to fetch audit logs from |
75-
| `render_organization_id` | No | `""` | Organization ID for Enterprise audit logs |
76-
| `aws_iam_user_name` | No | `render-audit-log-processor` | Name of the IAM user created for S3 access |
77-
| `render_cronjob_name` | No | `render-auditlogs` | Name of the Render Cron Job |
78-
| `render_cronjob_schedule` | No | `1/15 * * * *` | Cron schedule (default: every 15 minutes) |
79-
| `render_cronjob_plan` | No | `starter` | Render plan for the Cron Job |
80-
| `render_cronjob_region` | No | `oregon` | Region to deploy the Cron Job |
81-
| `render_project_name` | No | `audit-logs` | Name of the Render project |
70+
| Variable | Required | Default | Description |
71+
| --------------------------- | -------- | ---------------------------- | ------------------------------------------------------ |
72+
| `aws_s3_bucket_name` | Yes | - | Name of the S3 bucket to create for storing audit logs |
73+
| `render_api_key` | Yes | - | Render API key for accessing audit logs |
74+
| `render_workspace_ids` | No | `[]` | List of workspace IDs to fetch audit logs from |
75+
| `render_organization_id` | No | `""` | Organization ID for Enterprise audit logs |
76+
| `aws_iam_user_name` | No | `render-audit-log-processor` | Name of the IAM user created for S3 access |
77+
| `aws_s3_bucket_key_enabled` | No | `false` | Enable S3 bucket key to reduce KMS calls |
78+
| `aws_s3_kms_key_id` | No | `""` | ARN for KMS key to use for encryption |
79+
| `aws_s3_use_kms` | No | `false` | Use KMS for encryption (instead of SSE-S3) |
80+
| `render_cronjob_name` | No | `render-auditlogs` | Name of the Render Cron Job |
81+
| `render_cronjob_schedule` | No | `1/15 * * * *` | Cron schedule (default: every 15 minutes) |
82+
| `render_cronjob_plan` | No | `starter` | Render plan for the Cron Job |
83+
| `render_cronjob_region` | No | `oregon` | Region to deploy the Cron Job |
84+
| `render_project_name` | No | `audit-logs` | Name of the Render project |
8285

8386
## Architecture
8487

@@ -108,6 +111,11 @@ RENDER_API_KEY=your-api-key
108111
AWS_ACCESS_KEY_ID=your-aws-key
109112
AWS_SECRET_ACCESS_KEY=your-aws-secret
110113
AWS_REGION=us-west-2
114+
115+
# Optional: KMS encryption settings (defaults to SSE-S3 if not set)
116+
S3_USE_KMS=true
117+
S3_KMS_KEY_ID=arn:aws:kms:us-west-2:123456789012:key/your-key-id # Optional
118+
S3_BUCKET_KEY_ENABLED=true # Optional
111119
```
112120

113121
2. Run the application:

‎main.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,11 @@ func main() {
3030
}
3131

3232
// Create S3 uploader
33-
uploader, err := aws.NewUploader(ctx, s3.NewFromConfig(cfg.AWSConfig), cfg.S3Bucket, cfg.AWSRegion)
33+
uploader, err := aws.NewUploaderWithOptions(ctx, s3.NewFromConfig(cfg.AWSConfig), cfg.S3Bucket, cfg.AWSRegion, aws.UploaderOptions{
34+
UseKMS: cfg.S3UseKMS,
35+
KMSKeyID: cfg.S3KMSKeyID,
36+
BucketKeyEnabled: cfg.S3BucketKeyEnabled,
37+
})
3438
if err != nil {
3539
log.Fatal("Error creating S3 uploader:", err)
3640
}

‎pkg/aws/checkpoint.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,27 @@ func (u *Uploader) SaveCheckpoint(ctx context.Context, cp *Checkpoint, logType a
6060
return fmt.Errorf("error marshaling checkpoint: %w", err)
6161
}
6262

63-
_, err = u.client.PutObject(ctx, &s3.PutObjectInput{
64-
Bucket: aws.String(u.bucket),
65-
Key: aws.String(fmt.Sprintf("%s=%s/%s", logType, workspace, checkpointKey)),
66-
Body: bytes.NewReader(data),
67-
ContentType: aws.String("application/json"),
68-
ServerSideEncryption: types.ServerSideEncryptionAes256,
69-
})
63+
putInput := &s3.PutObjectInput{
64+
Bucket: aws.String(u.bucket),
65+
Key: aws.String(fmt.Sprintf("%s=%s/%s", logType, workspace, checkpointKey)),
66+
Body: bytes.NewReader(data),
67+
ContentType: aws.String("application/json"),
68+
}
69+
70+
// Configure server-side encryption
71+
if u.opts.UseKMS {
72+
putInput.ServerSideEncryption = types.ServerSideEncryptionAwsKms
73+
if u.opts.KMSKeyID != "" {
74+
putInput.SSEKMSKeyId = aws.String(u.opts.KMSKeyID)
75+
}
76+
if u.opts.BucketKeyEnabled {
77+
putInput.BucketKeyEnabled = aws.Bool(true)
78+
}
79+
} else {
80+
putInput.ServerSideEncryption = types.ServerSideEncryptionAes256
81+
}
82+
83+
_, err = u.client.PutObject(ctx, putInput)
7084
if err != nil {
7185
return fmt.Errorf("error writing checkpoint to S3: %w", err)
7286
}

‎pkg/aws/checkpoint_test.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,38 @@ func TestSaveCheckpoint(t *testing.T) {
158158
require.NoError(t, err)
159159
})
160160

161+
t.Run("uses KMS with key ID and bucket key enabled", func(t *testing.T) {
162+
checkpoint := &awspkg.Checkpoint{
163+
LastCursor: "kms-cursor",
164+
LastTimestamp: testTime,
165+
}
166+
167+
const kmsKey = "arn:aws:kms:us-west-2:123456789012:key/abcdefab-1234-5678-9abc-def012345678"
168+
169+
s3Client := &mockS3Client{
170+
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
171+
require.Equal(t, "test-bucket", *params.Bucket)
172+
require.Equal(t, "workspace=test-workspace/checkpoint.json", *params.Key)
173+
require.Equal(t, types.ServerSideEncryptionAwsKms, params.ServerSideEncryption)
174+
require.NotNil(t, params.SSEKMSKeyId)
175+
require.Equal(t, kmsKey, *params.SSEKMSKeyId)
176+
require.NotNil(t, params.BucketKeyEnabled)
177+
require.True(t, *params.BucketKeyEnabled)
178+
return &s3.PutObjectOutput{}, nil
179+
},
180+
}
181+
182+
uploader, err := awspkg.NewUploaderWithOptions(ctx, s3Client, "test-bucket", "test-region", awspkg.UploaderOptions{
183+
UseKMS: true,
184+
KMSKeyID: kmsKey,
185+
BucketKeyEnabled: true,
186+
})
187+
require.NoError(t, err)
188+
189+
err = uploader.SaveCheckpoint(ctx, checkpoint, auditlogs.WorkspaceAuditLog, "test-workspace")
190+
require.NoError(t, err)
191+
})
192+
161193
t.Run("returns error on S3 error", func(t *testing.T) {
162194
checkpoint := &awspkg.Checkpoint{
163195
LastCursor: "test-cursor",

‎pkg/aws/uploader.go‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,27 @@ type S3Client interface {
2121
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
2222
}
2323

24+
type UploaderOptions struct {
25+
UseKMS bool
26+
KMSKeyID string
27+
BucketKeyEnabled bool
28+
}
29+
2430
type Uploader struct {
2531
client S3Client
2632
bucket string
33+
opts UploaderOptions
2734
}
2835

2936
func NewUploader(ctx context.Context, client S3Client, bucket, region string) (*Uploader, error) {
37+
return NewUploaderWithOptions(ctx, client, bucket, region, UploaderOptions{})
38+
}
39+
40+
func NewUploaderWithOptions(ctx context.Context, client S3Client, bucket, region string, opts UploaderOptions) (*Uploader, error) {
3041
return &Uploader{
3142
client: client,
3243
bucket: bucket,
44+
opts: opts,
3345
}, nil
3446
}
3547

@@ -56,13 +68,28 @@ func (u *Uploader) UploadAuditLogs(ctx context.Context, auditLogType auditlogs.L
5668
key := generateS3Key(auditLogType, id, data[0].AuditLog.Timestamp)
5769

5870
// Upload to S3
59-
_, err = u.client.PutObject(ctx, &s3.PutObjectInput{
60-
Bucket: aws.String(u.bucket),
61-
Key: aws.String(key),
62-
Body: bytes.NewReader(compressedData.Bytes()),
63-
ContentType: aws.String("application/gzip"),
64-
ServerSideEncryption: types.ServerSideEncryptionAes256,
65-
})
71+
putInput := &s3.PutObjectInput{
72+
Bucket: aws.String(u.bucket),
73+
Key: aws.String(key),
74+
Body: bytes.NewReader(compressedData.Bytes()),
75+
ContentType: aws.String("application/gzip"),
76+
}
77+
78+
// Configure server-side encryption
79+
if u.opts.UseKMS {
80+
putInput.ServerSideEncryption = types.ServerSideEncryptionAwsKms
81+
if u.opts.KMSKeyID != "" {
82+
putInput.SSEKMSKeyId = aws.String(u.opts.KMSKeyID)
83+
}
84+
if u.opts.BucketKeyEnabled {
85+
putInput.BucketKeyEnabled = aws.Bool(true)
86+
}
87+
} else {
88+
// Default to SSE-S3 (AES256)
89+
putInput.ServerSideEncryption = types.ServerSideEncryptionAes256
90+
}
91+
92+
_, err = u.client.PutObject(ctx, putInput)
6693
if err != nil {
6794
return "", fmt.Errorf("error uploading to S3: %w", err)
6895
}

‎pkg/aws/uploader_test.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"time"
1313

1414
"github.com/aws/aws-sdk-go-v2/service/s3"
15+
"github.com/aws/aws-sdk-go-v2/service/s3/types"
1516
"github.com/stretchr/testify/require"
1617

1718
"github.com/renderinc/render-auditlogs/pkg/auditlogs"
@@ -108,4 +109,100 @@ func TestUploadAuditLogs(t *testing.T) {
108109
require.Contains(t, err.Error(), "error uploading to S3")
109110
require.Empty(t, s3URI)
110111
})
112+
113+
t.Run("uses default SSE-S3 encryption when KMS not enabled", func(t *testing.T) {
114+
t.Parallel()
115+
s3Client := &mockS3Client{
116+
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
117+
require.Equal(t, "test-bucket", *params.Bucket)
118+
require.Equal(t, types.ServerSideEncryptionAes256, params.ServerSideEncryption)
119+
require.Nil(t, params.SSEKMSKeyId)
120+
require.Nil(t, params.BucketKeyEnabled)
121+
return &s3.PutObjectOutput{}, nil
122+
},
123+
}
124+
125+
uploader, err := aws.NewUploaderWithOptions(ctx, s3Client, "test-bucket", "test-region", aws.UploaderOptions{
126+
UseKMS: false,
127+
})
128+
require.NoError(t, err)
129+
130+
s3URI, err := uploader.UploadAuditLogs(ctx, auditlogs.WorkspaceAuditLog, "workspace-123", testData)
131+
132+
require.NoError(t, err)
133+
require.NotEmpty(t, s3URI)
134+
})
135+
136+
t.Run("uses KMS encryption without specific key ID", func(t *testing.T) {
137+
t.Parallel()
138+
s3Client := &mockS3Client{
139+
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
140+
require.Equal(t, "test-bucket", *params.Bucket)
141+
require.Equal(t, types.ServerSideEncryptionAwsKms, params.ServerSideEncryption)
142+
require.Nil(t, params.SSEKMSKeyId)
143+
require.Nil(t, params.BucketKeyEnabled)
144+
return &s3.PutObjectOutput{}, nil
145+
},
146+
}
147+
148+
uploader, err := aws.NewUploaderWithOptions(ctx, s3Client, "test-bucket", "test-region", aws.UploaderOptions{
149+
UseKMS: true,
150+
})
151+
require.NoError(t, err)
152+
153+
s3URI, err := uploader.UploadAuditLogs(ctx, auditlogs.WorkspaceAuditLog, "workspace-123", testData)
154+
155+
require.NoError(t, err)
156+
require.NotEmpty(t, s3URI)
157+
})
158+
159+
t.Run("uses KMS encryption with specific key ID", func(t *testing.T) {
160+
t.Parallel()
161+
s3Client := &mockS3Client{
162+
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
163+
require.Equal(t, "test-bucket", *params.Bucket)
164+
require.Equal(t, types.ServerSideEncryptionAwsKms, params.ServerSideEncryption)
165+
require.NotNil(t, params.SSEKMSKeyId)
166+
require.Equal(t, "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012", *params.SSEKMSKeyId)
167+
require.Nil(t, params.BucketKeyEnabled)
168+
return &s3.PutObjectOutput{}, nil
169+
},
170+
}
171+
172+
uploader, err := aws.NewUploaderWithOptions(ctx, s3Client, "test-bucket", "test-region", aws.UploaderOptions{
173+
UseKMS: true,
174+
KMSKeyID: "arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012",
175+
})
176+
require.NoError(t, err)
177+
178+
s3URI, err := uploader.UploadAuditLogs(ctx, auditlogs.WorkspaceAuditLog, "workspace-123", testData)
179+
180+
require.NoError(t, err)
181+
require.NotEmpty(t, s3URI)
182+
})
183+
184+
t.Run("uses KMS encryption with bucket key enabled", func(t *testing.T) {
185+
t.Parallel()
186+
s3Client := &mockS3Client{
187+
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
188+
require.Equal(t, "test-bucket", *params.Bucket)
189+
require.Equal(t, types.ServerSideEncryptionAwsKms, params.ServerSideEncryption)
190+
require.Nil(t, params.SSEKMSKeyId)
191+
require.NotNil(t, params.BucketKeyEnabled)
192+
require.True(t, *params.BucketKeyEnabled)
193+
return &s3.PutObjectOutput{}, nil
194+
},
195+
}
196+
197+
uploader, err := aws.NewUploaderWithOptions(ctx, s3Client, "test-bucket", "test-region", aws.UploaderOptions{
198+
UseKMS: true,
199+
BucketKeyEnabled: true,
200+
})
201+
require.NoError(t, err)
202+
203+
s3URI, err := uploader.UploadAuditLogs(ctx, auditlogs.WorkspaceAuditLog, "workspace-123", testData)
204+
205+
require.NoError(t, err)
206+
require.NotEmpty(t, s3URI)
207+
})
111208
}

‎pkg/env/env.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ type Config struct {
1616
WorkspaceIDS []string `required:"true" split_words:"true"`
1717
OrganizationID string `required:"false" split_words:"true"`
1818
S3Bucket string `required:"true" split_words:"true"`
19+
S3BucketKeyEnabled bool `required:"false" split_words:"true"`
20+
S3KMSKeyID string `required:"false" split_words:"true"`
21+
S3UseKMS bool `required:"false" split_words:"true"`
1922
RenderAPIKey string `required:"true" split_words:"true"`
2023
AWSAccessKeyID string `required:"true" split_words:"true"`
2124
AWSSecretAccessKey string `required:"true" split_words:"true"`

‎terraform/modules/render-audit-logs/render.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ resource "render_cron_job" "render-audit-logs" {
2727
"WORKSPACE_IDS" = { value = join(",", var.render_workspace_ids) }
2828
"RENDER_API_KEY" = { value = var.render_api_key }
2929
"S3_BUCKET" = { value = var.aws_s3_bucket_name }
30+
"S3_BUCKET_KEY_ENABLED" = { value = var.aws_s3_bucket_key_enabled }
31+
"S3_KMS_KEY_ID" = { value = var.aws_s3_kms_key_id }
32+
"S3_USE_KMS" = { value = var.aws_s3_use_kms }
3033
}
3134
}
3235

‎terraform/modules/render-audit-logs/variables.tf‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,21 @@ variable "aws_s3_bucket_name" {
22
type = string
33
}
44

5+
variable "aws_s3_bucket_key_enabled" {
6+
type = bool
7+
default = false
8+
}
9+
10+
variable "aws_s3_kms_key_id" {
11+
type = string
12+
default = ""
13+
}
14+
15+
variable "aws_s3_use_kms" {
16+
type = bool
17+
default = false
18+
}
19+
520
variable "aws_access_key" {
621
type = string
722
sensitive = true

‎terraform/variables.tf‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,26 @@ variable "aws_s3_bucket_name" {
22
type = string
33
}
44

5+
variable "aws_s3_bucket_key_enabled" {
6+
type = bool
7+
default = false
8+
}
9+
10+
variable "aws_s3_kms_key_id" {
11+
type = string
12+
default = ""
13+
}
14+
15+
variable "aws_s3_use_kms" {
16+
type = bool
17+
default = false
18+
}
19+
520
variable "aws_iam_user_name" {
621
type = string
722
default = "render-audit-log-processor"
823
}
924

10-
1125
variable "render_api_key" {
1226
type = string
1327
sensitive = true

0 commit comments

Comments
 (0)