From dfd552b02f6b65391ec67cee138247e26c49efcc Mon Sep 17 00:00:00 2001 From: Stephen Adamson Date: Tue, 17 Feb 2026 22:47:38 +0000 Subject: [PATCH] feat(storage): add S3/R2 abstraction with local Rustfs dev - Add backend/internal/s3/ with build-tag pattern (dev vs prod) - Dev: Uses local Rustfs container (S3-compatible) - Prod: Stub for R2 Cloudflare (add AWS SDK to implement) - Add S3 env vars to .env.example and .env - Add Rustfs service to compose.yml - Add Rustfs reset to local-dev-2.sh (wipes data on each run) --- .env.example | 16 +++++ backend/go.mod | 19 ++++++ backend/go.sum | 38 +++++++++++ backend/internal/s3/s3.go | 77 ++++++++++++++++++++++ backend/internal/s3/s3_dev.go | 118 ++++++++++++++++++++++++++++++++++ backend/main.go | 10 +++ compose.yml | 19 ++++++ local-dev-2.sh | 7 ++ 8 files changed, 304 insertions(+) create mode 100644 backend/internal/s3/s3.go create mode 100644 backend/internal/s3/s3_dev.go diff --git a/.env.example b/.env.example index 4142bfa..0655578 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,19 @@ POSTGRES_HOST=postgres POSTGRES_PORT=5432 JWT_SECRET_KEY="a-very-secret-key-that-should-be-in-env" + +# S3/R2 Configuration (for image storage) +# Dev: Uses local Rustfs container (see compose.yml) +# Prod: Use Cloudflare R2 credentials +S3_ENDPOINT=http://localhost:9000 +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +S3_BUCKET=crussell + +# Prod: Uncomment and fill these for R2 +# R2_ENDPOINT=https://.r2.cloudflarestorage.com +# R2_ACCESS_KEY= +# R2_SECRET_KEY= +# R2_BUCKET=crussell +# R2_PUBLIC_URL=https://pub-.r2.dev +# AWS_REGION=auto diff --git a/backend/go.mod b/backend/go.mod index c6d2a13..fded932 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,11 +3,30 @@ module crussell go 1.25.1 require ( + github.com/aws/aws-sdk-go-v2 v1.41.1 + github.com/aws/aws-sdk-go-v2/config v1.32.8 + github.com/aws/aws-sdk-go-v2/credentials v1.19.8 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 github.com/go-chi/jwtauth/v5 v5.3.3 golang.org/x/text v0.30.0 ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect diff --git a/backend/go.sum b/backend/go.sum index a6520d7..b1cd889 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,3 +1,41 @@ +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2/config v1.32.8 h1:iu+64gwDKEoKnyTQskSku72dAwggKI5sV6rNvgSMpMs= +github.com/aws/aws-sdk-go-v2/config v1.32.8/go.mod h1:MI2XvA+qDi3i9AJxX1E2fu730syEBzp/jnXrjxuHwgI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.8 h1:Jp2JYH1lRT3KhX4mshHPvVYsR5qqRec3hGvEarNYoR0= +github.com/aws/aws-sdk-go-v2/credentials v1.19.8/go.mod h1:fZG9tuvyVfxknv1rKibIz3DobRaFw1Poe8IKtXB3XYY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 h1:0jbJeuEHlwKJ9PfXtpSFc4MF+WIWORdhN1n30ITZGFM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/backend/internal/s3/s3.go b/backend/internal/s3/s3.go new file mode 100644 index 0000000..10c98f5 --- /dev/null +++ b/backend/internal/s3/s3.go @@ -0,0 +1,77 @@ +//go:build !dev +// +build !dev + +package s3 + +import ( + "context" + "fmt" + "io" + "log" + "os" +) + +var Client Uploader + +type Uploader interface { + Upload(ctx context.Context, bucket, key string, body io.Reader) error + Download(ctx context.Context, bucket, key string, w io.Writer) error + Delete(ctx context.Context, bucket, key string) error + GetURL(ctx context.Context, bucket, key string) (string, error) +} + +type S3Client struct { + bucket string + endpoint string + region string + accessKey string + secretKey string + publicURL string +} + +func Connect() error { + endpoint := os.Getenv("R2_ENDPOINT") + if endpoint == "" { + log.Println("WARNING: R2_ENDPOINT not set, S3 client not initialized") + return nil + } + + Client = &S3Client{ + endpoint: endpoint, + bucket: getEnv("R2_BUCKET", ""), + region: getEnv("AWS_REGION", "auto"), + accessKey: getEnv("R2_ACCESS_KEY", ""), + secretKey: getEnv("R2_SECRET_KEY", ""), + publicURL: getEnv("R2_PUBLIC_URL", ""), + } + + log.Printf("Connected to R2: bucket=%s, endpoint=%s", Client.(*S3Client).bucket, endpoint) + return nil +} + +func (s *S3Client) Upload(ctx context.Context, bucket, key string, body io.Reader) error { + return fmt.Errorf("S3 Upload not implemented: add AWS SDK v2 dependency") +} + +func (s *S3Client) Download(ctx context.Context, bucket, key string, w io.Writer) error { + return fmt.Errorf("S3 Download not implemented: add AWS SDK v2 dependency") +} + +func (s *S3Client) Delete(ctx context.Context, bucket, key string) error { + return fmt.Errorf("S3 Delete not implemented: add AWS SDK v2 dependency") +} + +func (s *S3Client) GetURL(ctx context.Context, bucket, key string) (string, error) { + return fmt.Sprintf("%s/%s/%s", s.publicURL, bucket, key), nil +} + +func getEnv(key, fallback string) string { + if val := os.Getenv(key); val != "" { + return val + } + if fallback != "" { + return fallback + } + log.Fatal("FATAL: Environment variable not set:", key) + return "" +} diff --git a/backend/internal/s3/s3_dev.go b/backend/internal/s3/s3_dev.go new file mode 100644 index 0000000..7e97fa8 --- /dev/null +++ b/backend/internal/s3/s3_dev.go @@ -0,0 +1,118 @@ +//go:build dev +// +build dev + +package s3 + +import ( + "context" + "fmt" + "io" + "log" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +var Client Uploader + +type Uploader interface { + Upload(ctx context.Context, bucket, key string, body io.Reader) error + Download(ctx context.Context, bucket, key string, w io.Writer) error + Delete(ctx context.Context, bucket, key string) error + GetURL(ctx context.Context, bucket, key string) (string, error) +} + +type S3Client struct { + client *s3.Client + bucket string + publicURL string +} + +func Connect() error { + endpoint := os.Getenv("S3_ENDPOINT") + if endpoint == "" { + endpoint = "http://localhost:9000" + } + + accessKey := os.Getenv("S3_ACCESS_KEY") + if accessKey == "" { + accessKey = "minioadmin" + } + + secretKey := os.Getenv("S3_SECRET_KEY") + if secretKey == "" { + secretKey = "minioadmin" + } + + bucket := os.Getenv("S3_BUCKET") + if bucket == "" { + bucket = "crussell" + } + + region := os.Getenv("AWS_REGION") + if region == "" { + region = "us-east-1" + } + + awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion(region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + accessKey, + secretKey, + "", + )), + ) + if err != nil { + return fmt.Errorf("failed to load AWS config: %w", err) + } + + Client = &S3Client{ + client: s3.NewFromConfig(awsCfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = true + }), + bucket: bucket, + publicURL: endpoint, + } + + log.Printf("Connected to local S3 (Rustfs): bucket=%s, endpoint=%s", bucket, endpoint) + return nil +} + +func (s *S3Client) Upload(ctx context.Context, bucket, key string, body io.Reader) error { + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: body, + }) + return err +} + +func (s *S3Client) Download(ctx context.Context, bucket, key string, w io.Writer) error { + result, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + return err + } + defer result.Body.Close() + + _, err = io.Copy(w, result.Body) + return err +} + +func (s *S3Client) Delete(ctx context.Context, bucket, key string) error { + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + return err +} + +func (s *S3Client) GetURL(ctx context.Context, bucket, key string) (string, error) { + return fmt.Sprintf("%s/%s/%s", s.publicURL, bucket, key), nil +} diff --git a/backend/main.go b/backend/main.go index e615766..061371b 100644 --- a/backend/main.go +++ b/backend/main.go @@ -3,6 +3,7 @@ package main import ( "crussell/auth" "crussell/internal/dav" + "crussell/internal/s3" "fmt" "log" "net/http" @@ -46,9 +47,18 @@ func initDav() { fmt.Println("DAV Service connected successfully") } +func initS3() { + if err := s3.Connect(); err != nil { + log.Printf("WARNING: Failed to connect to S3: %v", err) + } else { + fmt.Println("S3 client initialized") + } +} + func main() { initDB() initDav() + initS3() r := chi.NewRouter() diff --git a/compose.yml b/compose.yml index 1c427f7..7ffecef 100644 --- a/compose.yml +++ b/compose.yml @@ -59,19 +59,38 @@ services: ports: - "80:80" - "443:443" + - "9000:9000" # Rustfs S3 API volumes: - ./nginx/conf.d:/etc/nginx/conf.d - ./nginx/certs:/etc/nginx/certs - ./frontend/build:/usr/share/nginx/html # Serve frontend - ./sabredav:/var/www/dav + - rustfs_data:/data depends_on: - backend - sabredav networks: - appnet + rustfs: + image: rustfs/rustfs:latest + container_name: rustfs + restart: always + ports: + - "9001:9000" # Web console + environment: + - RUSTFS_ADMIN_ACCESS_KEY=minioadmin + - RUSTFS_ADMIN_SECRET_KEY=minioadmin + volumes: + - rustfs_data:/data + networks: + - appnet + command: > + bash -c "rustfs server /data --console-address :9000 --address :9000 --axum-server" + volumes: pgdata: + rustfs_data: networks: diff --git a/local-dev-2.sh b/local-dev-2.sh index 56de4ae..3314e6d 100755 --- a/local-dev-2.sh +++ b/local-dev-2.sh @@ -43,6 +43,13 @@ docker compose up postgres -d > /dev/null 2>&1 log_success "PostgreSQL reset complete" sleep 3 +# --- 3b. Rustfs Reset (wipes data each run) --- +log_step "Resetting Rustfs (S3-compatible storage)..." +docker compose down -v rustfs > /dev/null 2>&1 +docker compose up rustfs -d > /dev/null 2>&1 +log_success "Rustfs reset complete" +sleep 2 + # --- 4. Tmux Session Setup --- if tmux has-session -t $SESSION_NAME 2>/dev/null; then log_info "Killing existing tmux session..."