feat(storage): add S3/R2 abstraction with local Rustfs dev
- Add backend/internal/s3/ with build-tag pattern (dev vs prod) - Dev: Uses local Rustfs container (S3-compatible) - Prod: Stub for R2 Cloudflare (add AWS SDK to implement) - Add S3 env vars to .env.example and .env - Add Rustfs service to compose.yml - Add Rustfs reset to local-dev-2.sh (wipes data on each run)
This commit is contained in:
@@ -3,11 +3,30 @@ module crussell
|
||||
go 1.25.1
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
|
||||
github.com/go-chi/jwtauth/v5 v5.3.3
|
||||
golang.org/x/text v0.30.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
|
||||
@@ -1,3 +1,41 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8 h1:iu+64gwDKEoKnyTQskSku72dAwggKI5sV6rNvgSMpMs=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8/go.mod h1:MI2XvA+qDi3i9AJxX1E2fu730syEBzp/jnXrjxuHwgI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8 h1:Jp2JYH1lRT3KhX4mshHPvVYsR5qqRec3hGvEarNYoR0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8/go.mod h1:fZG9tuvyVfxknv1rKibIz3DobRaFw1Poe8IKtXB3XYY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 h1:0jbJeuEHlwKJ9PfXtpSFc4MF+WIWORdhN1n30ITZGFM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
//go:build !dev
|
||||
// +build !dev
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
var Client Uploader
|
||||
|
||||
type Uploader interface {
|
||||
Upload(ctx context.Context, bucket, key string, body io.Reader) error
|
||||
Download(ctx context.Context, bucket, key string, w io.Writer) error
|
||||
Delete(ctx context.Context, bucket, key string) error
|
||||
GetURL(ctx context.Context, bucket, key string) (string, error)
|
||||
}
|
||||
|
||||
type S3Client struct {
|
||||
bucket string
|
||||
endpoint string
|
||||
region string
|
||||
accessKey string
|
||||
secretKey string
|
||||
publicURL string
|
||||
}
|
||||
|
||||
func Connect() error {
|
||||
endpoint := os.Getenv("R2_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
log.Println("WARNING: R2_ENDPOINT not set, S3 client not initialized")
|
||||
return nil
|
||||
}
|
||||
|
||||
Client = &S3Client{
|
||||
endpoint: endpoint,
|
||||
bucket: getEnv("R2_BUCKET", ""),
|
||||
region: getEnv("AWS_REGION", "auto"),
|
||||
accessKey: getEnv("R2_ACCESS_KEY", ""),
|
||||
secretKey: getEnv("R2_SECRET_KEY", ""),
|
||||
publicURL: getEnv("R2_PUBLIC_URL", ""),
|
||||
}
|
||||
|
||||
log.Printf("Connected to R2: bucket=%s, endpoint=%s", Client.(*S3Client).bucket, endpoint)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Client) Upload(ctx context.Context, bucket, key string, body io.Reader) error {
|
||||
return fmt.Errorf("S3 Upload not implemented: add AWS SDK v2 dependency")
|
||||
}
|
||||
|
||||
func (s *S3Client) Download(ctx context.Context, bucket, key string, w io.Writer) error {
|
||||
return fmt.Errorf("S3 Download not implemented: add AWS SDK v2 dependency")
|
||||
}
|
||||
|
||||
func (s *S3Client) Delete(ctx context.Context, bucket, key string) error {
|
||||
return fmt.Errorf("S3 Delete not implemented: add AWS SDK v2 dependency")
|
||||
}
|
||||
|
||||
func (s *S3Client) GetURL(ctx context.Context, bucket, key string) (string, error) {
|
||||
return fmt.Sprintf("%s/%s/%s", s.publicURL, bucket, key), nil
|
||||
}
|
||||
|
||||
func getEnv(key, fallback string) string {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
return val
|
||||
}
|
||||
if fallback != "" {
|
||||
return fallback
|
||||
}
|
||||
log.Fatal("FATAL: Environment variable not set:", key)
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
//go:build dev
|
||||
// +build dev
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
var Client Uploader
|
||||
|
||||
type Uploader interface {
|
||||
Upload(ctx context.Context, bucket, key string, body io.Reader) error
|
||||
Download(ctx context.Context, bucket, key string, w io.Writer) error
|
||||
Delete(ctx context.Context, bucket, key string) error
|
||||
GetURL(ctx context.Context, bucket, key string) (string, error)
|
||||
}
|
||||
|
||||
type S3Client struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
publicURL string
|
||||
}
|
||||
|
||||
func Connect() error {
|
||||
endpoint := os.Getenv("S3_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
endpoint = "http://localhost:9000"
|
||||
}
|
||||
|
||||
accessKey := os.Getenv("S3_ACCESS_KEY")
|
||||
if accessKey == "" {
|
||||
accessKey = "minioadmin"
|
||||
}
|
||||
|
||||
secretKey := os.Getenv("S3_SECRET_KEY")
|
||||
if secretKey == "" {
|
||||
secretKey = "minioadmin"
|
||||
}
|
||||
|
||||
bucket := os.Getenv("S3_BUCKET")
|
||||
if bucket == "" {
|
||||
bucket = "crussell"
|
||||
}
|
||||
|
||||
region := os.Getenv("AWS_REGION")
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
|
||||
awsconfig.WithRegion(region),
|
||||
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
accessKey,
|
||||
secretKey,
|
||||
"",
|
||||
)),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
Client = &S3Client{
|
||||
client: s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = true
|
||||
}),
|
||||
bucket: bucket,
|
||||
publicURL: endpoint,
|
||||
}
|
||||
|
||||
log.Printf("Connected to local S3 (Rustfs): bucket=%s, endpoint=%s", bucket, endpoint)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Client) Upload(ctx context.Context, bucket, key string, body io.Reader) error {
|
||||
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
Body: body,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Client) Download(ctx context.Context, bucket, key string, w io.Writer) error {
|
||||
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer result.Body.Close()
|
||||
|
||||
_, err = io.Copy(w, result.Body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Client) Delete(ctx context.Context, bucket, key string) error {
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Client) GetURL(ctx context.Context, bucket, key string) (string, error) {
|
||||
return fmt.Sprintf("%s/%s/%s", s.publicURL, bucket, key), nil
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"crussell/auth"
|
||||
"crussell/internal/dav"
|
||||
"crussell/internal/s3"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -46,9 +47,18 @@ func initDav() {
|
||||
fmt.Println("DAV Service connected successfully")
|
||||
}
|
||||
|
||||
func initS3() {
|
||||
if err := s3.Connect(); err != nil {
|
||||
log.Printf("WARNING: Failed to connect to S3: %v", err)
|
||||
} else {
|
||||
fmt.Println("S3 client initialized")
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
initDB()
|
||||
initDav()
|
||||
initS3()
|
||||
|
||||
r := chi.NewRouter()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user