Web 直接上传 S3 技术方案验证 | Golang 实现

Published: 2024-08-21

Tags: Golang S3

本文总阅读量次

服务端接收用户上传文件时的一般方案是：“客户端 → 服务器 → S3”

这样带来一些问题，上传文件会占用服务器的 CPU、内存、带宽，大文件会占用磁盘存储空间，如果文件不需要进一步处理，一种优雅的方式是由客户端（e.g. Web 网页）直接上传文件到 S3，即： “客户端 → S3”

方式一：Pre-Signed URL 预签名

方案简介

服务端用已定义的 Key 向 S3 申请预签名 URL，这个 URL 带有失效时间，把它提供给客户端（e.g. Web 网页），可直接通过 HTTP PUT 请求，将数据直接以二进制流的方式上传。

适合上传单个大文件的上传

安全保证

已授权的临时 URL 暴露给外部上传，安全性保证有以下几点：

  • 服务端生成 Key 天然限定了资源 URI，客户端只能上传文件到指定位置
  • 服务端应该设置合理的 Timeout 超时时间，到达超时时间，S3 将中断上传，同时不会对文件进行保存
  • 服务端可以设置 Content-Type 和 Content-Length，这些信息会被签名到 Signed URL 中，客户端上传的内容不匹配时 S3 会跑出 403 错误

代码示例

以下代码模拟了获取 Pre-Signed URL、通过 Put 方式上传文件到 S3、上传成功后获取下载 Sign URL 的过程

package main

import (
    "bytes"
    "fmt"
    "net/http"
    "time"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/credentials"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

// GenerateUploadPresignedURL creates a presigned URL for uploading.
func GenerateUploadPresignedURL(svc *s3.S3, bucketName, objectKey string, expiration time.Duration) (string, error) {
    req, _ := svc.PutObjectRequest(&s3.PutObjectInput{
        Bucket: aws.String(bucketName),
        Key:    aws.String(objectKey),
        // ContentType:   aws.String("text/plain"), // "application/json"
        // ContentLength: aws.Int64(19),
    })

    // Generate presigned URL
    signedURL, err := req.Presign(expiration)
    if err != nil {
        return "", fmt.Errorf("error generating upload presigned URL: %w", err)
    }
    return signedURL, nil
}

// UploadContentToS3 uploads content to S3 using the provided presigned URL.
func UploadContentToS3(signedURL, content, contentType string) error {
    req, err := http.NewRequest("PUT", signedURL, bytes.NewBuffer([]byte(content)))
    if err != nil {
        return fmt.Errorf("error creating PUT request: %w", err)
    }
    req.Header.Set("Content-Type", contentType)

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        return fmt.Errorf("error executing PUT request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return fmt.Errorf("upload failed with status code %d", resp.StatusCode)
    }

    fmt.Println("Content uploaded successfully")
    return nil
}

// GenerateDownloadPresignedURL creates a presigned URL for downloading.
func GenerateDownloadPresignedURL(svc *s3.S3, bucketName, objectKey string, expiration time.Duration) (string, error) {
    req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
        Bucket: aws.String(bucketName),
        Key:    aws.String(objectKey),
    })

    // Generate presigned URL
    signedURL, err := req.Presign(expiration)
    if err != nil {
        return "", fmt.Errorf("error generating download presigned URL: %w", err)
    }
    return signedURL, nil
}

func GetContent() string {
    CNTimeLocation, err := time.LoadLocation("Asia/Shanghai")
    if err != nil {
        panic(err)
    }
    return time.Now().In(CNTimeLocation).Format("2006-01-02 15:04:05")
}

func main() {
    const (
        region     = "cn-northwest-1"
        bucketName = "<your-bucket-name>"
        objectKey  = "<s3-object-target-key>"
        expiration = 15 * time.Minute // URL expiration time
        accessKey  = "<your-access-key>"
        secretKey  = "<your-secret-key>"
    )

    // Initialize session with credentials
    sess, err := session.NewSession(&aws.Config{
        Region: aws.String(region),
        Credentials: credentials.NewStaticCredentials(
            accessKey, secretKey, "",
        ),
    })
    if err != nil {
        fmt.Printf("Error creating AWS session: %v\n", err)
        return
    }

    svc := s3.New(sess)

    // Generate and print upload presigned URL
    uploadURL, err := GenerateUploadPresignedURL(svc, bucketName, objectKey, expiration)
    if err != nil {
        fmt.Printf("Error generating upload presigned URL: %v\n", err)
        return
    }
    fmt.Printf("Upload URL: %s\n", uploadURL)

    // Upload content to S3
    content := GetContent()
    contentType := "text/plain"
    if err := UploadContentToS3(uploadURL, content, contentType); err != nil {
        fmt.Printf("Error uploading content: %v\n", err)
        return
    }

    // Generate and print download presigned URL
    downloadURL, err := GenerateDownloadPresignedURL(svc, bucketName, objectKey, expiration)
    if err != nil {
        fmt.Printf("Error generating download presigned URL: %v\n", err)
        return
    }
    fmt.Printf("Download URL: %s\n", downloadURL)
}

输出

值得注意的是 Put 方法申请的 URL 仅能用于上传，不能用于下载和查看，代码中申请的 GET 方法 Signed URL 可供查看和下载

交互流程

用户选择文件 → 前端计算 Content-Type、Content-Length、Content-MD5 等信息提交到服务器 → 服务器生成 Pre-Signed URL 返回给客户端 → 客户端上传文件到 S3 → 上传成功后，将上传成功信息通知到服务器

方式二：S3 URL + Signing Fields（Form Post）

方案简介

这是基于 AWS S3 独有特性的实现方式，优点是客户端使用更加灵活（可以给客户端更多自定义的权限）

由服务端生成 Form 字段键值（例如：X-Amz-Date、X-Amz-Signature），将这些信息返回给客户端，客户端提交 Post Form 表单到 S3 的时候携带

相较于 Pre-Signed URL Put 方式，Form 表单申请到的签名信息可以在有效期内复用，更适合多文件上传

补充 S3 不支持批量上传，如需多文件批量上传需由客户端实现，循环提交表单，这很适合批量上传小文件

相应的，批量上传场景在方式一中则比较繁琐，因为每提交一次文件都需要向服务器申请一次 Pre-Signed URL，客户端没办法自定义 Key 路径进行上传

Form Post 方式灵活的同时，因为使用了 AWS 的策略，其它 S3 兼容的存储无法使用这种方式，不够通用

安全保证

服务端可以通过策略限制上传文件的大小、类型为一个范围，例如：上传的文件需在 1 ～ 50MB 大小、文件类型需要是 image/* 类型等，同时，也需要设置合理的过期时间。

策略示例

这个策略是为 Amazon S3 使用的预签名 POST 上传协议编写的，它描述了允许上传操作必须满足的一系列条件，这种策略确保上传到 S3 存储桶的请求符合策略中指定的条件

{ "expiration": "2015-12-30T12:00:00.000Z",
  "conditions": [
    {"bucket": "sigv4examplebucket"},
    ["starts-with", "$key", "user/user1/"],
    {"acl": "public-read"},
    {"success_action_redirect": "http://sigv4examplebucket.s3.amazonaws.com/successful_upload.html"},
    ["starts-with", "$Content-Type", "image/"],
    {"x-amz-meta-uuid": "14365123651274"},
    {"x-amz-server-side-encryption": "AES256"},
    ["starts-with", "$x-amz-meta-tag", ""],

    {"x-amz-credential": "AKIAIOSFODNN7EXAMPLE/20151229/us-east-1/s3/aws4_request"},
    {"x-amz-algorithm": "AWS4-HMAC-SHA256"},
    {"x-amz-date": "20151229T000000Z" }
  ]
}

参考文档：https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-post-example.html

代码示例

需要使用 aws-sdk-go-v2 版本的 AWS SDK，Post 方式由 Form 参数指定传到那里、传什么文件

package main

import (
    "context"
    "crypto/hmac"
    "crypto/sha256"
    "encoding/base64"
    "encoding/hex"
    "encoding/json"
    "fmt"
    "log"
    "path/filepath"
    "time"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/credentials"
)

// AWS configuration constants
const (
    Region     = "cn-northwest-1"
    BucketName = "<your-bucket-name>"
    AccessKey  = "<your-access-key>"
    SecretKey  = "<your-secret-key>"
)

func main() {
    // Load AWS configuration
    cfg, err := loadAWSConfig(context.TODO())
    if err != nil {
        log.Fatalf("unable to load SDK config: %v", err)
    }

    // Define the object key
    keyPrefix := "s3-web-upload"

    // Set time values
    timeStamp := time.Now().UTC()
    shortDate, amzDate := formatDates(timeStamp)

    // Create the AWS credential string
    credential := fmt.Sprintf("%s/%s/%s/s3/aws4_request", AccessKey, shortDate, Region)

    // Create the policy
    expiration := timeStamp.Add(15 * time.Minute)
    policy := createPolicy(BucketName, keyPrefix, credential, amzDate, expiration)

    // Retrieve signing credentials
    creds, err := cfg.Credentials.Retrieve(context.TODO())
    if err != nil {
        log.Fatalf("unable to retrieve credentials: %v", err)
    }

    // Sign the policy
    signature, err := generateSignature(policy, creds, Region, shortDate)
    if err != nil {
        log.Fatalf("unable to sign policy: %v", err)
    }

    // Print the necessary fields for POST request
    printPostFields(BucketName, policy, creds, signature, credential, amzDate)

    // Generate and print the CURL command, upload txt
    objectKey := "s3-web-upload/test03-post-form/hello.txt"
    curlCommand := generateCURLCommand(BucketName, objectKey, "text/plain", policy, credential, amzDate, signature)
    fmt.Println("\nCURL Command:")
    fmt.Println(curlCommand)

    // Generate and print the CURL command, upload jpg
    objectKey = "s3-web-upload/test03-post-form/hello.jpg"
    curlCommand = generateCURLCommand(BucketName, objectKey, "image/jpeg", policy, credential, amzDate, signature)
    fmt.Println("\nCURL Command:")
    fmt.Println(curlCommand)
}

// loadAWSConfig loads AWS configuration using provided credentials and region
func loadAWSConfig(ctx context.Context) (aws.Config, error) {
    return config.LoadDefaultConfig(ctx,
        config.WithRegion(Region),
        config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(AccessKey, SecretKey, "")),
    )
}

// formatDates generates formatted date strings for use in AWS policies and signatures
func formatDates(timeStamp time.Time) (shortDate, amzDate string) {
    shortDate = timeStamp.Format("20060102")
    amzDate = timeStamp.Format("20060102T150405Z")
    return
}

type s3PostPolicy struct {
    Expiration string        `json:"expiration"`
    Conditions []interface{} `json:"conditions"`
}

// createPolicy generates a base64 encoded JSON policy document with given conditions
func createPolicy(bucket, keyPrefix, credential, amzDate string, expiration time.Time) string {
    conditions := []interface{}{
        map[string]string{"bucket": bucket},
        map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
        map[string]string{"x-amz-credential": credential},
        map[string]string{"x-amz-date": amzDate},
        []interface{}{"content-length-range", 0, 10485760}, // 10MB
        []interface{}{"starts-with", "$key", keyPrefix},
        []interface{}{"starts-with", "$Content-Type", ""}, // e.g. image/
    }

    policy := s3PostPolicy{
        Expiration: expiration.UTC().Format(time.RFC3339),
        Conditions: conditions,
    }

    policyBytes, _ := json.Marshal(policy)
    return base64.StdEncoding.EncodeToString(policyBytes)
}

// generateSignature creates an AWS v4 signature for the provided policy
func generateSignature(policy string, creds aws.Credentials, region, date string) (string, error) {
    signingKey := deriveSigningKey(creds.SecretAccessKey, date, region, "s3")
    signature := hmacSHA256(signingKey, policy)
    return hex.EncodeToString(signature), nil
}

// deriveSigningKey derives the signing key used for AWS signature v4
func deriveSigningKey(secret, date, region, service string) []byte {
    kDate := hmacSHA256([]byte("AWS4"+secret), date)
    kRegion := hmacSHA256(kDate, region)
    kService := hmacSHA256(kRegion, service)
    return hmacSHA256(kService, "aws4_request")
}

// hmacSHA256 performs HMAC-SHA256 hashing algorithm with given key and data
func hmacSHA256(key []byte, data string) []byte {
    h := hmac.New(sha256.New, key)
    h.Write([]byte(data))
    return h.Sum(nil)
}

// printPostFields displays the constructed fields needed for S3 POST requests
func printPostFields(bucketName, policy string, creds aws.Credentials, signature, credential, amzDate string) {
    fmt.Printf("URL: https://%s.s3.%s.amazonaws.com.cn\n", bucketName, Region)
    fmt.Println("Fields:")
    //fmt.Printf("Key: %s\n", key)
    fmt.Printf("AWSAccessKeyId: %s\n", creds.AccessKeyID)
    fmt.Printf("Policy: %s\n", policy)
    fmt.Printf("x-amz-signature: %s\n", signature)
    fmt.Printf("x-amz-credential: %s\n", credential)
    fmt.Printf("x-amz-algorithm: AWS4-HMAC-SHA256\n")
    fmt.Printf("x-amz-date: %s\n", amzDate)
}

// generateCURLCommand generates a CURL command for uploading a file to S3
func generateCURLCommand(bucketName, key, contentType, policy, credential, amzDate, signature string) string {
    filename := filepath.Base(key)
    curlTemplate := `curl -X POST \
  -F "key=%s" \
  -F "Content-Type=%s" \
  -F "X-Amz-Credential=%s" \
  -F "X-Amz-Algorithm=AWS4-HMAC-SHA256" \
  -F "X-Amz-Date=%s" \
  -F "Policy=%s" \
  -F "X-Amz-Signature=%s" \
  -F "file=@%s" \
  https://%s.s3.%s.amazonaws.com.cn/`

    return fmt.Sprintf(curlTemplate, key, contentType, credential, amzDate, policy, signature, filename, bucketName, Region)
}

输出

运行这两个 CURL 示例命令，即可提交当前文件夹下的 hello.txt 和 hello.jpg 文件（先准备好）

未报错则说明上传成功，否则 S3 会抛出错误原因。

交互流程

用户选择文件 → 从服务器获取 URL + Form Fields → 客户端上传文件到 S3（循环上传多个文件） → 上传成功后，将上传成功信息提交给服务器

参考