JSON-OBJECT Software Engineering Blog

Professional Senior Backend Engineer. Specializing in high volume traffic and distributed processing with Kotlin and Spring Boot as core technologies.

View on GitHub
23 December 2022

Spring Boot, Amazon S3에 최대 5TB 대용량 파일 Multipart 업로드 구현하기

by Taehyeong Lee

개요

S3 멀티파트 업로드 흐름

S3 멀티파트 업로드를 써야 하는 이유

S3 멀티파트 업로드시 고려할 점

// Amazon S3 콘솔 로그인 > 버킷 > 권한 > CORS(Cross-origin 리소스 공유) > 편집
[
  {
    "AllowedHeaders": [
      "*"
    ],
    "AllowedMethods": [
      "POST",
      "GET",
      "HEAD",
      "PUT"
    ],
    "AllowedOrigins": [
      "*"
    ],
    "ExposeHeaders": [
      "ETag"
    ]
  }
]

build.gradle.kts

dependencies {
    implementation("software.amazon.awssdk:s3:2.19.13")
}

AmazonS3Util 작성

import software.amazon.awssdk.core.sync.RequestBody
import software.amazon.awssdk.regions.Region
import software.amazon.awssdk.services.s3.S3Client
import software.amazon.awssdk.services.s3.model.*
import software.amazon.awssdk.services.s3.presigner.S3Presigner
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest
import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest
import java.io.File
import java.io.InputStream
import java.time.Duration

object AmazonS3Util {
    private fun s3ClientV2(): S3Client {

        return S3Client
            .builder()
            .region(Region.of("{region}"))
            .build()
    }

    private fun s3PresignerV2(): S3Presigner {

        return S3Presigner
            .builder()
            .region(Region.of("{region}"))
            .build()
    }


    fun createMultipartUploadV2(
        bucket: String,
        key: String
    ): CreateMultipartUploadResponse {

        return s3ClientV2().createMultipartUpload(
            CreateMultipartUploadRequest.builder().bucket(bucket).key(key).build()
        )
    }

    private fun generateWriteOnlyMultipartPresignedUrlV2(
        bucket: String,
        key: String,
        duration: Duration,
        uploadId: String,
        partNumber: Int
    ): String {

        return s3PresignerV2().presignUploadPart { request: UploadPartPresignRequest.Builder ->
            request.signatureDuration(duration)
                .uploadPartRequest { uploadPartRequest: UploadPartRequest.Builder ->
                    uploadPartRequest.bucket(bucket)
                        .key(key)
                        .partNumber(partNumber)
                        .uploadId(uploadId)
                }
        }.url().toString()
    }

    fun generateWriteOnlyMultipartPresignedUrlsV2(
        bucket: String,
        key: String,
        duration: Duration,
        uploadId: String,
        partSize: Int
    ): List<FileMultipartUploadUrlDTO> {

        val multipartPresignedUrls = mutableListOf<FileMultipartUploadUrlDTO>()
        (1..partSize).forEach { partNumber ->
            multipartPresignedUrls.add(
                FileMultipartUploadUrlDTO(
                    partNumber, generateWriteOnlyMultipartPresignedUrlV2(bucket, key, duration, uploadId, partNumber)
                )
            )
        }

        return multipartPresignedUrls
    }

    fun completeMultipartUploadV2(
        bucket: String,
        key: String,
        uploadId: String,
        parts: List<CompletedPart>
    ): CompleteMultipartUploadResponse {

        return s3ClientV2().completeMultipartUpload { request ->
            request
                .bucket(bucket)
                .key(key)
                .uploadId(uploadId)
                .multipartUpload(CompletedMultipartUpload.builder().parts(parts).build())
        }
    }
    
    fun abortMultipartUploadV2(
        bucket: String,
        key: String,
        uploadId: String
    ): AbortMultipartUploadResponse {

        return s3ClientV2().abortMultipartUpload { request ->
            request
                .bucket(bucket)
                .key(key)
                .uploadId(uploadId)
        }
    }

    fun calculateMultipartCount(originalFileSize: Long, requestCount: Long = 10): Int {

        val minPartSize: Long = 5242880
        val maxPartSize: Long = 2147483648
        val recommendedMinOriginalFileSize: Long = 104857600
        val maxPartCount: Long = 10000
        val correctedPartCount: Long = if (requestCount > maxPartCount) {
            maxPartCount
        } else {
            requestCount
        }

        if (originalFileSize < recommendedMinOriginalFileSize) return 1
        if (originalFileSize / correctedPartCount < minPartSize) {
            return (originalFileSize / minPartSize).toInt()
        }
        if (originalFileSize / correctedPartCount > maxPartSize) {
            return (originalFileSize / maxPartSize).toInt()
        }

        return requestCount.toInt()
    }
}

data class FileMultipartUploadUrlDTO(

    var partNumber: Int = 1,
    var uploadUrl: String = ""

) : Serializable

1. 멀티파트 업로드 목록 요청

// 특정 Bucket의 Key에 멀티파트 업로드를 하기 위한 uploadId 값을 요청
val uploadId = AmazonS3Util.createMultipartUploadV2("{bucket}", "{key}").uploadId()

// 멀티파트 업로드시 분할 업로드 개수를 계산
val multipartCount = AmazonS3Util.calculateMultipartCount({fileSize})

// 멀티파트 업로드 URL 목록을 생성
val multipartUploadUrls = AmazonS3Util.generateWriteOnlyMultipartPresignedUrlsV2(
    "{bucket}",
    "{key}",
    Duration.ofMinutes(60),
    uploadId,
    multipartCount
)

[
    {
        "partNumber": 1,
        "uploadUrl": "{url}"
    },
    {
        "partNumber": 2,
        "uploadUrl": "{url}"
    }
]

2-1. 멀티파트 업로드 실행: 리눅스 사이드

$ curl -v -k -T multipart.mp4.1 "{url}"
< HTTP/1.1 100 Continue
* We are completely uploaded and fine
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< x-amz-id-2: /lZyTJxaI8ZWoEUDNpY7AHzSUhaZegPw2/Fg3Riy2EZwQHNFbIOIuGfCuIufbwu0MAgLmrzx5Yw=
< x-amz-request-id: 9K40M0MSTVB0ZWDK
< Date: Wed, 21 Sep 2022 06:11:39 GMT
< ETag: "0b41b1b5c7228c08597fe7ae9ea06abc"
< Server: AmazonS3
< Content-Length: 0

2-2. 멀티파트 업로드 실행: 브라우저 사이드

const chunkInterval = Math.floor(file.size / {서버가 응답한 파트 개수});
let chunkedStart = 0;

const chunkWithUrlList = {서버가 응답한 파트 목록}.map(({
    partNumber,
    uploadUrl
}, i) => {
    if (i === {서버가 응답한 파트 개수}.length - 1) {
        chunkEnd = file.size;
    } else {
        chunkEnd = chunkedStart + chunkInterval;
    }

    const chunk = file.slice(chunkedStart, chunkEnd);
    chunkedStart = chunkEnd;

    return {
        uploadUrl,
        partNumber,
        chunk,
    }
});

const fulfilledList = [];
const rejectedList = [];

await Promise.allSettled(chunkWithUrlList.map(
    ({
        uploadUrl,
        partNumber,
        chunk
    }) => fetch(
        uploadUrl, {
            method: 'PUT',
            body: chunk,
        }).then((res) => {
        console.log(`partNumber : ${partNumber} / ETag : ${res.headers.get('ETag')}`)
        return {
            partNumber,
            eTag: res.headers.get('ETag').replace(/"/g, ''),
        }
    })
)).then((res) => {
    console.log(`upload result : ${res}`)
    res.forEach((el) => {
        if (el.status === 'fulfilled') {
            fulfilledList.push(el.value);
            return;
        }

        rejectedList.push(el.value);
    });
});

// 각 파트에 대한 fetch는 ETag와 partNumber를 기억하고, 모든 파트에 대한 fetch 완료 시점에 그대로 결과 목록을 서버에 요청

3-1. 멀티파트 업로드 완료 요청

// 멀티파트 업로드 완료 요청
AmazonS3Util.completeMultipartUploadV2(
    "{bucket}",
    "{key}",
    "{uploadId}",
    parts = listOf(CompletedPart.builder().partNumber({partNumber}).eTag("{eTag}").build())
)

3-2. 멀티파트 업로드 취소 요청

// 멀티파트 업로드 취소 요청
AmazonS3Util.abortMultipartUploadV2(
    "{bucket}",
    "{key}",
    "{uploadId}"
)

참고 글

tags: Spring Boot - AWS