CtrlK
BlogDocsLog inGet started
Tessl Logo

giuseppe-trisciuoglio/developer-kit

Comprehensive developer toolkit providing reusable skills for Java/Spring Boot, TypeScript/NestJS/React/Next.js, Python, PHP, AWS CloudFormation, AI/RAG, DevOps, and more.

82

Quality

82%

Does it follow best practices?

Impact

Pending

No eval scenarios have been run

SecuritybySnyk

Risky

Do not use without reviewing

Validation failed for skills in this tile
One or more skills have errors that need to be fixed before they can move to Implementation and Discovery review.
Overview
Quality
Evals
Security
Files

s3-transfer-patterns.mdplugins/developer-kit-java/skills/aws-sdk-java-v2-s3/references/

S3 Transfer Patterns Reference

S3 Transfer Manager Advanced Patterns

Configuration and Optimization

Custom Transfer Manager Configuration

import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.UploadFileRequest;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import java.time.Duration;

public S3TransferManager createOptimizedTransferManager(S3Client s3Client) {
    return S3TransferManager.builder()
        .s3Client(s3Client)
        .storageProvider(ApacheHttpClient.builder()
            .maxConnections(200)
            .connectionTimeout(Duration.ofSeconds(5))
            .socketTimeout(Duration.ofSeconds(60))
            .build())
        .build();
}

Parallel Upload Configuration

public void configureParallelUploads() {
    S3TransferManager transferManager = S3TransferManager.create();

    FileUpload upload = transferManager.uploadFile(
        UploadFileRequest.builder()
            .putObjectRequest(req -> req
                .bucket("my-bucket")
                .key("large-file.bin"))
            .source(Paths.get("large-file.bin"))
            .build());

    // Track upload progress
    upload.progressFuture().thenAccept(progress -> {
        System.out.println("Upload progress: " + progress.progressPercent());
    });

    // Handle completion
    upload.completionFuture().thenAccept(result -> {
        System.out.println("Upload completed with ETag: " +
            result.response().eTag());
    });
}

Advanced Upload Patterns

Multipart Upload with Progress Monitoring

public void multipartUploadWithProgress(S3Client s3Client, String bucketName,
                                     String key, String filePath) {
    int partSize = 5 * 1024 * 1024; // 5 MB parts
    File file = new File(filePath);

    CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder()
        .bucket(bucketName)
        .key(key)
        .build();

    CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(createRequest);
    String uploadId = createResponse.uploadId();

    List<CompletedPart> completedParts = new ArrayList<>();
    long uploadedBytes = 0;
    long totalBytes = file.length();

    try (FileInputStream fis = new FileInputStream(file)) {
        byte[] buffer = new byte[partSize];
        int partNumber = 1;

        while (true) {
            int bytesRead = fis.read(buffer);
            if (bytesRead == -1) break;

            byte[] partData = Arrays.copyOf(buffer, bytesRead);

            UploadPartRequest uploadRequest = UploadPartRequest.builder()
                .bucket(bucketName)
                .key(key)
                .uploadId(uploadId)
                .partNumber(partNumber)
                .build();

            UploadPartResponse uploadResponse = s3Client.uploadPart(
                uploadRequest, RequestBody.fromBytes(partData));

            completedParts.add(CompletedPart.builder()
                .partNumber(partNumber)
                .eTag(uploadResponse.eTag())
                .build());

            uploadedBytes += bytesRead;
            partNumber++;

            // Log progress
            double progress = (double) uploadedBytes / totalBytes * 100;
            System.out.printf("Upload progress: %.2f%%%n", progress);
        }

        CompleteMultipartUploadRequest completeRequest =
            CompleteMultipartUploadRequest.builder()
                .bucket(bucketName)
                .key(key)
                .uploadId(uploadId)
                .multipartUpload(CompletedMultipartUpload.builder()
                    .parts(completedParts)
                    .build())
                .build();

        s3Client.completeMultipartUpload(completeRequest);

    } catch (Exception e) {
        // Abort on failure
        AbortMultipartUploadRequest abortRequest =
            AbortMultipartUploadRequest.builder()
                .bucket(bucketName)
                .key(key)
                .uploadId(uploadId)
                .build();

        s3Client.abortMultipartUpload(abortRequest);
        throw new RuntimeException("Multipart upload failed", e);
    }
}

Resume Interrupted Uploads

public void resumeUpload(S3Client s3Client, String bucketName, String key,
                       String filePath, String existingUploadId) {
    ListMultipartUploadsRequest listRequest = ListMultipartUploadsRequest.builder()
        .bucket(bucketName)
        .prefix(key)
        .build();

    ListMultipartUploadsResponse listResponse = s3Client.listMultipartUploads(listRequest);

    // Check if upload already exists
    boolean uploadExists = listResponse.uploads().stream()
        .anyMatch(upload -> upload.key().equals(key) &&
                           upload.uploadId().equals(existingUploadId));

    if (uploadExists) {
        // Resume existing upload
        continueExistingUpload(s3Client, bucketName, key, existingUploadId, filePath);
    } else {
        // Start new upload
        multipartUploadWithProgress(s3Client, bucketName, key, filePath);
    }
}

private void continueExistingUpload(S3Client s3Client, String bucketName,
                                 String key, String uploadId, String filePath) {
    // List already uploaded parts
    ListPartsRequest listPartsRequest = ListPartsRequest.builder()
        .bucket(bucketName)
        .key(key)
        .uploadId(uploadId)
        .build();

    ListPartsResponse listPartsResponse = s3Client.listParts(listPartsRequest);

    List<CompletedPart> completedParts = listPartsResponse.parts().stream()
        .map(part -> CompletedPart.builder()
            .partNumber(part.partNumber())
            .eTag(part.eTag())
            .build())
        .collect(Collectors.toList());

    // Upload remaining parts
    // ... implementation of remaining parts upload
}

Advanced Download Patterns

Partial File Download

public void downloadPartialFile(S3Client s3Client, String bucketName, String key,
                              String destPath, long startByte, long endByte) {
    GetObjectRequest request = GetObjectRequest.builder()
        .bucket(bucketName)
        .key(key)
        .range("bytes=" + startByte + "-" + endByte)
        .build();

    try (ResponseInputStream<GetObjectResponse> response = s3Client.getObject(request);
         OutputStream outputStream = new FileOutputStream(destPath)) {

        response.transferTo(outputStream);
        System.out.println("Partial download completed: " +
            (endByte - startByte + 1) + " bytes");
    }
}

Parallel Downloads

import java.util.concurrent.*;
import java.util.stream.*;

public void parallelDownloads(S3Client s3Client, String bucketName,
                           String key, String destPath, int chunkCount) {
    long fileSize = getFileSize(s3Client, bucketName, key);
    long chunkSize = fileSize / chunkCount;

    ExecutorService executor = Executors.newFixedThreadPool(chunkCount);
    List<Future<Void>> futures = new ArrayList<>();

    for (int i = 0; i < chunkCount; i++) {
        long start = i * chunkSize;
        long end = (i == chunkCount - 1) ? fileSize - 1 : start + chunkSize - 1;

        Future<Void> future = executor.submit(() -> {
            downloadPartialFile(s3Client, bucketName, key,
                destPath + ".part" + i, start, end);
            return null;
        });

        futures.add(future);
    }

    // Wait for all downloads to complete
    for (Future<Void> future : futures) {
        try {
            future.get();
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException("Download failed", e);
        }
    }

    // Combine chunks
    combineChunks(destPath, chunkCount);

    executor.shutdown();
}

private void combineChunks(String baseName, int chunkCount) throws IOException {
    try (OutputStream outputStream = new FileOutputStream(baseName)) {
        for (int i = 0; i < chunkCount; i++) {
            String chunkFile = baseName + ".part" + i;
            try (InputStream inputStream = new FileInputStream(chunkFile)) {
                inputStream.transferTo(outputStream);
            }
            new File(chunkFile).delete();
        }
    }
}

Error Handling and Retry

Upload with Exponential Backoff

import software.amazon.awssdk.core.retry.conditions.*;
import software.amazon.awssdk.core.retry.*;
import software.amazon.awssdk.core.retry.backoff.*;

public void resilientUpload(S3Client s3Client, String bucketName, String key,
                           String filePath) {
    PutObjectRequest request = PutObjectRequest.builder()
        .bucket(bucketName)
        .key(key)
        .build();

    // Configure retry policy
    S3Client retryS3Client = S3Client.builder()
        .overrideConfiguration(b -> b
            .retryPolicy(RetryPolicy.builder()
                .numRetries(5)
                .retryBackoffStrategy(
                    ExponentialRetryBackoff.builder()
                        .baseDelay(Duration.ofSeconds(1))
                        .maxBackoffTime(Duration.ofSeconds(30))
                        .build())
                .retryCondition(
                    RetryCondition.or(
                        RetryCondition.defaultRetryCondition(),
                        RetryCondition.create(response ->
                            response.httpResponse().is5xxServerError()))
                )
                .build()))
        .build();

    retryS3Client.putObject(request, RequestBody.fromFile(Paths.get(filePath)));
}

Upload with Checkpoint

import java.nio.file.*;

public void uploadWithCheckpoint(S3Client s3Client, String bucketName,
                               String key, String filePath) {
    String checkpointFile = filePath + ".checkpoint";
    Path checkpointPath = Paths.get(checkpointFile);

    long startPos = 0;
    if (Files.exists(checkpointPath)) {
        // Read checkpoint
        try {
            startPos = Long.parseLong(Files.readString(checkpointPath));
        } catch (IOException e) {
            throw new RuntimeException("Failed to read checkpoint", e);
        }
    }

    if (startPos > 0) {
        // Resume upload
        continueUploadFromCheckpoint(s3Client, bucketName, key, filePath, startPos);
    } else {
        // Start new upload
        startNewUpload(s3Client, bucketName, key, filePath);
    }

    // Update checkpoint
    long endPos = new File(filePath).length();
    try {
        Files.writeString(checkpointPath, String.valueOf(endPos));
    } catch (IOException e) {
        throw new RuntimeException("Failed to write checkpoint", e);
    }
}

private void continueUploadFromCheckpoint(S3Client s3Client, String bucketName,
                                        String key, String filePath, long startPos) {
    // Implement resume logic
}

private void startNewUpload(S3Client s3Client, String bucketName,
                          String key, String filePath) {
    // Implement initial upload logic
}

Performance Tuning

Buffer Configuration

public S3Client configureLargeBuffer() {
    return S3Client.builder()
        .overrideConfiguration(b -> b
            .apiCallAttemptTimeout(Duration.ofMinutes(5))
            .apiCallTimeout(Duration.ofMinutes(10)))
        .build();
}

public S3TransferManager configureHighThroughput() {
    return S3TransferManager.builder()
        .multipartUploadThreshold(8 * 1024 * 1024) // 8 MB
        .multipartUploadPartSize(10 * 1024 * 1024) // 10 MB
        .build();
}

Network Optimization

public S3Client createOptimizedS3Client() {
    return S3Client.builder()
        .httpClientBuilder(ApacheHttpClient.builder()
            .maxConnections(200)
            .connectionPoolStrategy(ConnectionPoolStrategy.defaultStrategy())
            .socketTimeout(Duration.ofSeconds(30))
            .connectionTimeout(Duration.ofSeconds(5))
            .connectionAcquisitionTimeout(Duration.ofSeconds(30))
            .build())
        .region(Region.US_EAST_1)
        .build();
}

Monitoring and Metrics

Upload Progress Tracking

public void uploadWithProgressTracking(S3Client s3Client, String bucketName,
                                    String key, String filePath) {
    PutObjectRequest request = PutObjectRequest.builder()
        .bucket(bucketName)
        .key(key)
        .build();

    // Create progress listener
    software.amazon.awssdk.core.ProgressListener progressListener =
        progressEvent -> {
            System.out.println("Transferred: " +
                progressEvent.transferredBytes() + " bytes");
            System.out.println("Progress: " +
                progressEvent.progressPercent() + "%");
        };

    Response<PutObjectResponse> response = s3Client.putObject(
        request,
        RequestBody.fromFile(Paths.get(filePath)),
        software.amazon.awssdk.core.sync.RequestBody.fromFile(Paths.get(filePath))
            .contentLength(new File(filePath).length()),
        progressListener);

    System.out.println("Upload complete. ETag: " +
        response.response().eTag());
}

Throughput Measurement

public void measureUploadThroughput(S3Client s3Client, String bucketName,
                                  String key, String filePath) {
    long startTime = System.currentTimeMillis();
    long fileSize = new File(filePath).length();

    PutObjectRequest request = PutObjectRequest.builder()
        .bucket(bucketName)
        .key(key)
        .build();

    s3Client.putObject(request, RequestBody.fromFile(Paths.get(filePath)));

    long endTime = System.currentTimeMillis();
    long duration = endTime - startTime;
    double throughput = (fileSize * 1000.0) / duration / (1024 * 1024); // MB/s

    System.out.printf("Upload throughput: %.2f MB/s%n", throughput);
}

Testing and Validation

Upload Validation

public void validateUpload(S3Client s3Client, String bucketName, String key,
                         String localFilePath) {
    // Download file from S3
    byte[] s3Content = downloadObject(s3Client, bucketName, key);

    // Read local file
    byte[] localContent = Files.readAllBytes(Paths.get(localFilePath));

    // Validate content matches
    if (!Arrays.equals(s3Content, localContent)) {
        throw new RuntimeException("Upload validation failed: content mismatch");
    }

    // Verify file size
    long s3Size = s3Content.length;
    long localSize = localContent.length;
    if (s3Size != localSize) {
        throw new RuntimeException("Upload validation failed: size mismatch");
    }

    System.out.println("Upload validation successful");
}

plugins

developer-kit-java

skills

aws-sdk-java-v2-s3

references

s3-object-operations.md

s3-spring-boot-integration.md

s3-transfer-patterns.md

SKILL.md

README.md

tile.json