Do you need to copy a lot of S3 objects from one bucket to another, maybe even with a different key in the destination bucket, but you don’t want or can use S3 batch jobs? Here you go, blazingly fast.

import logging
import multiprocessing
from typing import List

import boto3

logger = logging.getLogger("your.logger.name")

# Shared queue
queue = multiprocessing.JoinableQueue()

# AWS credentials
AWS_ACCESS_KEY = "your-aws-access-key"
AWS_SECRET_KEY = "your-aws-secret-key"


def move_objects(
    keys: List[str],
    source_bucket: str,
    destination_bucket: str,
    pool_size: int = 100
) -> None:
    """
    Very fast concurrent copying of all objects specified in keys
    between two S3 buckets.

    :param keys: List of key in the source bucket to move to the new bucket
    :param source_bucket: Source bucket name
    :param destination_bucket: Destination bucket name
    :param pool_size: Worker pool size
    :return:
    """
    logger.info(
        "Migrating all objects from bucket %s to bucket %s" %
        (source_bucket, destination_bucket)
    )

    # Start pool of workers. Don't forget the coma ...
    pool = multiprocessing.Pool(pool_size, _worker, (queue, ))
    #                                          ... here --^

    # The workers are daemon to make them exit when the queue joins
    pool.daemon = True

    n = len(keys)
    for i, key in enumerate(rows):
        # If the objects should moved to a different location in the destination
        # bucket then modify the new_key variable accordingly
        new_key = key
        queue.put((source_s3_bucket, key, destination_s3_bucket, new_key, i, n))

    # No more work, wait and exit
    queue.join()


def _worker(q: multiprocessing.JoinableQueue) -> None:
    """
    Worker that processes the items in the shared queue

    :param q: Queue
    """
    # One client session per worker. One can also remove the aws_access_key_id and
    # aws_secret_access_key parameters to make the boto client automatically read
    # the credentials from ~/.aws/credentials which comes with the awscli
    s3_client = boto3.client(
        "s3",
        aws_access_key_id=AWS_ACCESS_KEY,
        aws_secret_access_key=AWS_SECRET_KEY
    )

    while True:
        try:
            item = q.get(True)
            source_s3_bucket, old_key, destination_s3_bucket, new_key, i, n = item

            if i % 1000 == 0:
                logger.info("Moving object %d / %d" % (i, n))

            try:
                # Move item directly without downloading & uploading to the client
                copy_source = {"Bucket": source_s3_bucket, "Key": old_key}
                s3_client.copy_object(
                    CopySource=copy_source,
                    Bucket=destination_s3_bucket,
                    Key=new_key
                )
            except Exception as e:
                logger.error("Moving object %s failed: %s" % (old_key, e))
        finally:
            # Mark queue item as done
            q.task_done()


if __name__ == "__main__":
    # From source -> destination bucket
    source_bucket = "source-bucket-name"
    destination_bucket = "destination-bucket-name"

    aws_client = boto3.client(
        "s3",
        aws_access_key_id=AWS_ACCESS_KEY,
        aws_secret_access_key=AWS_SECRET_KEY
    )

    # Get all keys in the source bucket
    keys = [
        obj["Key"]
        for obj in aws_.list_objects(Bucket=source_bucket)["Contents"]
    ]

    # All all objects from source to destination bucket
    move_all_objects(keys, "source-bucket-name", "destination-bucket-name")