Cumulus utilities helping with S3 functionality

boto3_resource(**kwargs)

Define boto3 resource

Returns:

Type Description
boto3.resource

resource object with default options with or without user defined attributes

Source code in cumulus_packager/utils/boto.py
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def boto3_resource(**kwargs):
    """Define boto3 resource

    Returns
    -------
    boto3.resource
        resource object with default options with or without user defined attributes
    """
    kwargs_ = {
        "aws_access_key_id": AWS_ACCESS_KEY_ID,
        "aws_secret_access_key": AWS_SECRET_ACCESS_KEY,
        "region_name": AWS_DEFAULT_REGION,
        **kwargs,
    }

    return boto3.resource(**kwargs_)

s3_download_file(bucket, key, dst='/tmp', prefix=None)

Wrapper supporting S3 downloading a file

Parameters:

Name Type Description Default
bucket str

S3 Bucket

required
key str

S3 key object

required
prefix str, optional

Add prefix to filename, by default ""

None
dst str, optional

FQP to temporary directory, by default "/tmp"

'/tmp'

Returns:

Type Description
str | False

FQPN to downloaded file | False if failed

Raises:

Type Description
Exception

ClientError

Source code in cumulus_packager/utils/boto.py
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
def s3_download_file(bucket: str, key: str, dst: str = "/tmp", prefix: str = None):
    """Wrapper supporting S3 downloading a file

    Parameters
    ----------
    bucket : str
        S3 Bucket
    key : str
        S3 key object
    prefix : str, optional
        Add prefix to filename, by default ""
    dst : str, optional
        FQP to temporary directory, by default "/tmp"

    Returns
    -------
    str | False
        FQPN to downloaded file | False if failed

    Raises
    ------
    Exception
        ClientError
    """
    file = os.path.basename(key)
    file = prefix + "-" + file if prefix else file

    filename = os.path.join(dst, file)
    logger.debug(f"S3 Download File: {filename}")

    # download the file
    try:
        if (
            s3 := boto3_resource(
                service_name="s3",
                endpoint_url=ENDPOINT_URL_S3,
            )
        ) is None:
            raise Exception(ClientError)
        s3.meta.client.download_file(
            Bucket=bucket,
            Key=key,
            Filename=filename,
        )
    except ClientError as ex:
        logger.error(f"{type(ex).__name__}: {this}: {ex} - key: {key}")
        return False
    return filename

s3_upload_file(file_name, bucket, key=None)

Wrapper supporting S3 uploading a file

Parameters:

Name Type Description Default
file_name str

file to upload

required
bucket str

S3 bucket

required
key str, optional

S3 object key, by default None

None

Returns:

Type Description
bool

boolean describing successful upload

Raises:

Type Description
Exception

ClientError

Source code in cumulus_packager/utils/boto.py
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
def s3_upload_file(file_name: str, bucket: str, key: str = None):
    """Wrapper supporting S3 uploading a file

    Parameters
    ----------
    file_name : str
        file to upload
    bucket : str
        S3 bucket
    key : str, optional
        S3 object key, by default None

    Returns
    -------
    bool
        boolean describing successful upload

    Raises
    ------
    Exception
        ClientError
    """
    # If S3 object_name was not specified, use file_name
    if key is None:
        key = os.path.basename(file_name)

    # Upload the file
    try:
        if (
            s3 := boto3_resource(
                service_name="s3",
                endpoint_url=ENDPOINT_URL_S3,
            )
        ) is None:
            raise Exception(ClientError)
        s3.meta.client.upload_file(Filename=file_name, Bucket=bucket, Key=key)
        logger.debug(f"{file_name}\t{bucket=}\t{key=}")
    except (ClientError, Exception) as ex:
        logger.error(f"{type(ex).__name__}: {this}: {ex} - key: {key}")
        return False
    return True