Module libnova.common.filesystem.S3.Storage

S3 Storage Wrapper

This wrapper allows the semi-automatic initialization of the different S3 classes to ease the interaction of the library with the data available in Amazon S3.

Expand source code
#!/usr/bin/env python
# coding: utf-8

"""S3 Storage Wrapper

This wrapper allows the semi-automatic initialization of the different S3 classes to ease the interaction
of the library with the data available in Amazon S3.
"""

import boto3
from botocore.config import Config

import libnova.Util
from libnova.common     import api
from libnova.common.api import Driver, AccessMethod

AWS_ENDPOINT          = ''
AWS_REGION            = ''
AWS_ACCESS_KEY_ID     = ''
AWS_SECRET_ACCESS_KEY = ''
AWS_SECURE            = True
AWS_BUCKET            = ''


def initialize(secure=True, override_host='', override_region='', override_access_key='', override_secret_key='', override_bucket=''):
    """Initialize the wrapper given a set of settings

    If any field is not provided, it will be taken from the S3 Access Method for the current user

    Args:
        secure (bool): Whether to use HTTPS to interact with the S3 server
        override_host (str): Override the endpoint with the given value
        override_region (str): Override the region with the given value
        override_access_key (str): Override the access key with the given value
        override_secret_key (str): Override the secret key with the given value
        override_bucket (str): Override the bucket key with the given value
    """

    # Bypass API retrieval if all fields have been overriden
    if(
        bool(override_host) and
        bool(override_region) and
        bool(override_access_key) and
        bool(override_secret_key) and
        bool(override_bucket)
    ):
        __initialize(override_host, override_region, override_access_key, override_secret_key, override_bucket, secure)
        return

    s3_data = api.AccessMethod.get('s3')
    if s3_data is None:
        print("Startup error: Cannot retrieve S3 data from the platform")
        exit(1)

    hostname = s3_data['hostname']
    if override_host != '':
        hostname = override_host

    region = 'us-east-1'
    if override_region != '':
        region = override_region

    access_key = s3_data['email']
    if override_access_key != '':
        access_key = override_access_key

    secret_key = s3_data['password']
    if override_secret_key != '':
        secret_key = override_secret_key

    bucket = ''
    if 'bucket' in s3_data:
        bucket = s3_data['bucket']
        if override_bucket != '':
            bucket = override_bucket

    __initialize(hostname, region, access_key, secret_key, bucket, secure)

def __initialize(endpoint, region, access_key, secret_key, bucket = '', secure=True):
    """Initialize the wrapper given a set of settings

    This function will actually store the settings in the class

    Args:
        endpoint (str): Override the endpoint with the given value
        region (str): Override the region with the given value
        access_key (str): Override the access key with the given value
        secret_key (str): Override the secret key with the given value
        bucket (str): Override the bucket key with the given value
        secure (bool): Whether to use HTTPS to interact with the S3 server
    """

    global AWS_ENDPOINT
    global AWS_REGION
    global AWS_ACCESS_KEY_ID
    global AWS_SECRET_ACCESS_KEY
    global AWS_SECURE
    global AWS_BUCKET

    AWS_ENDPOINT           = endpoint
    AWS_REGION             = region
    AWS_ACCESS_KEY_ID      = access_key
    AWS_SECRET_ACCESS_KEY  = secret_key
    AWS_SECURE             = secure
    AWS_BUCKET             = bucket


def initialize_storage(Storage: libnova.common.api.Storage):
    """Initialize the wrapper using a `Storage` object as reference

    Args:
        Storage (libnova.common.api.Storage): Use this `Storage` as reference to initialize the S3 Storage wrapper
    """

    global AWS_ENDPOINT
    global AWS_REGION
    global AWS_ACCESS_KEY_ID
    global AWS_SECRET_ACCESS_KEY
    global AWS_SECURE
    global AWS_BUCKET

    AWS_ACCESS_KEY_ID     = Storage.extra_data["access_key"]
    AWS_SECRET_ACCESS_KEY = Storage.extra_data["secret_key"]
    AWS_REGION            = Storage.extra_data["region"]


def get_client():
    """Retrieve an S3 Client instance

    Returns:
        S3.Client: An Amazon S3 Client
    """

    return boto3.client(
        's3',
        aws_access_key_id    =AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
        region_name          =AWS_REGION
    )


def get_resource():
    """Retrieve an S3 Service Resource

    Returns:
        S3.ServiceResource: An Amazon S3 Service Resource
    """

    return boto3.resource(
        's3',
        aws_access_key_id    =AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
        region_name          =AWS_REGION
    )


def is_latest_version(bucket, container_id, object_name, version_id):
    """Not implemented
    """
    pass


if __name__ == "__main__":
    print('This file cannot be executed directly!')

Functions

def get_client()

Retrieve an S3 Client instance

Returns

S3.Client
An Amazon S3 Client
Expand source code
def get_client():
    """Retrieve an S3 Client instance

    Returns:
        S3.Client: An Amazon S3 Client
    """

    return boto3.client(
        's3',
        aws_access_key_id    =AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
        region_name          =AWS_REGION
    )
def get_resource()

Retrieve an S3 Service Resource

Returns

S3.ServiceResource
An Amazon S3 Service Resource
Expand source code
def get_resource():
    """Retrieve an S3 Service Resource

    Returns:
        S3.ServiceResource: An Amazon S3 Service Resource
    """

    return boto3.resource(
        's3',
        aws_access_key_id    =AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
        region_name          =AWS_REGION
    )
def initialize(secure=True, override_host='', override_region='', override_access_key='', override_secret_key='', override_bucket='')

Initialize the wrapper given a set of settings

If any field is not provided, it will be taken from the S3 Access Method for the current user

Args

secure : bool
Whether to use HTTPS to interact with the S3 server
override_host : str
Override the endpoint with the given value
override_region : str
Override the region with the given value
override_access_key : str
Override the access key with the given value
override_secret_key : str
Override the secret key with the given value
override_bucket : str
Override the bucket key with the given value
Expand source code
def initialize(secure=True, override_host='', override_region='', override_access_key='', override_secret_key='', override_bucket=''):
    """Initialize the wrapper given a set of settings

    If any field is not provided, it will be taken from the S3 Access Method for the current user

    Args:
        secure (bool): Whether to use HTTPS to interact with the S3 server
        override_host (str): Override the endpoint with the given value
        override_region (str): Override the region with the given value
        override_access_key (str): Override the access key with the given value
        override_secret_key (str): Override the secret key with the given value
        override_bucket (str): Override the bucket key with the given value
    """

    # Bypass API retrieval if all fields have been overriden
    if(
        bool(override_host) and
        bool(override_region) and
        bool(override_access_key) and
        bool(override_secret_key) and
        bool(override_bucket)
    ):
        __initialize(override_host, override_region, override_access_key, override_secret_key, override_bucket, secure)
        return

    s3_data = api.AccessMethod.get('s3')
    if s3_data is None:
        print("Startup error: Cannot retrieve S3 data from the platform")
        exit(1)

    hostname = s3_data['hostname']
    if override_host != '':
        hostname = override_host

    region = 'us-east-1'
    if override_region != '':
        region = override_region

    access_key = s3_data['email']
    if override_access_key != '':
        access_key = override_access_key

    secret_key = s3_data['password']
    if override_secret_key != '':
        secret_key = override_secret_key

    bucket = ''
    if 'bucket' in s3_data:
        bucket = s3_data['bucket']
        if override_bucket != '':
            bucket = override_bucket

    __initialize(hostname, region, access_key, secret_key, bucket, secure)
def initialize_storage(Storage: libnova.common.api.Storage)

Initialize the wrapper using a Storage object as reference

Args

Storage : libnova.common.api.Storage
Use this Storage as reference to initialize the S3 Storage wrapper
Expand source code
def initialize_storage(Storage: libnova.common.api.Storage):
    """Initialize the wrapper using a `Storage` object as reference

    Args:
        Storage (libnova.common.api.Storage): Use this `Storage` as reference to initialize the S3 Storage wrapper
    """

    global AWS_ENDPOINT
    global AWS_REGION
    global AWS_ACCESS_KEY_ID
    global AWS_SECRET_ACCESS_KEY
    global AWS_SECURE
    global AWS_BUCKET

    AWS_ACCESS_KEY_ID     = Storage.extra_data["access_key"]
    AWS_SECRET_ACCESS_KEY = Storage.extra_data["secret_key"]
    AWS_REGION            = Storage.extra_data["region"]
def is_latest_version(bucket, container_id, object_name, version_id)

Not implemented

Expand source code
def is_latest_version(bucket, container_id, object_name, version_id):
    """Not implemented
    """
    pass