Module `libnova.common.filesystem.S3.File`

Expand source code

#!/usr/bin/env python
# coding: utf-8

import botocore

from libnova.common.filesystem    import S3
from libnova.common.filesystem.S3 import FileStream


def get_stream(storage, file):
    """Retrieve a stream from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        file (libnova.common.api.File): A platform `File` object

    Returns:
        io.RawIOBase: An S3 object stream
    """

    bucket    = storage.extra_data["bucket"]
    file_path = file.container_id + file.fullpath

    return get_file_stream(bucket, file_path)


def get_file_stream(bucket, file_path):
    """Retrieve a stream from an S3 Object given a bucket and a file path

    Args:
        bucket (str): An S3 bucket
        file_path (str): A file path inside the bucket

    Returns:
        io.RawIOBase: An S3 object stream
    """

    #client = S3.Storage.get_resource()
    #
    #s3object = client.Object(bucket_name=bucket, key=file_path)
    #if s3object is not None:
    #    return FileStream.FileStream(s3object)

    client = S3.Storage.get_client()

    s3object = client.get_object(Bucket=bucket, Key=file_path)
    if s3object is not None:
        return s3object['Body']._raw_stream

    print("Can't get stream from remote file /" + str(bucket) + "/" + file_path)
    return None


def upload_storage_container_file(storage, container, object_key, local_file_path):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_file(storage.extra_data["bucket"], container.id, object_key, local_file_path)


def upload_storage_container_stream(storage, container, object_key, stream):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    upload_stream(storage.extra_data["bucket"], container.id, object_key, stream)


def upload_file(bucket, container_id, object_key, local_file_path):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_stream(bucket, container_id, object_key, open(local_file_path, 'rb'))


def upload_stream(bucket, container_id, object_key, stream):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    __upload_stream(bucket, str(container_id) + '/' + object_key.lstrip("/"), stream)


def __upload_stream(bucket, object_key, stream):
    """Upload a file to an S3 `bucket` and `object_key`,
    using `stream` as the data source to upload

    Args:
        bucket (str): The target S3 bucket
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    client = S3.Storage.get_resource()

    client.Object(bucket, object_key.lstrip("/")).put(Body=stream)


def get_versions(storage, container, object_key=""):
    """Get the versions of a given S3 object key

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The S3 object key
    """

    resource = S3.Storage.get_resource()

    bucket   = storage.extra_data["bucket"]
    return resource.Bucket(bucket).object_versions.filter(Prefix=str(container.id)+"/"+object_key.lstrip('/'))

if __name__ == "__main__":
    print('This file cannot be executed directly!')

Functions

def get_file_stream(bucket, file_path)

Retrieve a stream from an S3 Object given a bucket and a file path

Args

bucket : str: An S3 bucket
file_path : str: A file path inside the bucket

Returns

io.RawIOBase: An S3 object stream

Expand source code

def get_file_stream(bucket, file_path):
    """Retrieve a stream from an S3 Object given a bucket and a file path

    Args:
        bucket (str): An S3 bucket
        file_path (str): A file path inside the bucket

    Returns:
        io.RawIOBase: An S3 object stream
    """

    #client = S3.Storage.get_resource()
    #
    #s3object = client.Object(bucket_name=bucket, key=file_path)
    #if s3object is not None:
    #    return FileStream.FileStream(s3object)

    client = S3.Storage.get_client()

    s3object = client.get_object(Bucket=bucket, Key=file_path)
    if s3object is not None:
        return s3object['Body']._raw_stream

    print("Can't get stream from remote file /" + str(bucket) + "/" + file_path)
    return None

def get_stream(storage, file)

Retrieve a stream from an S3 Object given a platform Storage and File

Args

storage : libnova.common.api.Storage: A platform Storage object
file : libnova.common.api.File: A platform File object

Returns

io.RawIOBase: An S3 object stream

Expand source code

def get_stream(storage, file):
    """Retrieve a stream from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        file (libnova.common.api.File): A platform `File` object

    Returns:
        io.RawIOBase: An S3 object stream
    """

    bucket    = storage.extra_data["bucket"]
    file_path = file.container_id + file.fullpath

    return get_file_stream(bucket, file_path)

def get_versions(storage, container, object_key='')

Get the versions of a given S3 object key

Args

storage : libnova.common.api.Storage: A platform Storage object
container : libnova.common.api.Container: A platform Container object
object_key : str: The S3 object key

Expand source code

def get_versions(storage, container, object_key=""):
    """Get the versions of a given S3 object key

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The S3 object key
    """

    resource = S3.Storage.get_resource()

    bucket   = storage.extra_data["bucket"]
    return resource.Bucket(bucket).object_versions.filter(Prefix=str(container.id)+"/"+object_key.lstrip('/'))

def upload_file(bucket, container_id, object_key, local_file_path)

Upload a file to an S3 bucket using a given platform Container id to a target object_key, using local_file_path as the source file to upload

Args

bucket : str: The target S3 bucket
container_id : int: A platform Container id
object_key : str: The target S3 object key
local_file_path : str: The local file to use as the file source to upload to S3

Expand source code

def upload_file(bucket, container_id, object_key, local_file_path):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_stream(bucket, container_id, object_key, open(local_file_path, 'rb'))

def upload_storage_container_file(storage, container, object_key, local_file_path)

Upload a file to S3 using a given platform Storage and Container object to a target object_key, using local_file_path as the source file to upload

Args

storage : libnova.common.api.Storage: A platform Storage object
container : libnova.common.api.Container: A platform Container object
object_key : str: The target S3 object key
local_file_path : str: The local file to use as the file source to upload to S3

Expand source code

def upload_storage_container_file(storage, container, object_key, local_file_path):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_file(storage.extra_data["bucket"], container.id, object_key, local_file_path)

def upload_storage_container_stream(storage, container, object_key, stream)

Upload a file to S3 using a given platform Storage and Container object to a target object_key, using stream as the data source to upload

Args

storage : libnova.common.api.Storage: A platform Storage object
container : libnova.common.api.Container: A platform Container object
object_key : str: The target S3 object key
stream : io.RawIOBase: The stream to use as data source

Expand source code

def upload_storage_container_stream(storage, container, object_key, stream):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    upload_stream(storage.extra_data["bucket"], container.id, object_key, stream)

def upload_stream(bucket, container_id, object_key, stream)

Upload a file to an S3 bucket using a given platform Container id to a target object_key, using stream as the data source to upload

Args

bucket : str: The target S3 bucket
container_id : int: A platform Container id
object_key : str: The target S3 object key
stream : io.RawIOBase: The stream to use as data source

Expand source code

def upload_stream(bucket, container_id, object_key, stream):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    __upload_stream(bucket, str(container_id) + '/' + object_key.lstrip("/"), stream)