Module libnova.common.filesystem.S3.File

Expand source code 
#!/usr/bin/env python
# coding: utf-8

import botocore
import urllib.parse

from libnova.common.filesystem    import S3
from libnova.common.filesystem.S3 import FileStream


def get_stream(storage, file):
    """Retrieve a stream from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        file (libnova.common.api.File): A platform `File` object

    Returns:
        io.RawIOBase: An S3 object stream
    """

    bucket    = storage.extra_data["bucket"]
    file_path = file.container_id + file.fullpath

    return get_file_stream(bucket, file_path)


def get_container_file_stream(storage, container_id, object_key):
    """Retrieve a stream from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key

    Returns:
        io.RawIOBase: An S3 object stream
    """

    bucket    = storage.extra_data["bucket"]

    return get_file_stream(bucket, str(container_id) + object_key)


def get_bytes(storage, file):
    """Retrieve an array of bytes from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        file (libnova.common.api.File): A platform `File` object

    Returns:
        bytearray: The byte array of data from the file
    """

    file_stream = S3.File.get_stream(
        # The storage is needed to set the source bucket of the file
        storage,
        file
    )

    file_content = b''
    if file_stream is not None:
        file_stream_buffer = file_stream.read(8 * 1024 * 1024)
        while file_stream_buffer:
            file_content = b''.join([file_content, file_stream_buffer])
            file_stream_buffer = file_stream.read(8 * 1024 * 1024)

    return file_content


def get_container_file_bytes(storage, container_id, object_key):
    """Retrieve an array of bytes from an S3 Object using a given platform `Container` and the `object_key`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container_id (int): A platform `Container` id
        object_key (str): The S3 object key

    Returns:
        bytearray: The byte array of data from the file
    """

    file_stream = S3.File.get_container_file_stream(
        # The storage is needed to set the source bucket of the file
        storage,
        container_id,
        object_key
    )

    file_content = b''
    if file_stream is not None:
        file_stream_buffer = file_stream.read(8 * 1024 * 1024)
        while file_stream_buffer:
            file_content = b''.join([file_content, file_stream_buffer])
            file_stream_buffer = file_stream.read(8 * 1024 * 1024)

    return file_content


def get_file_stream(bucket, file_path):
    """Retrieve a stream from an S3 Object given a bucket and a file path

    Args:
        bucket (str): An S3 bucket
        file_path (str): A file path inside the bucket

    Returns:
        io.RawIOBase: An S3 object stream
    """

    #client = S3.Storage.get_resource()
    #
    #s3object = client.Object(bucket_name=bucket, key=file_path)
    #if s3object is not None:
    #    return FileStream.FileStream(s3object)

    client = S3.Storage.get_client()

    s3object = client.get_object(Bucket=bucket, Key=file_path)
    if s3object is not None:
        return s3object['Body']._raw_stream

    print("Can't get stream from remote file /" + str(bucket) + "/" + file_path)
    return None


def upload_storage_container_file(storage, container, object_key, local_file_path):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_file(storage.extra_data["bucket"], container.id, object_key, local_file_path)


def upload_storage_container_stream(storage, container, object_key, stream):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    upload_stream(storage.extra_data["bucket"], container.id, object_key, stream)


def upload_file(bucket, container_id, object_key, local_file_path):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_stream(bucket, container_id, object_key, open(local_file_path, 'rb'))


def upload_stream(bucket, container_id, object_key, stream):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    __upload_stream(bucket, str(container_id) + '/' + object_key.lstrip("/"), stream)


def __upload_stream(bucket, object_key, stream):
    """Upload a file to an S3 `bucket` and `object_key`,
    using `stream` as the data source to upload

    Args:
        bucket (str): The target S3 bucket
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    client = S3.Storage.get_resource()

    client.Object(bucket, object_key.lstrip("/")).put(Body=stream)


def get_directory_contents(bucket, container_id, object_key):
    client = S3.Storage.get_resource()

    bucket = client.Bucket(bucket)
    prefix = str(container_id) + '/' + object_key.strip('/') + '/'

    result = bucket.meta.client.list_objects(Bucket=bucket.name, Prefix=prefix, Delimiter='/')

    prefixes = [remote_prefix[len(str(container_id)):]  for remote_prefix  in list(map(lambda item: item.get('Prefix'), result.get('CommonPrefixes', []))) if     remote_prefix  != prefix]
    contents = [remote_content[len(str(container_id)):] for remote_content in list(map(lambda item: item.get('Key'),    result.get('Contents', [])))       if not remote_content.endswith('/')]

    return prefixes + contents


def create_directory(bucket, container_id, object_key):
    client = S3.Storage.get_client()

    client.put_object(Bucket=bucket, Key=str(container_id) + '/' + object_key.strip('/') + '/')


def directory_exists(bucket, container_id, object_key):
    client = S3.Storage.get_client()

    result = client.list_objects(Bucket=bucket, Prefix=str(container_id) + '/' + object_key.lstrip('/'))
    if 'Contents' in result:
        return True
    return False


def move(bucket, source_object_key,  target_object_key, preserve_meta = True):
    client = S3.Storage.get_resource()

    try:
        copy_source = {
            'Bucket': bucket,
            'Key':    source_object_key
        }
        extra = {}

        if preserve_meta:
            extra = {'Metadata': get_file_metadata(bucket, source_object_key)}

        client.meta.client.copy(copy_source, bucket, target_object_key, ExtraArgs=extra)
    except Exception as e:
        return False
    finally:
        client.Object(bucket_name=bucket, key=source_object_key).delete()
        return True


def head(bucket, object_key):
    try:
        return S3.Storage.get_client().head_object(Bucket=bucket, Key=object_key)
    except:
        pass
    return None


def get_container_file_metadata(storage, container_id, object_key, custom_fields_only = True):
    """Get the metadata of a given S3 object key and a platform `Container` id

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container_id (int): A platform `Container` id
        object_key (str): The S3 object key
        custom_fields_only (bool): Skip the fields set by S3 and return only our own metadata
    """
    return get_file_metadata(storage.extra_data["bucket"], str(container_id) + '/' + object_key.lstrip('/'), custom_fields_only)

def get_file_metadata(bucket, object_key, custom_fields_only=True):
    """Get the metadata of a given S3 object key

    Args:
        bucket (str): An S3 bucket
        object_key (str): The S3 object key
        custom_fields_only (bool): Skip the fields set by S3 and return only our own metadata
    """

    resource = S3.Storage.get_resource()

    meta = {}

    s3head = head(bucket, object_key)
    if s3head is not None:
        if 'ResponseMetadata' in s3head:
            if 'HTTPHeaders' in s3head['ResponseMetadata']:
                if custom_fields_only:
                    for header_key in s3head['ResponseMetadata']['HTTPHeaders'].keys():
                        if header_key.startswith('x-amz-meta-'):
                            meta[header_key[len('x-amz-meta-'):]] = s3head['ResponseMetadata']['HTTPHeaders'][header_key]
                else:
                    meta = s3head['ResponseMetadata']['HTTPHeaders']

    return meta


def container_file_head(bucket, container_id, object_key):
    return head(bucket, str(container_id) + '/' + object_key.lstrip('/'))


def exists(bucket, container_id, object_key):
    client = S3.Storage.get_client()

    try:
        head = client.head_object(Bucket=bucket, Key=str(container_id) + '/' + object_key.lstrip('/'))
        return True
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == "404":
            return False
        raise


def presign(bucket, object_key, expiration=3600, range=[]):
    client = S3.Storage.get_client()

    try:
        request_params = {
            'Bucket': bucket,
            'Key':    object_key
        }
        if len(range) == 2:
            request_params['Range'] = 'bytes=[{}-{}]'.format(range[0], range[1])
        return client.generate_presigned_url(
            'get_object',
            Params= {
                'Bucket': bucket,
                'Key':    object_key
            },
            ExpiresIn=expiration
        )
    except:
        pass
    return ""


def presign_container_file(bucket, container_id, object_key, expiration=3600, range=[]):
    return presign(bucket, str(container_id)+"/"+object_key.lstrip('/'), expiration, range)


def get_versions(storage, container, object_key=""):
    """Get the versions of a given S3 object key

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The S3 object key
    """

    resource = S3.Storage.get_resource()

    bucket   = storage.extra_data["bucket"]
    return resource.Bucket(bucket).object_versions.filter(Prefix=str(container.id)+"/"+object_key.lstrip('/'))

if __name__ == "__main__":
    print('This file cannot be executed directly!')

Functions

def container_file_head(bucket, container_id, object_key)
Expand source code 
def container_file_head(bucket, container_id, object_key):
    return head(bucket, str(container_id) + '/' + object_key.lstrip('/'))
def create_directory(bucket, container_id, object_key)
Expand source code 
def create_directory(bucket, container_id, object_key):
    client = S3.Storage.get_client()

    client.put_object(Bucket=bucket, Key=str(container_id) + '/' + object_key.strip('/') + '/')
def directory_exists(bucket, container_id, object_key)
Expand source code 
def directory_exists(bucket, container_id, object_key):
    client = S3.Storage.get_client()

    result = client.list_objects(Bucket=bucket, Prefix=str(container_id) + '/' + object_key.lstrip('/'))
    if 'Contents' in result:
        return True
    return False
def exists(bucket, container_id, object_key)
Expand source code 
def exists(bucket, container_id, object_key):
    client = S3.Storage.get_client()

    try:
        head = client.head_object(Bucket=bucket, Key=str(container_id) + '/' + object_key.lstrip('/'))
        return True
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == "404":
            return False
        raise
def get_bytes(storage, file)

Retrieve an array of bytes from an S3 Object given a platform Storage and File

Args

storage : libnova.common.api.Storage
A platform Storage object
file : libnova.common.api.File
A platform File object

Returns

bytearray
The byte array of data from the file
Expand source code 
def get_bytes(storage, file):
    """Retrieve an array of bytes from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        file (libnova.common.api.File): A platform `File` object

    Returns:
        bytearray: The byte array of data from the file
    """

    file_stream = S3.File.get_stream(
        # The storage is needed to set the source bucket of the file
        storage,
        file
    )

    file_content = b''
    if file_stream is not None:
        file_stream_buffer = file_stream.read(8 * 1024 * 1024)
        while file_stream_buffer:
            file_content = b''.join([file_content, file_stream_buffer])
            file_stream_buffer = file_stream.read(8 * 1024 * 1024)

    return file_content
def get_container_file_bytes(storage, container_id, object_key)

Retrieve an array of bytes from an S3 Object using a given platform Container and the object_key

Args

storage : libnova.common.api.Storage
A platform Storage object
container_id : int
A platform Container id
object_key : str
The S3 object key

Returns

bytearray
The byte array of data from the file
Expand source code 
def get_container_file_bytes(storage, container_id, object_key):
    """Retrieve an array of bytes from an S3 Object using a given platform `Container` and the `object_key`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container_id (int): A platform `Container` id
        object_key (str): The S3 object key

    Returns:
        bytearray: The byte array of data from the file
    """

    file_stream = S3.File.get_container_file_stream(
        # The storage is needed to set the source bucket of the file
        storage,
        container_id,
        object_key
    )

    file_content = b''
    if file_stream is not None:
        file_stream_buffer = file_stream.read(8 * 1024 * 1024)
        while file_stream_buffer:
            file_content = b''.join([file_content, file_stream_buffer])
            file_stream_buffer = file_stream.read(8 * 1024 * 1024)

    return file_content
def get_container_file_metadata(storage, container_id, object_key, custom_fields_only=True)

Get the metadata of a given S3 object key and a platform Container id

Args

storage : libnova.common.api.Storage
A platform Storage object
container_id : int
A platform Container id
object_key : str
The S3 object key
custom_fields_only : bool
Skip the fields set by S3 and return only our own metadata
Expand source code 
def get_container_file_metadata(storage, container_id, object_key, custom_fields_only = True):
    """Get the metadata of a given S3 object key and a platform `Container` id

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container_id (int): A platform `Container` id
        object_key (str): The S3 object key
        custom_fields_only (bool): Skip the fields set by S3 and return only our own metadata
    """
    return get_file_metadata(storage.extra_data["bucket"], str(container_id) + '/' + object_key.lstrip('/'), custom_fields_only)
def get_container_file_stream(storage, container_id, object_key)

Retrieve a stream from an S3 Object given a platform Storage and File

Args

storage : libnova.common.api.Storage
A platform Storage object
container_id : int
A platform Container id
object_key : str
The target S3 object key

Returns

io.RawIOBase
An S3 object stream
Expand source code 
def get_container_file_stream(storage, container_id, object_key):
    """Retrieve a stream from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key

    Returns:
        io.RawIOBase: An S3 object stream
    """

    bucket    = storage.extra_data["bucket"]

    return get_file_stream(bucket, str(container_id) + object_key)
def get_directory_contents(bucket, container_id, object_key)
Expand source code 
def get_directory_contents(bucket, container_id, object_key):
    client = S3.Storage.get_resource()

    bucket = client.Bucket(bucket)
    prefix = str(container_id) + '/' + object_key.strip('/') + '/'

    result = bucket.meta.client.list_objects(Bucket=bucket.name, Prefix=prefix, Delimiter='/')

    prefixes = [remote_prefix[len(str(container_id)):]  for remote_prefix  in list(map(lambda item: item.get('Prefix'), result.get('CommonPrefixes', []))) if     remote_prefix  != prefix]
    contents = [remote_content[len(str(container_id)):] for remote_content in list(map(lambda item: item.get('Key'),    result.get('Contents', [])))       if not remote_content.endswith('/')]

    return prefixes + contents
def get_file_metadata(bucket, object_key, custom_fields_only=True)

Get the metadata of a given S3 object key

Args

bucket : str
An S3 bucket
object_key : str
The S3 object key
custom_fields_only : bool
Skip the fields set by S3 and return only our own metadata
Expand source code 
def get_file_metadata(bucket, object_key, custom_fields_only=True):
    """Get the metadata of a given S3 object key

    Args:
        bucket (str): An S3 bucket
        object_key (str): The S3 object key
        custom_fields_only (bool): Skip the fields set by S3 and return only our own metadata
    """

    resource = S3.Storage.get_resource()

    meta = {}

    s3head = head(bucket, object_key)
    if s3head is not None:
        if 'ResponseMetadata' in s3head:
            if 'HTTPHeaders' in s3head['ResponseMetadata']:
                if custom_fields_only:
                    for header_key in s3head['ResponseMetadata']['HTTPHeaders'].keys():
                        if header_key.startswith('x-amz-meta-'):
                            meta[header_key[len('x-amz-meta-'):]] = s3head['ResponseMetadata']['HTTPHeaders'][header_key]
                else:
                    meta = s3head['ResponseMetadata']['HTTPHeaders']

    return meta
def get_file_stream(bucket, file_path)

Retrieve a stream from an S3 Object given a bucket and a file path

Args

bucket : str
An S3 bucket
file_path : str
A file path inside the bucket

Returns

io.RawIOBase
An S3 object stream
Expand source code 
def get_file_stream(bucket, file_path):
    """Retrieve a stream from an S3 Object given a bucket and a file path

    Args:
        bucket (str): An S3 bucket
        file_path (str): A file path inside the bucket

    Returns:
        io.RawIOBase: An S3 object stream
    """

    #client = S3.Storage.get_resource()
    #
    #s3object = client.Object(bucket_name=bucket, key=file_path)
    #if s3object is not None:
    #    return FileStream.FileStream(s3object)

    client = S3.Storage.get_client()

    s3object = client.get_object(Bucket=bucket, Key=file_path)
    if s3object is not None:
        return s3object['Body']._raw_stream

    print("Can't get stream from remote file /" + str(bucket) + "/" + file_path)
    return None
def get_stream(storage, file)

Retrieve a stream from an S3 Object given a platform Storage and File

Args

storage : libnova.common.api.Storage
A platform Storage object
file : libnova.common.api.File
A platform File object

Returns

io.RawIOBase
An S3 object stream
Expand source code 
def get_stream(storage, file):
    """Retrieve a stream from an S3 Object given a platform `Storage` and `File`

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        file (libnova.common.api.File): A platform `File` object

    Returns:
        io.RawIOBase: An S3 object stream
    """

    bucket    = storage.extra_data["bucket"]
    file_path = file.container_id + file.fullpath

    return get_file_stream(bucket, file_path)
def get_versions(storage, container, object_key='')

Get the versions of a given S3 object key

Args

storage : libnova.common.api.Storage
A platform Storage object
container : libnova.common.api.Container
A platform Container object
object_key : str
The S3 object key
Expand source code 
def get_versions(storage, container, object_key=""):
    """Get the versions of a given S3 object key

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The S3 object key
    """

    resource = S3.Storage.get_resource()

    bucket   = storage.extra_data["bucket"]
    return resource.Bucket(bucket).object_versions.filter(Prefix=str(container.id)+"/"+object_key.lstrip('/'))
def head(bucket, object_key)
Expand source code 
def head(bucket, object_key):
    try:
        return S3.Storage.get_client().head_object(Bucket=bucket, Key=object_key)
    except:
        pass
    return None
def move(bucket, source_object_key, target_object_key, preserve_meta=True)
Expand source code 
def move(bucket, source_object_key,  target_object_key, preserve_meta = True):
    client = S3.Storage.get_resource()

    try:
        copy_source = {
            'Bucket': bucket,
            'Key':    source_object_key
        }
        extra = {}

        if preserve_meta:
            extra = {'Metadata': get_file_metadata(bucket, source_object_key)}

        client.meta.client.copy(copy_source, bucket, target_object_key, ExtraArgs=extra)
    except Exception as e:
        return False
    finally:
        client.Object(bucket_name=bucket, key=source_object_key).delete()
        return True
def presign(bucket, object_key, expiration=3600, range=[])
Expand source code 
def presign(bucket, object_key, expiration=3600, range=[]):
    client = S3.Storage.get_client()

    try:
        request_params = {
            'Bucket': bucket,
            'Key':    object_key
        }
        if len(range) == 2:
            request_params['Range'] = 'bytes=[{}-{}]'.format(range[0], range[1])
        return client.generate_presigned_url(
            'get_object',
            Params= {
                'Bucket': bucket,
                'Key':    object_key
            },
            ExpiresIn=expiration
        )
    except:
        pass
    return ""
def presign_container_file(bucket, container_id, object_key, expiration=3600, range=[])
Expand source code 
def presign_container_file(bucket, container_id, object_key, expiration=3600, range=[]):
    return presign(bucket, str(container_id)+"/"+object_key.lstrip('/'), expiration, range)
def upload_file(bucket, container_id, object_key, local_file_path)

Upload a file to an S3 bucket using a given platform Container id to a target object_key, using local_file_path as the source file to upload

Args

bucket : str
The target S3 bucket
container_id : int
A platform Container id
object_key : str
The target S3 object key
local_file_path : str
The local file to use as the file source to upload to S3
Expand source code 
def upload_file(bucket, container_id, object_key, local_file_path):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_stream(bucket, container_id, object_key, open(local_file_path, 'rb'))
def upload_storage_container_file(storage, container, object_key, local_file_path)

Upload a file to S3 using a given platform Storage and Container object to a target object_key, using local_file_path as the source file to upload

Args

storage : libnova.common.api.Storage
A platform Storage object
container : libnova.common.api.Container
A platform Container object
object_key : str
The target S3 object key
local_file_path : str
The local file to use as the file source to upload to S3
Expand source code 
def upload_storage_container_file(storage, container, object_key, local_file_path):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `local_file_path` as the source file to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        local_file_path (str): The local file to use as the file source to upload to S3
    """

    upload_file(storage.extra_data["bucket"], container.id, object_key, local_file_path)
def upload_storage_container_stream(storage, container, object_key, stream)

Upload a file to S3 using a given platform Storage and Container object to a target object_key, using stream as the data source to upload

Args

storage : libnova.common.api.Storage
A platform Storage object
container : libnova.common.api.Container
A platform Container object
object_key : str
The target S3 object key
stream : io.RawIOBase
The stream to use as data source
Expand source code 
def upload_storage_container_stream(storage, container, object_key, stream):
    """Upload a file to S3 using a given platform `Storage` and `Container` object to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        storage (libnova.common.api.Storage): A platform `Storage` object
        container (libnova.common.api.Container): A platform `Container` object
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    upload_stream(storage.extra_data["bucket"], container.id, object_key, stream)
def upload_stream(bucket, container_id, object_key, stream)

Upload a file to an S3 bucket using a given platform Container id to a target object_key, using stream as the data source to upload

Args

bucket : str
The target S3 bucket
container_id : int
A platform Container id
object_key : str
The target S3 object key
stream : io.RawIOBase
The stream to use as data source
Expand source code 
def upload_stream(bucket, container_id, object_key, stream):
    """Upload a file to an S3 `bucket` using a given platform `Container` id to a target `object_key`,
    using `stream` as the data source to upload

    Args:
        bucket (str): The target S3 bucket
        container_id (int): A platform `Container` id
        object_key (str): The target S3 object key
        stream (io.RawIOBase): The stream to use as data source
    """

    __upload_stream(bucket, str(container_id) + '/' + object_key.lstrip("/"), stream)