# -*- coding: utf-8 -*-
"""
Source
======
This module provides the base class for data sources and sinks.
"""
__author__ = 'Christoph Kirst <christoph.kirst.ck@gmail.com>'
__license__ = 'GPLv3 - GNU General Pulic License v3 (see LICENSE.txt)'
__copyright__ = 'Copyright © 2020 by Christoph Kirst'
__webpage__ = 'http://idisco.info'
__download__ = 'http://www.github.com/ChristophKirst/ClearMap2'
import numpy as np
import ClearMap.IO.FileUtils as fu
from ClearMap.Utils.Formatting import ensure
###############################################################################
### Source base class
###############################################################################
[docs]class Source(object):
"""Base abstract source class."""
#__slots__ = ();
def __init__(self, name = None):
"""Initialization."""
if name is not None:
self._name = name;
@property
def name(self):
"""The name of this source.
Returns
-------
name : str
Name of this source.
"""
if hasattr(self, '_name'):
return self._name;
else:
return type(self).__name__;
@name.setter
def name(self, value):
self._name = ensure(value, str);
@property
def shape(self):
"""The shape of the source.
Returns
-------
shape : tuple
The shape of the source.
"""
return None;
@shape.setter
def shape(self, value):
raise ValueError('Cannot set shape for this source.');
@property
def dtype(self):
"""The data type of the source.
Returns
-------
dtype : dtype
The data type of the source.
"""
return None;
@dtype.setter
def dtype(self, value):
raise ValueError('Cannot set dtype for this source.');
@property
def order(self):
"""The contiguous order of the underlying data array.
Returns
-------
order : str
Returns 'C' for C contigous and 'F' for fortran contigous, None otherwise.
"""
return None;
@order.setter
def order(self, value):
raise ValueError('Cannot set order for this source.');
# @property
# def memory(self):
# """The memory type of the source.
#
# Returns
# -------
# memory : str
# Returns 'shared' for a shared memory buffer, 'memmap' for a memory map to a file, None otherwise.
# """
# return None;
#
# @memory.setter
# def memory(self, value):
# raise ValueError('Cannot set memory type for this source.');
@property
def location(self):
"""The location where the data of the source is stored.
Returns
-------
location : str or None
Returns the location of the data source or None if this source lives in memory only.
"""
return None;
@location.setter
def location(self, value):
raise ValueError('Cannot set location for this source.');
### Derived properties
@property
def ndim(self):
"""The number of dimensions of the source.
Returns
-------
ndim : int
The number of dimension of the source.
"""
return len(self.shape);
@property
def size(self):
"""The size of the source.
Returns
-------
size : int
The number of data items in the source.
"""
return np.prod(self.shape);
# @property
# def shared(self):
# """Returns True if the source is in shared memory.
#
# Returns
# -------
# shared : bool
# True if the source is in shared memory.
# """
# return self.memory == 'shared';
#
# @shared.setter
# def shared(self, value):
# if value is True:
# self.memory == 'shared';
# elif self.memory == 'shared':
# self.memory = None;
#
#
# @property
# def memmap(self):
# """Returns True if the source is a memory map.
#
# Returns
# -------
# memmap : bool
# True if the source is memory mapped.
# """
# return self.memory == 'memmap';
#
# @memmap.setter
# def memmap(self, value):
# if value is True:
# self.memory == 'memmap';
# elif self.memory == 'memmap':
# self.memory = None;
### Functionality
[docs] def exists(self):
if self.location is not None:
return fu.is_file(self.location);
else:
return False;
### Source conversions
[docs] def as_virtual(self):
"""Return virtual source without array data to pass in parallel processing.
Returns
-------
source : Source class
The source class without array data.
"""
#return VirtualSource(source = self.source);
raise NotImplementedError('virtual source not implemented for this source!')
[docs] def as_real(self):
return self;
[docs] def as_buffer(self):
raise NotImplementedError('buffer not implemented for this source!')
[docs] def as_memory(self):
return np.array(self.as_buffer());
### Data
def __getitem__(self, *args):
raise KeyError('No getitem routine for this source!')
def __setitem__(self, *args):
raise KeyError('No setitem routine for this source!')
[docs] def read(self, *args, **kwargs):
raise KeyError('No read routine for this source!');
[docs] def write(self, *args, **kwargs):
raise KeyError('No write routine for this source!')
### Formatting
def __str__(self):
try:
name = self.name;
name = '%s' % name if name is not None else '';
except:
#print('name')
name ='';
try:
shape = self.shape
shape ='%r' % ((shape,)) if shape is not None else '';
except:
#print('shape')
shape = '';
try:
dtype = self.dtype;
dtype = '[%s]' % dtype if dtype is not None else '';
except:
#print('dtype')
dtype = '';
try:
order = self.order;
order = '|%s|' % order if order is not None else '';
except:
#print('order')
order = '';
# try:
# memory = self.memory;
# memory = '<%s>' % memory if memory is not None else '';
# except:
# memory = '';
try:
location = self.location;
location = '%s' % location if location is not None else '';
if len(location) > 100:
location = location[:50] + '...' + location[-50:]
if len(location) > 0:
location = '{%s}' % location;
except:
#print('location')
location = '';
# try:
# array = self.array.__str__();
# if len(array) > 100:
# e = array[100:].find('\n');
# if e != -1:
# array = array[:100 + e] + '...';
# if len(array) > 0:
# array = '\n' + array;
# except:
# array = '';
return name + shape + dtype + order + location; # + array
def __repr__(self):
return self.__str__();
###############################################################################
### Abstract and VirtualSource base class
###############################################################################
#TODO: memory -> device argument
[docs]class AbstractSource(Source):
"""Abstract source to handle data sources without data in memory.
Note
----
This class handles essential info about a source and to how access its data.
"""
#__slots__ = ('_shape', '_dtype', '_order', '_location')
def __init__(self, source = None, shape = None, dtype = None, order = None, location = None, name = None):
"""Source class construtor.
Arguments
---------
shape : tuple of int or None
Shape of the source, if None try to determine from source.
dtype : dtype or None
The data type of the source, if None try to detemrine from source.
order : 'C' or 'F' or None
The order of the source, c or fortran contiguous.
memory : str or None
The memory type of the source, 'memmap' uses memory mapped array and 'shared'returns a shared memory.
location : str or None
The location of the source.
"""
super(AbstractSource, self).__init__(name = name);
if source is not None:
if shape is None and hasattr(source, 'shape'):
shape = source.shape;
if dtype is None and hasattr(source, 'dtype'):
dtype = source.dtype;
if order is None and hasattr(source, 'order'):
order = source.order;
#if memory is None and hasattr(source, 'memory'):
# memory = memory.order;
if location is None and hasattr(source, 'location'):
location = source.location;
self._shape = ensure(shape, tuple);
self._dtype = ensure(dtype, np.dtype);
self._order = ensure(order, str);
#self._memory = ensure(memory, str);
self._location = ensure(location, str);
@property
def shape(self):
"""The shape of the source.
Returns
-------
shape : tuple
The shape of the source.
"""
return self._shape;
@shape.setter
def shape(self, value):
self._shape = ensure(value, tuple);
@property
def dtype(self):
"""The data type of the source.
Returns
-------
dtype : dtype
The data type of the source.
"""
return self._dtype;
@dtype.setter
def dtype(self, value):
self._dtype = ensure(value, np.dtype);
@property
def order(self):
"""The continguous order of the data array of the source.
Returns
-------
order : str
Returns 'C' for C and 'F' for fortran contiguous arrays, None otherwise.
"""
return self._order;
@order.setter
def order(self, value):
if value not in [None, 'C', 'F']:
raise ValueError("Order %r not in [None, 'C' or 'F']!" % value);
self._order = ensure(value, str);
@property
def location(self):
"""The location of the source's data.
Returns
-------
location : str or None
Returns the location of the data source or None if there is none.
"""
return self._location;
@location.setter
def location(self, value):
self._location = ensure(value, str);
[docs] def as_virtual(self):
return self;
[docs] def as_real(self):
raise RuntimeError('The abstract source cannot be converted to a real source!')
[docs] def as_buffer(self):
raise RuntimeError('The abstract source cannot be converted to a buffer!')
[docs]class VirtualSource(AbstractSource):
"""Virtual source to handle data sources without data in memory.
Note
----
This class is fast to serialize and useful as a source pointer in paralle processing.
"""
def __init__(self, source = None, shape = None, dtype = None, order = None, location = None, name = None):
AbstractSource.__init__(self, source=source, shape=shape, dtype=dtype, order=order, location=location, name=name)
def __getitem__(self, *args):
return self.as_real().__getitem__(*args);
def __setitem__(self, *args):
self.as_real().__setitem__(*args);
[docs] def read(self, *args, **kwargs):
return self.as_real().read(*args, **kwargs);
[docs] def write(self, *args, **kwargs):
self.as_real().write(*args, **kwargs);
#def __getattr__(self, name):
# return getattr(self.as_real(), name);
###############################################################################
### Tests
###############################################################################
def _test():
import ClearMap.IO.Source as src
#reload(src)
s = src.VirtualSource(shape = (50,50), dtype = float, location = '/home/test.npy', order = 'F')
print(s)
s.size
s.ndim
Source
