RFC: first draft of insertion API

.gitignore

@@ -1,17 +1,21 @@
+# Created by .gitignore support plugin (hsz.mobi)
+### Python template
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 
+# tmp/backup files
+*~
 # C extensions
 *.so
-
 # Distribution / packaging
 .Python
 env/
 bin/
 build/
 develop-eggs/
 dist/
+downloads/
 eggs/
 lib/
 lib64/
@@ -22,11 +26,14 @@ var/
 .installed.cfg
 *.egg
 doc/_build
-
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
-
 # Unit test / coverage reports
 htmlcov/
 .tox/
@@ -35,57 +42,47 @@ cover/
 .cache
 nosetests.xml
 coverage.xml
-cover/
-
+cover/*
 # Translations
 *.mo
-
 # Mr Developer
 .mr.developer.cfg
 .project
 .pydevproject
-
 # Rope
 .ropeproject
-
 # Django stuff:
 *.log
 *.pot
-
 # Sphinx documentation
 docs/_build/
-
 #mac
 .DS_Store
-*~
-
 #pycharm
 .idea/*
-
 #Dolphin browser files
 .directory/
 .directory
-
 #Binary data files
 *.volume
 *.am
 *.tiff
 *.tif
 *.dat
 *.DAT
-
 #generated documntation files
 generated/
-
 #ipython notebook
 .ipynb_checkpoints/
-
 #vim
 *.swp
-
 #data files
 *.zip
 *.jpg
-
 # ctags
 .tags*
+# PyBuilder
+target/
+# PyCharm
+.idea/
+#ipython notebook stuff

.travis.yml

@@ -0,0 +1,18 @@
+language: python
+
+python:
+  - 3.5
+  - "nightly"
+
+cache:
+  directories:
+    # cache for wheels generated by pip
+    - $HOME/.cache/pip
+
+install:
+  - python setup.py install
+  - pip install six pytest pytest-cov pandas
+  - 'pip install https://github.com/tacaswell/document/zipball/master#egg=document'
+
+script:
+  - py.test --cov-report term-missing --cov=analysisbucket.client

analysisbucket/client.py

@@ -0,0 +1,233 @@
+from functools import singledispatch
+import numpy as np
+import pandas as pd
+from uuid import uuid4
+import os
+import os.path
+import pathlib
+import shutil
+from time import time as ttime
+
+from document import Document
+BASE_PATH = os.path.expanduser('~/.cache/ab')
+
+
+class HeaderDocument(Document):
+    __slots__ = ('_ac')
+
+    def __init__(self, bound_ac, name, *args, **kwargs):
+        object.__setattr__(self, '_ac', bound_ac)
+        super().__init__(name, *args, **kwargs)
+
+    def Document(self):
+        return Document(*self.to_name_dict_pair())
+
+    def add_result(self, **data):
+        return self._ac.add_result(self, **data)
+
+
+class AnalysisClient:
+    def __init__(self, *, norm=None):
+        if norm is None:
+            norm = normalize
+        self._normalize = norm
+
+    def create_header(self, name, **kwargs):
+        """Create an analysis Header
+
+        Create the header for an analysis run.
+
+        The schema for this needs to be filled in.
+
+        Parameters
+        ----------
+        name : str
+            Name of the analysis pipeline that this header is
+            recording
+
+        Returns
+        -------
+        a_head : doc
+           Document (dict) representing the header
+        """
+        # TODO talk to database
+        return HeaderDocument(self, 'result_header',
+                              {'time': ttime(),
+                               'uid': str(uuid4()),
+                               'name': name,
+                               **kwargs})
+
+    def add_result(self, head, **data):
+        """Add results to a header
+
+        Auto-magically generate result documents to go with
+        the given header.  All key word arguments are bundled into
+        the result Document.
+
+        Parameters
+        ----------
+        head: Document
+            Document of the header to attach the result to
+        data: type
+            description
+
+        Returns
+        -------
+        result : Document
+        """
+        data_dict = {}
+        data_keys = {}
+        for k, v in data.items():
+            nd, kd = self._normalize(v)
+            data_dict[k] = nd
+            data_keys[k] = kd
+
+        desc = self.add_result_descriptor(head, data_keys, auto_gen=True)
+
+        res = self.add_result_document(desc, data_dict)
+        return res
+
+    def add_result_document(self, descriptor, data_doc):
+        """Low-level add result
+
+        This method assumes you have done most of the hard work (ex
+        dealing with filestore) your self.
+
+        Parameters
+        ----------
+        descriptor : Document
+            The result descriptor that describes the document
+
+        data_doc : Document
+            A (almost?) full result document.
+
+        Returns
+        -------
+        res : Document
+            The actual document inserted into the database
+
+        """
+        # TODO talk to database
+        return Document('result', {'data': data_doc,
+                                   'descriptor': descriptor,
+                                   'uid': str(uuid4())})
+
+    def add_result_descriptor(self, head, data_keys, **kwargs):
+        """Create an result descriptor
+
+        """
+        # TODO talk to database
+        return Document('res_desc', {'header': head,
+                                     'data_keys': data_keys,
+                                     'uid': str(uuid4()),
+                                     **kwargs})
+
+
+@singledispatch
+def normalize(data):
+    """normalize data for storage
+
+    Parameters
+    ----------
+    data : object
+       The data to be stored
+
+    Returns
+    -------
+    data : object
+       Normalized data to be shoved into the document store (eg mongo)
+
+    data_key : dict
+       Entry for the descriptor document
+
+    """
+    return data, {}
+
+
+@normalize.register(np.ndarray)
+def _norm_np(data):
+    os.makedirs(BASE_PATH, exist_ok=True)
+    fname = os.path.join(BASE_PATH,  str(uuid4()) + '.npy')
+    np.save(fname, data)
+    return fname, {'shape': data.shape,
+                   'dtype': 'array',
+                   'external': 'FILEPATH:npy'}
+
+
+@normalize.register(pd.DataFrame)
+def _norm_pd_df(data):
+    os.makedirs(BASE_PATH, exist_ok=True)
+    fname = os.path.join(BASE_PATH,  str(uuid4()) + '.csv')
+    data.to_csv(fname)
+    return fname, {'shape': data.shape,
+                   'dtype': 'table',
+                   'external': 'FILEPATH:csv',
+                   'columns': list(data.columns)}
+
+
+@normalize.register(pd.Series)
+def _norm_pd_S(data):
+    # TODO this does not properly round trip, comes back as a
+    # DataFrame, not a Series.  Via FS this would be easy to do.
+    return normalize(pd.DataFrame(data))
+
+
+@normalize.register(pathlib.Path)
+def _norm_path(data):
+    os.makedirs(BASE_PATH, exist_ok=True)
+    old_name, ext = os.path.splitext(data.name)
+    fname = os.path.join(BASE_PATH, str(uuid4()) + ext)
+    data = data.absolute()
+    shutil.copy2(data.as_posix(), fname)
+    return fname, {'shape': (),
+                   'dtype': 'path'}
+
+
+def open_file(data, key_desc):
+    """A quick and dirty FS stand-in
+
+    This handles the case where:
+      - exactly one datum per file
+      - no parameters other than the handler and filename
+        are required
+
+    Parameters
+    ----------
+    data : str
+        The data to open as a file
+
+    key_desc : dict
+        The data_key entry for this value
+
+    Returns
+    -------
+    data : object
+        However the file should be interpreted
+    """
+    handler_map = {'csv': lambda fname: pd.read_csv(fname, index_col=0),
+                   'npy': np.load}
+
+    klass, ext = key_desc['external'].split(':')
+    assert klass == 'FILEPATH'
+    return handler_map[ext](data)
+
+
+def fill_result(res):
+    data = res['data']
+    data_keys = res['descriptor']['data_keys']
+    out = {}
+    for k in data.keys():
+        dk = data_keys[k]
+        d = data[k]
+        if 'external' in dk:
+            d = open_file(d, dk)
+        out[k] = d
+
+    if isinstance(res, Document):
+        _name, r_dict = res.to_name_dict_pair()
+        r_dict['data'] = out
+        return Document(_name, r_dict)
+    else:
+        r_dict = dict(res)
+        r_dict['data'] = out
+        return r_dict

analysisbucket/tests/__init__.py

@@ -0,0 +1 @@
+