From c253e25d573523a07b69e9740f9b49359a016d13 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 00:52:51 -0500 Subject: [PATCH 01/13] ENH: first draft of insertion API --- analysisbucket/client.py | 149 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 analysisbucket/client.py diff --git a/analysisbucket/client.py b/analysisbucket/client.py new file mode 100644 index 0000000..7c99da3 --- /dev/null +++ b/analysisbucket/client.py @@ -0,0 +1,149 @@ +from functools import singledispatch +import numpy as np +import pandas as pd +from uuid import uuid4 +import os +import os.path +import pathlib +import shutil + +BASE_PATH = os.path.expanduser('~/.cache/ab') + + +class AnalysisClient: + def create_header(self, name, **kwargs): + """Create an analysis Header + + Create the header for an analysis run. + + The schema for this needs to be filled in. + + Parameters + ---------- + name : str + Name of the analysis pipeline that this header is + recording + + Returns + ------- + a_head : doc + Document (dict) representing the header + """ + pass + + def add_result(self, head, **data): + """Add results to a header + + Auto-magically generate result documents to go with + the given header. All key word arguments are bundled into + the result Document. + + Parameters + ---------- + head: Document + Document of the header to attach the result to + data: type + description + + Returns + ------- + result : Document + """ + data_dict = {} + data_keys = {} + for k, v in data.items(): + nd, kd = normalize(v) + data_dict[k] = nd + data_keys[k] = kd + + desc = self.add_result_descriptor(head, data_keys, auto_gen=True) + + return {'data': data_dict, 'descriptor': desc} + + def add_result_document(self, descriptor, data_doc): + """Low-level add result + + This method assumes you have done most of the hard work (ex + dealing with filestore) your self. + + Parameters + ---------- + descriptor : Document + The result descriptor that describes the document + + data_doc : Document + A (almost?) full result document. + + Returns + ------- + res : Document + The actual document inserted into the database + + """ + pass + + def add_result_descriptor(self, head, data_keys, **kwargs): + """Create an result descriptor + + """ + return {'header': head, + 'data_keys': data_keys, + **kwargs} + + +@singledispatch +def normalize(data): + """normalize data for storage + + Parameters + ---------- + data : object + The data to be stored + + Returns + ------- + data : object + Normalized data to be shoved into the document store (eg mongo) + + data_key : dict + Entry for the descriptor document + + """ + return data, {} + + +@normalize.register(np.ndarray) +def _norm_np(data): + os.makedirs(BASE_PATH, exist_ok=True) + fname = os.path.join(BASE_PATH, str(uuid4()) + '.npy') + np.save(fname, data) + return fname, {'shape': data.shape, + 'dtype': 'array', + 'external': 'FILEPATH:npy'} + + +@normalize.register(pd.DataFrame) +def _norm_pd_df(data): + os.makedirs(BASE_PATH, exist_ok=True) + fname = os.path.join(BASE_PATH, str(uuid4()) + '.csv') + data.to_csv(fname) + return fname, {'shape': data.shape, + 'dtype': 'table', + 'external': 'FILEPATH:csv', + 'columns': list(data.columns)} + + +@normalize.register(pd.Series) +def _norm_pd_S(data): + return normalize(pd.DataFrame(data)) + + +@normalize.register(pathlib.Path) +def _norm_path(data): + os.makedirs(BASE_PATH, exist_ok=True) + old_name, ext = os.path.splitext(data.name) + fname = os.path.join(BASE_PATH, str(uuid4()) + ext) + data = data.absolute() + shutil.copy2(data.as_posix(), fname) + return fname, {'shape': (), + 'dtype': 'path'} From 4d9623decdc09d155cf91e39e69f1409279aafa5 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 18:28:52 -0500 Subject: [PATCH 02/13] ENH: make the instances have their own _normalize Over engineering --- analysisbucket/client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index 7c99da3..5d5783f 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -11,6 +11,11 @@ class AnalysisClient: + def __init__(self, *, norm=None): + if norm is None: + norm = normalize + self._normalize = norm + def create_header(self, name, **kwargs): """Create an analysis Header @@ -52,7 +57,7 @@ def add_result(self, head, **data): data_dict = {} data_keys = {} for k, v in data.items(): - nd, kd = normalize(v) + nd, kd = self._normalize(v) data_dict[k] = nd data_keys[k] = kd From b999d8017bebe1ef856bfd09fbe31c146b72691e Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 18:29:37 -0500 Subject: [PATCH 03/13] ENH: fleshout document generation a bit --- analysisbucket/client.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index 5d5783f..5a17faf 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -6,6 +6,7 @@ import os.path import pathlib import shutil +from time import time as ttime BASE_PATH = os.path.expanduser('~/.cache/ab') @@ -34,7 +35,11 @@ def create_header(self, name, **kwargs): a_head : doc Document (dict) representing the header """ - pass + # TODO talk to database + return {'date': ttime(), + 'uid': str(uuid4()), + 'name': name, + **kwargs} def add_result(self, head, **data): """Add results to a header @@ -63,7 +68,8 @@ def add_result(self, head, **data): desc = self.add_result_descriptor(head, data_keys, auto_gen=True) - return {'data': data_dict, 'descriptor': desc} + res = self.add_result_document(desc, data_dict) + return res def add_result_document(self, descriptor, data_doc): """Low-level add result @@ -85,14 +91,19 @@ def add_result_document(self, descriptor, data_doc): The actual document inserted into the database """ - pass + # TODO talk to database + return {'data': data_doc, + 'descriptor': descriptor, + 'uid': str(uuid4())} def add_result_descriptor(self, head, data_keys, **kwargs): """Create an result descriptor """ + # TODO talk to database return {'header': head, 'data_keys': data_keys, + 'uid': str(uuid4()), **kwargs} From 1b35b51d4e4b73cfe8555031df0e5cec02395fea Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 19:07:00 -0500 Subject: [PATCH 04/13] ENH: quick function for by-passing FS --- analysisbucket/client.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index 5a17faf..d1d443e 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -163,3 +163,32 @@ def _norm_path(data): shutil.copy2(data.as_posix(), fname) return fname, {'shape': (), 'dtype': 'path'} + + +def open_file(data, key_desc): + """A quick and dirty FS stand-in + + This handles the case where: + - exactly one datum per file + - no parameters other than the handler and filename + are required + + Parameters + ---------- + data : str + The data to open as a file + + key_desc : dict + The data_key entry for this value + + Returns + ------- + data : object + However the file should be interpreted + """ + handler_map = {'csv': lambda fname: pd.read_csv(fname, index_col=0), + 'npy': np.load} + + klass, ext = key_desc['external'].split(':') + assert klass == 'FILEPATH' + return handler_map[ext](data) From c49968ab7ab4a70dbd676c607e1e910aa7708f1c Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 21:36:45 -0500 Subject: [PATCH 05/13] ENH: return Document objects This depends on the stand-alone document package --- analysisbucket/client.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index d1d443e..7e33884 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -8,6 +8,7 @@ import shutil from time import time as ttime +from document import Document BASE_PATH = os.path.expanduser('~/.cache/ab') @@ -36,10 +37,11 @@ def create_header(self, name, **kwargs): Document (dict) representing the header """ # TODO talk to database - return {'date': ttime(), - 'uid': str(uuid4()), - 'name': name, - **kwargs} + return Document('result_header', + {'date': ttime(), + 'uid': str(uuid4()), + 'name': name, + **kwargs}) def add_result(self, head, **data): """Add results to a header @@ -92,19 +94,19 @@ def add_result_document(self, descriptor, data_doc): """ # TODO talk to database - return {'data': data_doc, - 'descriptor': descriptor, - 'uid': str(uuid4())} + return Document('result', {'data': data_doc, + 'descriptor': descriptor, + 'uid': str(uuid4())}) def add_result_descriptor(self, head, data_keys, **kwargs): """Create an result descriptor """ # TODO talk to database - return {'header': head, - 'data_keys': data_keys, - 'uid': str(uuid4()), - **kwargs} + return Document('res_desc', {'header': head, + 'data_keys': data_keys, + 'uid': str(uuid4()), + **kwargs}) @singledispatch From fb4fedea126c9101d6391c9b980bcfad44966c6e Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 21:54:54 -0500 Subject: [PATCH 06/13] ENH: add 'magic' Document for headers --- analysisbucket/client.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index 7e33884..3c24776 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -12,6 +12,20 @@ BASE_PATH = os.path.expanduser('~/.cache/ab') +class HeaderDocument(Document): + __slots__ = ('_ac') + + def __init__(self, bound_ac, name, *args, **kwargs): + object.__setattr__(self, '_ac', bound_ac) + super().__init__(name, *args, **kwargs) + + def Document(self): + return Document(*self.to_name_dict_pair()) + + def add_result(self, **data): + return self._ac.add_result(self, **data) + + class AnalysisClient: def __init__(self, *, norm=None): if norm is None: @@ -37,11 +51,11 @@ def create_header(self, name, **kwargs): Document (dict) representing the header """ # TODO talk to database - return Document('result_header', - {'date': ttime(), - 'uid': str(uuid4()), - 'name': name, - **kwargs}) + return HeaderDocument(self, 'result_header', + {'date': ttime(), + 'uid': str(uuid4()), + 'name': name, + **kwargs}) def add_result(self, head, **data): """Add results to a header From 6db6e2de663ccf30428fa625673964e7648e7e8a Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 22:49:02 -0500 Subject: [PATCH 07/13] MNT: date -> time We use the time convention everywhere else. --- analysisbucket/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index 3c24776..a35620a 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -52,7 +52,7 @@ def create_header(self, name, **kwargs): """ # TODO talk to database return HeaderDocument(self, 'result_header', - {'date': ttime(), + {'time': ttime(), 'uid': str(uuid4()), 'name': name, **kwargs}) From 9d50ab9d0192a986b72e9c59b350613e34b82225 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 22:50:11 -0500 Subject: [PATCH 08/13] BUG: series normalization only sortof work --- analysisbucket/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index a35620a..188e897 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -167,6 +167,8 @@ def _norm_pd_df(data): @normalize.register(pd.Series) def _norm_pd_S(data): + # TODO this does not properly round trip, comes back as a + # DataFrame, not a Series. Via FS this would be easy to do. return normalize(pd.DataFrame(data)) From 671c61352e2a353d8f0fec91b8d338008deeca9b Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 23:33:06 -0500 Subject: [PATCH 09/13] ENH: add data-broker light functionality This is here for proof-of-concept testing the API, may not survive. --- analysisbucket/client.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/analysisbucket/client.py b/analysisbucket/client.py index 188e897..25cd801 100644 --- a/analysisbucket/client.py +++ b/analysisbucket/client.py @@ -210,3 +210,24 @@ def open_file(data, key_desc): klass, ext = key_desc['external'].split(':') assert klass == 'FILEPATH' return handler_map[ext](data) + + +def fill_result(res): + data = res['data'] + data_keys = res['descriptor']['data_keys'] + out = {} + for k in data.keys(): + dk = data_keys[k] + d = data[k] + if 'external' in dk: + d = open_file(d, dk) + out[k] = d + + if isinstance(res, Document): + _name, r_dict = res.to_name_dict_pair() + r_dict['data'] = out + return Document(_name, r_dict) + else: + r_dict = dict(res) + r_dict['data'] = out + return r_dict From d1cdc53b97efa038ceab7215138e70ebb03494e1 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 23 Dec 2015 23:56:22 -0500 Subject: [PATCH 10/13] TST: add full test coverage for client.py --- analysisbucket/tests/__init__.py | 1 + analysisbucket/tests/test_client.py | 60 ++++++++++++++++++ analysisbucket/tests/test_norm.py | 96 +++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 analysisbucket/tests/__init__.py create mode 100644 analysisbucket/tests/test_client.py create mode 100644 analysisbucket/tests/test_norm.py diff --git a/analysisbucket/tests/__init__.py b/analysisbucket/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/analysisbucket/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/analysisbucket/tests/test_client.py b/analysisbucket/tests/test_client.py new file mode 100644 index 0000000..b325a6a --- /dev/null +++ b/analysisbucket/tests/test_client.py @@ -0,0 +1,60 @@ +from analysisbucket.client import AnalysisClient, fill_result +from numpy.testing import assert_array_equal +from pandas.util.testing import assert_frame_equal + +import numpy as np +import pandas as pd + + +def test_header_creation(): + ac = AnalysisClient() + md = {k: ord(k) for k in 'abcdef'} + h = ac.create_header('testing', uid='fixed', time=0, **md) + + for k, v in md.items(): + assert h[k] == v + + assert h.name == 'testing' + assert h.uid == 'fixed' + assert h.time == 0 + + +def test_add_result(): + ac = AnalysisClient() + + md = {k: ord(k) for k in 'abcdef'} + + h = ac.create_header('testing', uid='fixed', time=0, **md) + h = h.Document() + + tab = pd.DataFrame({'a': range(15), 'b': 'a'*15}) + arr = np.arange(15) + + res = ac.add_result(h, tab=tab, arr=arr) + + for d in [res['data'], res.descriptor.data_keys]: + for k in ['tab', 'arr']: + assert k in d + res = fill_result(res) + + assert_array_equal(res.data['arr'], arr) + assert_frame_equal(res.data['tab'], tab) + + +def test_HeaderDocument_add_result(): + ac = AnalysisClient() + + md = {k: ord(k) for k in 'abcdef'} + + h = ac.create_header('testing', uid='fixed', time=0, **md) + + res = h.add_result(a=1, b=2) + + for d in [res['data'], res.descriptor.data_keys]: + for k in ['a', 'b']: + assert k in d + _name, res = res.to_name_dict_pair() + res = fill_result(res) + + assert res['data']['a'] == 1 + assert res['data']['b'] == 2 diff --git a/analysisbucket/tests/test_norm.py b/analysisbucket/tests/test_norm.py new file mode 100644 index 0000000..5176566 --- /dev/null +++ b/analysisbucket/tests/test_norm.py @@ -0,0 +1,96 @@ +from pathlib import Path +from uuid import uuid4 + +from analysisbucket.client import normalize, open_file + + +import numpy as np +from numpy.testing import assert_array_equal + +import pandas as pd +from pandas.util.testing import assert_frame_equal, assert_series_equal + + +def _norm_basic_helper(data): + r, dk = normalize(data) + assert dk == {} + assert r == data + + +def test_norm_basic(): + for d in (1, 'abc', [1, 2, 3]): + yield _norm_basic_helper, d + + +def _norm_np_helper(data): + r, dk = normalize(data) + assert isinstance(r, str) + rd = open_file(r, dk) + assert_array_equal(rd, data) + assert dk['shape'] == data.shape + assert dk['external'] == 'FILEPATH:npy' + assert dk['dtype'] == 'array' + + +def test_norm_np(): + tests = [np.arange(5), + np.linspace(0, 1, 20), + np.arange(25).reshape(5, 5)] + for d in tests: + yield _norm_np_helper, d + + +def _norm_pdS_helper(data): + r, dk = normalize(data) + assert isinstance(r, str) + rd = open_file(r, dk) + assert dk['external'] == 'FILEPATH:csv' + assert dk['dtype'] == 'table' + # account for the to disk process upcasting to DataFrame + assert dk['shape'] == data.shape + (1, ) + assert_series_equal(data, rd[data.name]) + + +def _norm_pddf_helper(data): + r, dk = normalize(data) + assert isinstance(r, str) + rd = open_file(r, dk) + assert dk['external'] == 'FILEPATH:csv' + assert dk['dtype'] == 'table' + assert dk['shape'] == data.shape + assert_frame_equal(data, rd) + + +def test_norm_pd(): + helper_mapper = {pd.DataFrame: _norm_pddf_helper, + pd.Series: _norm_pdS_helper} + dd = pd.DataFrame({'a': range(15), 'b': 'a'*15}) + tests = [dd, dd['a']] + + for t in tests: + yield helper_mapper[type(t)], t + + +def test_norm_path(): + p = Path('/tmp/{}.txt'.format(str(uuid4()))) + + lorem_ipsum = """Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do +eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad +minim veniam, quis nostrud exercitation ullamco laboris nisi ut +aliquip ex ea commodo consequat. Duis aute irure dolor in +reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla +pariatur. Excepteur sint occaecat cupidatat non proident, sunt in +culpa qui officia deserunt mollit anim id est laborum.""" + + with p.open(mode='w') as f: + f.write(lorem_ipsum) + + r, dk = normalize(p) + + with open(r, 'r') as f: + for inp, out in zip(lorem_ipsum.split('\n'), f): + assert inp.strip() == out.strip() + + with open(r, 'r') as f1, p.open() as f2: + for inp, out in zip(f1, f2): + assert inp == out From db411ecb872efb4f630dd57d5b52a78658ec18bb Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Thu, 24 Dec 2015 00:03:05 -0500 Subject: [PATCH 11/13] DEV: add .gitignore --- .gitignore | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 1a1d8e2..b8a7a8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,13 @@ +# Created by .gitignore support plugin (hsz.mobi) +### Python template # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +# tmp/backup files +*~ # C extensions *.so - # Distribution / packaging .Python env/ @@ -12,6 +15,7 @@ bin/ build/ develop-eggs/ dist/ +downloads/ eggs/ lib/ lib64/ @@ -22,11 +26,14 @@ var/ .installed.cfg *.egg doc/_build - +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt pip-delete-this-directory.txt - # Unit test / coverage reports htmlcov/ .tox/ @@ -35,37 +42,27 @@ cover/ .cache nosetests.xml coverage.xml -cover/ - +cover/* # Translations *.mo - # Mr Developer .mr.developer.cfg .project .pydevproject - # Rope .ropeproject - # Django stuff: *.log *.pot - # Sphinx documentation docs/_build/ - #mac .DS_Store -*~ - #pycharm .idea/* - #Dolphin browser files .directory/ .directory - #Binary data files *.volume *.am @@ -73,19 +70,19 @@ docs/_build/ *.tif *.dat *.DAT - #generated documntation files generated/ - #ipython notebook .ipynb_checkpoints/ - #vim *.swp - #data files *.zip *.jpg - # ctags .tags* +# PyBuilder +target/ +# PyCharm +.idea/ +#ipython notebook stuff From f2fcd4e53e1a3dfc2782f371cd2c8101b33f48a8 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Thu, 24 Dec 2015 00:03:23 -0500 Subject: [PATCH 12/13] BLD: add basic setup.py only deals with client.py right now --- setup.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..86fcf44 --- /dev/null +++ b/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup + +setup(name='analysisbucket', + version='0.0.0', + author='Brookhaven National Laboratory', + py_modules=['analysisbucket'], + description='', + requires=['six'] + ) From acec333e41cadd53571428a41ba3667a6911887b Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Thu, 24 Dec 2015 00:03:43 -0500 Subject: [PATCH 13/13] TST: add bare-bones travis.yml --- .travis.yml | 18 ++++++++++++++++++ tools/travis_tools.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 .travis.yml create mode 100644 tools/travis_tools.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0595aa5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python + +python: + - 3.5 + - "nightly" + +cache: + directories: + # cache for wheels generated by pip + - $HOME/.cache/pip + +install: + - python setup.py install + - pip install six pytest pytest-cov pandas + - 'pip install https://github.com/tacaswell/document/zipball/master#egg=document' + +script: + - py.test --cov-report term-missing --cov=analysisbucket.client diff --git a/tools/travis_tools.sh b/tools/travis_tools.sh new file mode 100644 index 0000000..0710891 --- /dev/null +++ b/tools/travis_tools.sh @@ -0,0 +1,26 @@ +# Tools for working with travis-ci +export WHEELHOST="travis-wheels.scikit-image.org" +export WHEELHOUSE="http://${WHEELHOST}/" + +retry () { + # https://gist.github.com/fungusakafungus/1026804 + local retry_max=5 + local count=$retry_max + while [ $count -gt 0 ]; do + "$@" && break + count=$(($count - 1)) + sleep 1 + done + + [ $count -eq 0 ] && { + echo "Retry failed [$retry_max]: $@" >&2 + return 1 + } + return 0 +} + + +wheelhouse_pip_install() { + # Install pip requirements via travis wheelhouse + retry pip install --timeout=60 --no-index --trusted-host $WHEELHOST --find-links $WHEELHOUSE $@ +}