save_trajs support for fragmented data source (#1514)

* save_trajs support for fragmented data source

* update build matrix

* use deeptime clustering backend

Author: clonker

.circleci/config.yml

@@ -21,7 +21,7 @@ jobs:
       - run: conda config --set quiet true
       - run: conda install conda-build -c defaults
       - run: mkdir workspace
-      - run: conda build devtools/conda-recipe --python=3.7 --no-test --output-folder workspace
+      - run: conda build devtools/conda-recipe --python=3.9 --numpy=1.19 --no-test --output-folder workspace
       - persist_to_workspace:
           root: workspace
           paths:
@@ -46,7 +46,7 @@ jobs:
       - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
       - attach_workspace:
           at: workspace
-      - run: conda build devtools/conda-recipe --python=3.7 --test --output-folder workspace
+      - run: conda build devtools/conda-recipe --python=3.9 --numpy=1.19 --test --output-folder workspace
       - run: bash <(curl -s https://codecov.io/bash) -f $HOME/coverage.xml
       - store_test_results:
           path: /tmp/circleci-test-results

devtools/azure-pipelines-linux.yml

@@ -6,15 +6,12 @@ jobs:
 
   strategy:
     matrix:
-      Python36:
-        CONDA_PY: '3.6'
-        CONDA_NPY: '1.16'
       Python37:
         CONDA_PY: '3.7'
         CONDA_NPY: '1.18'
       Python38:
         CONDA_PY: '3.8'
-        CONDA_NPY: '1.19'
+        CONDA_NPY: '1.18'
       Python39:
         CONDA_PY: '3.9'
         CONDA_NPY: '1.19'

devtools/azure-pipelines-osx.yml

@@ -5,12 +5,12 @@ jobs:
     vmImage: 'macOS-10.14'
   strategy:
     matrix:
-      Python36:
-        CONDA_PY: '3.6'
-        CONDA_NPY: '1.17'
       Python37:
         CONDA_PY: '3.7'
-        CONDA_NPY: '1.19'
+        CONDA_NPY: '1.18'
+      Python38:
+        CONDA_PY: '3.8'
+        CONDA_NPY: '1.18'
       Python39:
         CONDA_PY: '3.9'
         CONDA_NPY: '1.19'

devtools/azure-pipelines-win.yml

@@ -7,10 +7,10 @@ jobs:
     matrix:
       Python37:
         CONDA_PY: '3.7'
-        CONDA_NPY: '1.19'
+        CONDA_NPY: '1.18'
       Python38:
         CONDA_PY: '3.8'
-        CONDA_NPY: '1.19'
+        CONDA_NPY: '1.18'
       Python39:
         CONDA_PY: '3.9'
         CONDA_NPY: '1.19'

devtools/conda-recipe/meta.yaml

@@ -35,7 +35,8 @@ requirements:
     - setuptools
     - pip
     - pybind11
-    - deeptime
+    - pybind11-abi
+    - deeptime >=0.3.0
 
   run:
     - bhmm >=0.6.3

pyemma/_base/serialization/tests/test_cli.py

@@ -15,46 +15,44 @@
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+import shutil
 import tempfile
-import unittest
 
+import pytest
+from numpy.testing import assert_
 
 from pyemma._base.serialization.cli import main
 from pyemma.coordinates import source, tica, cluster_kmeans
 
 
-class TestListModelCLI(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
+@pytest.fixture
+def model_file():
+    file = None
+    try:
         from pyemma.datasets import get_bpti_test_data
-
         d = get_bpti_test_data()
         trajs, top = d['trajs'], d['top']
         s = source(trajs, top=top)
 
         t = tica(s, lag=1)
 
         c = cluster_kmeans(t)
-        cls.model_file = tempfile.mktemp()
-        c.save(cls.model_file, save_streaming_chain=True)
-
-    @classmethod
-    def tearDownClass(cls):
-        import os
-        os.unlink(cls.model_file)
-
-    def test_recursive(self):
-        """ check the whole chain has been printed"""
-        from pyemma.util.contexts import Capturing
-        with Capturing() as out:
-            main(['--recursive', self.model_file])
-        assert out
-        all_out = '\n'.join(out)
-        self.assertIn('FeatureReader', all_out)
-        self.assertIn('TICA', all_out)
-        self.assertIn('Kmeans', all_out)
-
-
-if __name__ == '__main__':
-    unittest.main()
+        file = tempfile.mktemp()
+        c.save(file, save_streaming_chain=True)
+
+        yield file
+    finally:
+        if file is not None:
+            shutil.rmtree(file, ignore_errors=True)
+
+
+def test_recursive(model_file):
+    """ check the whole chain has been printed"""
+    from pyemma.util.contexts import Capturing
+    with Capturing() as out:
+        main(['--recursive', model_file])
+    assert out
+    all_out = '\n'.join(out)
+    assert_('FeatureReader' in all_out)
+    assert_('TICA' in all_out)
+    assert_('Kmeans' in all_out)

pyemma/coordinates/api.py

@@ -21,6 +21,8 @@
 
 .. currentmodule:: pyemma.coordinates.api
 """
+from pathlib import Path
+
 import numpy as _np
 import logging as _logging
 
@@ -632,7 +634,7 @@ def discretizer(reader,
     return disc
 
 
-def save_traj(traj_inp, indexes, outfile, top=None, stride = 1, chunksize=None, image_molecules=False, verbose=True):
+def save_traj(traj_inp, indexes, outfile, top=None, stride=1, chunksize=None, image_molecules=False, verbose=True):
     r""" Saves a sequence of frames as a single trajectory.
 
     Extracts the specified sequence of time/trajectory indexes from traj_inp
@@ -753,6 +755,8 @@ def save_traj(traj_inp, indexes, outfile, top=None, stride = 1, chunksize=None,
         return traj
     # or to disk as a molecular trajectory file
     else:
+        if isinstance(outfile, Path):
+            outfile = str(outfile.resolve())
         traj.save(outfile)
     if verbose:
         _logger.info("Created file %s" % outfile)
@@ -839,9 +843,12 @@ def save_trajs(traj_inp, indexes, prefix='set_', fmt=None, outfiles=None,
 
     # Determine output format of the molecular trajectory file
     if fmt is None:
+        fname = traj_inp.filenames[0]
+        while hasattr(fname, '__getitem__') and not isinstance(fname, (str, bytes)):
+            fname = fname[0]
         import os
 
-        _, fmt = os.path.splitext(traj_inp.filenames[0])
+        _, fmt = os.path.splitext(fname)
     else:
         fmt = '.' + fmt