From 97082166d010815211c39aa8033d95a57c9844be Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Tue, 27 Jun 2017 17:56:56 +1000 Subject: [PATCH 01/12] Comment existing SSHDataGrabber._list_outputs() to make interpretable --- nipype/interfaces/io.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index c4fdd75216..873735b591 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -2439,9 +2439,13 @@ def _list_outputs(self): isdefined(self.inputs.field_template) and \ key in self.inputs.field_template: template = self.inputs.field_template[key] + if not args: + # Connect over SSH client = self._get_ssh_client() sftp = client.open_sftp() + + # Get the files in the base dir, and filter for desired files sftp.chdir(self.inputs.base_directory) filelist = sftp.listdir() if self.inputs.template_expression == 'fnmatch': @@ -2451,7 +2455,9 @@ def _list_outputs(self): filelist = list(filter(regexp.match, filelist)) else: raise ValueError('template_expression value invalid') + if len(filelist) == 0: + # no files msg = 'Output key: %s Template: %s returned no files' % ( key, template) if self.inputs.raise_on_empty: @@ -2459,12 +2465,16 @@ def _list_outputs(self): else: warn(msg) else: + # found files, sort and save to outputs if self.inputs.sort_filelist: filelist = human_order_sorted(filelist) outputs[key] = list_to_filename(filelist) + + # actually download the files, if desired if self.inputs.download_files: for f in filelist: sftp.get(f, f) + for argnum, arglist in enumerate(args): maxlen = 1 for arg in arglist: @@ -2498,9 +2508,13 @@ def _list_outputs(self): e.message + ": Template %s failed to convert with args %s" % (template, str(tuple(argtuple)))) + + # Connect over SSH client = self._get_ssh_client() sftp = client.open_sftp() sftp.chdir(self.inputs.base_directory) + + # Get all files in the dir, and filter for desired files filledtemplate_dir = os.path.dirname(filledtemplate) filledtemplate_base = os.path.basename(filledtemplate) filelist = sftp.listdir(filledtemplate_dir) @@ -2512,18 +2526,24 @@ def _list_outputs(self): outfiles = list(filter(regexp.match, filelist)) else: raise ValueError('template_expression value invalid') + if len(outfiles) == 0: msg = 'Output key: %s Template: %s returned no files' % ( key, filledtemplate) + + # no files if self.inputs.raise_on_empty: raise IOError(msg) else: warn(msg) outputs[key].append(None) else: + # found files, sort and save to outputs if self.inputs.sort_filelist: outfiles = human_order_sorted(outfiles) outputs[key].append(list_to_filename(outfiles)) + + # actually download the files, if desired if self.inputs.download_files: for f in outfiles: try: @@ -2532,10 +2552,16 @@ def _list_outputs(self): except IOError: iflogger.info('remote file %s not found', f) + + # disclude where there was any invalid matches if any([val is None for val in outputs[key]]): outputs[key] = [] + + # no outputs is None, not empty list if len(outputs[key]) == 0: outputs[key] = None + + # one output is the item, not a list elif len(outputs[key]) == 1: outputs[key] = outputs[key][0] From 0259a2b0347b7999e2d8ae5dae7cc49d8137d0f7 Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Wed, 28 Jun 2017 09:33:41 +1000 Subject: [PATCH 02/12] TST: Add a test for SSH function --- nipype/interfaces/tests/test_io.py | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 6aafb5b6b9..440e89a749 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -5,6 +5,7 @@ from builtins import str, zip, range, open from future import standard_library import os +import copy import simplejson import glob import shutil @@ -37,6 +38,12 @@ except ImportError: noboto3 = True +try: + import paramiko + no_paramiko = False +except ImportError: + no_paramiko = True + # Check for fakes3 standard_library.install_aliases() from subprocess import check_call, CalledProcessError @@ -611,3 +618,45 @@ def test_bids_infields_outfields(tmpdir): bg = nio.BIDSDataGrabber() for outfield in ['anat', 'func']: assert outfield in bg._outputs().traits() + + +@pytest.mark.skipif(no_paramiko, reason="paramiko library is not available") +def test_SSHDataGrabber(tmpdir): + """Test SSHDataGrabber by connecting to localhost and finding this test + file. + """ + old_cwd = tmpdir.chdir() + + # ssh client that connects to localhost, current user, regardless of + # ~/.ssh/config + def _mock_get_ssh_client(self): + proxy = None + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect('localhost', username=os.getenv('USER'), sock=proxy) + return client + MockSSHDataGrabber = copy.copy(nio.SSHDataGrabber) + MockSSHDataGrabber._get_ssh_client = _mock_get_ssh_client + + this_dir = os.path.dirname(__file__) + this_file = os.path.basename(__file__) + this_test = this_file[:-3] # without .py + + ssh_grabber = MockSSHDataGrabber(infields=['test'], + outfields=['test_file']) + # ssh_grabber.base_dir = str(tmpdir) + ssh_grabber.inputs.base_directory = this_dir + ssh_grabber.inputs.hostname = 'localhost' + ssh_grabber.inputs.field_template = dict(test_file='%s.py') + ssh_grabber.inputs.template = '' + ssh_grabber.inputs.template_args = dict(test_file=[['test']]) + ssh_grabber.inputs.test = this_test + ssh_grabber.inputs.sort_filelist = True + + runtime = ssh_grabber.run() + + # did we successfully get this file? + assert runtime.outputs.test_file == str(tmpdir.join(this_file)) + + old_cwd.chdir() From 97c4ed5f93886910faa853a7ff51799e0008b4c5 Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Wed, 28 Jun 2017 09:36:13 +1000 Subject: [PATCH 03/12] REF: Repeated code for fetching SSH files into one function --- nipype/interfaces/io.py | 127 ++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 76 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 873735b591..168cce050c 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -2412,6 +2412,55 @@ def __init__(self, infields=None, outfields=None, **kwargs): and self.inputs.template[-1] != '$'): self.inputs.template += '$' + def _get_files_over_ssh(self, template): + """Get the files matching template over an SSH connection.""" + # Connect over SSH + client = self._get_ssh_client() + sftp = client.open_sftp() + sftp.chdir(self.inputs.base_directory) + + # Get all files in the dir, and filter for desired files + template_dir = os.path.dirname(template) + template_base = os.path.basename(template) + filelist = sftp.listdir(template_dir) + if self.inputs.template_expression == 'fnmatch': + outfiles = fnmatch.filter(filelist, template_base) + elif self.inputs.template_expression == 'regexp': + regexp = re.compile(template_base) + outfiles = list(filter(regexp.match, filelist)) + else: + raise ValueError('template_expression value invalid') + + if len(outfiles) == 0: + # no files + msg = 'Output key: %s Template: %s returned no files' % ( + key, template) + if self.inputs.raise_on_empty: + raise IOError(msg) + else: + warn(msg) + + # return value + outfiles = None + + else: + # found files, sort and save to outputs + if self.inputs.sort_filelist: + outfiles = human_order_sorted(outfiles) + + # actually download the files, if desired + if self.inputs.download_files: + for f in outfiles: + try: + sftp.get(os.path.join(template_dir, f), f) + except IOError: + iflogger.info('remote file %s not found' % f) + + # return value + outfiles = list_to_filename(outfiles) + + return outfiles + def _list_outputs(self): try: paramiko @@ -2441,39 +2490,7 @@ def _list_outputs(self): template = self.inputs.field_template[key] if not args: - # Connect over SSH - client = self._get_ssh_client() - sftp = client.open_sftp() - - # Get the files in the base dir, and filter for desired files - sftp.chdir(self.inputs.base_directory) - filelist = sftp.listdir() - if self.inputs.template_expression == 'fnmatch': - filelist = fnmatch.filter(filelist, template) - elif self.inputs.template_expression == 'regexp': - regexp = re.compile(template) - filelist = list(filter(regexp.match, filelist)) - else: - raise ValueError('template_expression value invalid') - - if len(filelist) == 0: - # no files - msg = 'Output key: %s Template: %s returned no files' % ( - key, template) - if self.inputs.raise_on_empty: - raise IOError(msg) - else: - warn(msg) - else: - # found files, sort and save to outputs - if self.inputs.sort_filelist: - filelist = human_order_sorted(filelist) - outputs[key] = list_to_filename(filelist) - - # actually download the files, if desired - if self.inputs.download_files: - for f in filelist: - sftp.get(f, f) + outputs[key] = self._get_files_over_ssh(template) for argnum, arglist in enumerate(args): maxlen = 1 @@ -2509,49 +2526,7 @@ def _list_outputs(self): ": Template %s failed to convert with args %s" % (template, str(tuple(argtuple)))) - # Connect over SSH - client = self._get_ssh_client() - sftp = client.open_sftp() - sftp.chdir(self.inputs.base_directory) - - # Get all files in the dir, and filter for desired files - filledtemplate_dir = os.path.dirname(filledtemplate) - filledtemplate_base = os.path.basename(filledtemplate) - filelist = sftp.listdir(filledtemplate_dir) - if self.inputs.template_expression == 'fnmatch': - outfiles = fnmatch.filter(filelist, - filledtemplate_base) - elif self.inputs.template_expression == 'regexp': - regexp = re.compile(filledtemplate_base) - outfiles = list(filter(regexp.match, filelist)) - else: - raise ValueError('template_expression value invalid') - - if len(outfiles) == 0: - msg = 'Output key: %s Template: %s returned no files' % ( - key, filledtemplate) - - # no files - if self.inputs.raise_on_empty: - raise IOError(msg) - else: - warn(msg) - outputs[key].append(None) - else: - # found files, sort and save to outputs - if self.inputs.sort_filelist: - outfiles = human_order_sorted(outfiles) - outputs[key].append(list_to_filename(outfiles)) - - # actually download the files, if desired - if self.inputs.download_files: - for f in outfiles: - try: - sftp.get( - os.path.join(filledtemplate_dir, f), f) - except IOError: - iflogger.info('remote file %s not found', - f) + outputs[key].append(self._get_files_over_ssh(filledtemplate)) # disclude where there was any invalid matches if any([val is None for val in outputs[key]]): From b8d244605a7ed721000e46c34161c51ab3a27cb5 Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Wed, 28 Jun 2017 10:48:44 +1000 Subject: [PATCH 04/12] TST: Failing test to demonstrate SSH doesn't copy related files --- nipype/interfaces/tests/test_io.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 440e89a749..100669d633 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -622,11 +622,16 @@ def test_bids_infields_outfields(tmpdir): @pytest.mark.skipif(no_paramiko, reason="paramiko library is not available") def test_SSHDataGrabber(tmpdir): - """Test SSHDataGrabber by connecting to localhost and finding this test - file. + """Test SSHDataGrabber by connecting to localhost and collecting some data. """ old_cwd = tmpdir.chdir() + source_dir = tmpdir.mkdir('source') + source_hdr = source_dir.join('somedata.hdr') + source_dat = source_dir.join('somedata.img') + source_hdr.ensure() # create + source_dat.ensure() # create + # ssh client that connects to localhost, current user, regardless of # ~/.ssh/config def _mock_get_ssh_client(self): @@ -639,24 +644,24 @@ def _mock_get_ssh_client(self): MockSSHDataGrabber = copy.copy(nio.SSHDataGrabber) MockSSHDataGrabber._get_ssh_client = _mock_get_ssh_client - this_dir = os.path.dirname(__file__) - this_file = os.path.basename(__file__) - this_test = this_file[:-3] # without .py - + # grabber to get files from source_dir matching test.hdr ssh_grabber = MockSSHDataGrabber(infields=['test'], outfields=['test_file']) - # ssh_grabber.base_dir = str(tmpdir) - ssh_grabber.inputs.base_directory = this_dir + ssh_grabber.inputs.base_directory = str(source_dir) ssh_grabber.inputs.hostname = 'localhost' - ssh_grabber.inputs.field_template = dict(test_file='%s.py') + ssh_grabber.inputs.field_template = dict(test_file='%s.hdr') ssh_grabber.inputs.template = '' ssh_grabber.inputs.template_args = dict(test_file=[['test']]) - ssh_grabber.inputs.test = this_test + ssh_grabber.inputs.test = 'somedata' ssh_grabber.inputs.sort_filelist = True runtime = ssh_grabber.run() - # did we successfully get this file? - assert runtime.outputs.test_file == str(tmpdir.join(this_file)) + # did we successfully get the header? + assert runtime.outputs.test_file == str(tmpdir.join(source_hdr.basename)) + # did we successfully get the data? + assert (tmpdir.join(source_hdr.basename) # header file + .new(ext='.img') # data file + .check(file=True, exists=True)) # exists? old_cwd.chdir() From 94920220c0d86c6aee6a89065cb6399fa3789baf Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Wed, 28 Jun 2017 10:49:28 +1000 Subject: [PATCH 05/12] ENH: SSHDataGrabber also downloads related files --- nipype/interfaces/io.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 168cce050c..8cae09c1a3 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -31,6 +31,7 @@ import shutil import subprocess import re +import copy import tempfile from os.path import join, dirname from warnings import warn @@ -38,7 +39,9 @@ import sqlite3 from .. import config, logging -from ..utils.filemanip import copyfile, list_to_filename, filename_to_list +from ..utils.filemanip import ( + copyfile, list_to_filename, filename_to_list, + get_related_files, related_filetype_sets) from ..utils.misc import human_order_sorted, str2bool from .base import ( TraitedSpec, traits, Str, File, Directory, BaseInterface, InputMultiPath, @@ -2422,12 +2425,12 @@ def _get_files_over_ssh(self, template): # Get all files in the dir, and filter for desired files template_dir = os.path.dirname(template) template_base = os.path.basename(template) - filelist = sftp.listdir(template_dir) + every_file_in_dir = sftp.listdir(template_dir) if self.inputs.template_expression == 'fnmatch': - outfiles = fnmatch.filter(filelist, template_base) + outfiles = fnmatch.filter(every_file_in_dir, template_base) elif self.inputs.template_expression == 'regexp': regexp = re.compile(template_base) - outfiles = list(filter(regexp.match, filelist)) + outfiles = list(filter(regexp.match, every_file_in_dir)) else: raise ValueError('template_expression value invalid') @@ -2450,7 +2453,18 @@ def _get_files_over_ssh(self, template): # actually download the files, if desired if self.inputs.download_files: - for f in outfiles: + files_to_download = copy.copy(outfiles) # make sure new list! + + # check to see if there are any related files to download + for file_to_download in files_to_download: + related_to_current = get_related_files( + file_to_download, include_this_file=False) + existing_related_not_downloading = [ + f for f in related_to_current + if f in every_file_in_dir and f not in files_to_download] + files_to_download.extend(existing_related_not_downloading) + + for f in files_to_download: try: sftp.get(os.path.join(template_dir, f), f) except IOError: From b991c34e2ee0ccd409c831afc9dc4ffbc6a78ab2 Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Thu, 29 Jun 2017 15:47:08 +1000 Subject: [PATCH 06/12] TST: Depend paramiko so SSHDataGrabber tests run --- .travis.yml | 2 +- Dockerfile | 109 +++++++++++++++++++++++++++++++++++++++++++++++++ nipype/info.py | 3 +- 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 Dockerfile diff --git a/.travis.yml b/.travis.yml index d57a4205d2..fa5c199a8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ python: env: - INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1 - INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1 -- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" CI_SKIP_TEST=1 +- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit,ssh" CI_SKIP_TEST=1 - INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" PIP_FLAGS="--pre" CI_SKIP_TEST=1 addons: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..22d49813b2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,109 @@ +# Copyright (c) 2016, The developers of the Stanford CRN +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of crn_base nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# +# Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile +# +FROM nipype/base:latest +MAINTAINER The nipype developers https://github.com/nipy/nipype + +ARG PYTHON_VERSION_MAJOR=3 + +# Installing and setting up miniconda +RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh && \ + bash Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \ + rm Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh + +ENV PATH=/usr/local/miniconda/bin:$PATH \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + ACCEPT_INTEL_PYTHON_EULA=yes \ + MKL_NUM_THREADS=1 \ + OMP_NUM_THREADS=1 +# MKL/OMP_NUM_THREADS: unless otherwise specified, each process should +# only use one thread - nipype will handle parallelization + +# Installing precomputed python packages +ARG PYTHON_VERSION_MINOR=6 +RUN conda config --add channels conda-forge; sync && \ + conda config --set always_yes yes --set changeps1 no; sync && \ + conda install -y python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \ + mkl \ + numpy \ + scipy \ + scikit-learn \ + matplotlib \ + pandas \ + libxml2 \ + libxslt \ + traits=4.6.0 \ + psutil \ + paramiko \ + icu=58.1 && \ + sync; + +# matplotlib cleanups: set default backend, precaching fonts +RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/matplotlib/mpl-data/matplotlibrc && \ + python -c "from matplotlib import font_manager" + +# Install CI scripts +COPY docker/files/run_* /usr/bin/ +RUN chmod +x /usr/bin/run_* + +# Replace imglob with a Python3 compatible version +COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py +RUN rm -rf ${FSLDIR}/bin/imglob && \ + chmod +x /usr/bin/fsl_imglob.py && \ + ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob + +# Installing dev requirements (packages that are not in pypi) +WORKDIR /src/ +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt && \ + rm -rf ~/.cache/pip + +# Installing nipype +COPY . /src/nipype +RUN cd /src/nipype && \ + pip install -e .[all] && \ + rm -rf ~/.cache/pip + +WORKDIR /work/ + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="NIPYPE" \ + org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ + org.label-schema.url="http://nipype.readthedocs.io" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url="https://github.com/nipy/nipype" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" diff --git a/nipype/info.py b/nipype/info.py index 8a7e3b4348..93b5dd2eb8 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -163,7 +163,8 @@ def get_nipype_gitversion(): 'profiler': ['psutil>=5.0'], 'duecredit': ['duecredit'], 'xvfbwrapper': ['xvfbwrapper'], - 'pybids': ['pybids'] + 'pybids': ['pybids'], + 'ssh': ['paramiko'], # 'mesh': ['mayavi'] # Enable when it works } From d08a4b17760435082d83689f7a0640f7b90ff843 Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Wed, 16 Aug 2017 15:35:41 +1000 Subject: [PATCH 07/12] FIX: Missing variable, key --- nipype/interfaces/io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 8cae09c1a3..ee8b1a28c7 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -2436,8 +2436,7 @@ def _get_files_over_ssh(self, template): if len(outfiles) == 0: # no files - msg = 'Output key: %s Template: %s returned no files' % ( - key, template) + msg = 'Output template: %s returned no files' % template if self.inputs.raise_on_empty: raise IOError(msg) else: From 0a1f741bbb9bf03b05116ca17f099a6b26edab89 Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Thu, 22 Feb 2018 09:38:27 +1000 Subject: [PATCH 08/12] FIX: Only run SSHDataGrabber tests in local SSH server is running. --- nipype/interfaces/tests/test_io.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 100669d633..298f9904c6 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -38,9 +38,26 @@ except ImportError: noboto3 = True +# Check for paramiko try: import paramiko no_paramiko = False + + # Check for localhost SSH Server + # FIXME: Tests requiring this are never run on CI + try: + proxy = None + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect('localhost', username=os.getenv('USER'), sock=proxy) + + no_local_ssh = False + + except (paramiko.SSHException, + paramiko.ssh_exception.NoValidConnectionsError): + no_local_ssh = True + except ImportError: no_paramiko = True @@ -621,6 +638,7 @@ def test_bids_infields_outfields(tmpdir): @pytest.mark.skipif(no_paramiko, reason="paramiko library is not available") +@pytest.mark.skipif(no_local_ssh, reason="SSH Server is not running") def test_SSHDataGrabber(tmpdir): """Test SSHDataGrabber by connecting to localhost and collecting some data. """ From 715416f57464528ac259c3e021ef9d23d4e27b0c Mon Sep 17 00:00:00 2001 From: Ashley Gillman Date: Fri, 2 Mar 2018 13:46:32 +1000 Subject: [PATCH 09/12] FIX: Forgot to set no_local_ssh in one tree; CircleCI throws OSError. --- nipype/interfaces/tests/test_io.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 298f9904c6..394a05b00f 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -55,11 +55,13 @@ no_local_ssh = False except (paramiko.SSHException, - paramiko.ssh_exception.NoValidConnectionsError): + paramiko.ssh_exception.NoValidConnectionsError, + OSError): no_local_ssh = True except ImportError: no_paramiko = True + no_local_ssh = True # Check for fakes3 standard_library.install_aliases() From ae6147511b232b398deb6657ace022139be56483 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 1 Mar 2018 23:05:27 -0500 Subject: [PATCH 10/12] DOCKER: Add paramiko to NeuroDocker script --- Dockerfile | 109 --------------------------------- docker/generate_dockerfiles.sh | 2 +- 2 files changed, 1 insertion(+), 110 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 22d49813b2..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2016, The developers of the Stanford CRN -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of crn_base nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -# -# Based on https://github.com/poldracklab/fmriprep/blob/9c92a3de9112f8ef1655b876de060a2ad336ffb0/Dockerfile -# -FROM nipype/base:latest -MAINTAINER The nipype developers https://github.com/nipy/nipype - -ARG PYTHON_VERSION_MAJOR=3 - -# Installing and setting up miniconda -RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh && \ - bash Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \ - rm Miniconda${PYTHON_VERSION_MAJOR}-4.2.12-Linux-x86_64.sh - -ENV PATH=/usr/local/miniconda/bin:$PATH \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - ACCEPT_INTEL_PYTHON_EULA=yes \ - MKL_NUM_THREADS=1 \ - OMP_NUM_THREADS=1 -# MKL/OMP_NUM_THREADS: unless otherwise specified, each process should -# only use one thread - nipype will handle parallelization - -# Installing precomputed python packages -ARG PYTHON_VERSION_MINOR=6 -RUN conda config --add channels conda-forge; sync && \ - conda config --set always_yes yes --set changeps1 no; sync && \ - conda install -y python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \ - mkl \ - numpy \ - scipy \ - scikit-learn \ - matplotlib \ - pandas \ - libxml2 \ - libxslt \ - traits=4.6.0 \ - psutil \ - paramiko \ - icu=58.1 && \ - sync; - -# matplotlib cleanups: set default backend, precaching fonts -RUN sed -i 's/\(backend *: \).*$/\1Agg/g' /usr/local/miniconda/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/matplotlib/mpl-data/matplotlibrc && \ - python -c "from matplotlib import font_manager" - -# Install CI scripts -COPY docker/files/run_* /usr/bin/ -RUN chmod +x /usr/bin/run_* - -# Replace imglob with a Python3 compatible version -COPY nipype/external/fsl_imglob.py /usr/bin/fsl_imglob.py -RUN rm -rf ${FSLDIR}/bin/imglob && \ - chmod +x /usr/bin/fsl_imglob.py && \ - ln -s /usr/bin/fsl_imglob.py ${FSLDIR}/bin/imglob - -# Installing dev requirements (packages that are not in pypi) -WORKDIR /src/ -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt && \ - rm -rf ~/.cache/pip - -# Installing nipype -COPY . /src/nipype -RUN cd /src/nipype && \ - pip install -e .[all] && \ - rm -rf ~/.cache/pip - -WORKDIR /work/ - -ARG BUILD_DATE -ARG VCS_REF -ARG VERSION -LABEL org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.name="NIPYPE" \ - org.label-schema.description="NIPYPE - Neuroimaging in Python: Pipelines and Interfaces" \ - org.label-schema.url="http://nipype.readthedocs.io" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/nipy/nipype" \ - org.label-schema.version=$VERSION \ - org.label-schema.schema-version="1.0" diff --git a/docker/generate_dockerfiles.sh b/docker/generate_dockerfiles.sh index 2f0ed5eaa7..e1f21b3139 100755 --- a/docker/generate_dockerfiles.sh +++ b/docker/generate_dockerfiles.sh @@ -103,7 +103,7 @@ function generate_main_dockerfile() { --arg PYTHON_VERSION_MAJOR=3 PYTHON_VERSION_MINOR=6 BUILD_DATE VCS_REF VERSION \ --miniconda env_name=neuro \ conda_install='python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} - icu=58.1 libxml2 libxslt matplotlib mkl numpy + icu=58.1 libxml2 libxslt matplotlib mkl numpy paramiko pandas psutil scikit-learn scipy traits=4.6.0' \ pip_opts="-e" \ pip_install="/src/nipype[all]" \ From 037fb196273cda929b425f9bab0094b670ed2194 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 2 Mar 2018 07:56:16 -0500 Subject: [PATCH 11/12] TEST: Use 127.0.0.1 over localhost --- nipype/interfaces/tests/test_io.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 394a05b00f..8ec83aad5c 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -50,7 +50,7 @@ client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect('localhost', username=os.getenv('USER'), sock=proxy) + client.connect('127.0.0.1', username=os.getenv('USER'), sock=proxy) no_local_ssh = False @@ -342,7 +342,7 @@ def test_datasink_to_s3(dummy_input, tmpdir): aws_access_key_id='mykey', aws_secret_access_key='mysecret', service_name='s3', - endpoint_url='http://localhost:4567', + endpoint_url='http://127.0.0.1:4567', use_ssl=False) resource.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host) @@ -659,7 +659,7 @@ def _mock_get_ssh_client(self): client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect('localhost', username=os.getenv('USER'), sock=proxy) + client.connect('127.0.0.1', username=os.getenv('USER'), sock=proxy) return client MockSSHDataGrabber = copy.copy(nio.SSHDataGrabber) MockSSHDataGrabber._get_ssh_client = _mock_get_ssh_client @@ -668,7 +668,7 @@ def _mock_get_ssh_client(self): ssh_grabber = MockSSHDataGrabber(infields=['test'], outfields=['test_file']) ssh_grabber.inputs.base_directory = str(source_dir) - ssh_grabber.inputs.hostname = 'localhost' + ssh_grabber.inputs.hostname = '127.0.0.1' ssh_grabber.inputs.field_template = dict(test_file='%s.hdr') ssh_grabber.inputs.template = '' ssh_grabber.inputs.template_args = dict(test_file=[['test']]) From 7b5d66798f6cb7b629d78f258139a1ba1a8cc9b3 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 2 Mar 2018 07:58:40 -0500 Subject: [PATCH 12/12] TEST: Add timeout to test SSH connections --- nipype/interfaces/tests/test_io.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 8ec83aad5c..1eea2b31f3 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -50,7 +50,8 @@ client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect('127.0.0.1', username=os.getenv('USER'), sock=proxy) + client.connect('127.0.0.1', username=os.getenv('USER'), sock=proxy, + timeout=10) no_local_ssh = False @@ -659,7 +660,8 @@ def _mock_get_ssh_client(self): client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect('127.0.0.1', username=os.getenv('USER'), sock=proxy) + client.connect('127.0.0.1', username=os.getenv('USER'), sock=proxy, + timeout=10) return client MockSSHDataGrabber = copy.copy(nio.SSHDataGrabber) MockSSHDataGrabber._get_ssh_client = _mock_get_ssh_client