Merge branch 'master' of github.com:nipy/heudiconv into feature/bids_filelock

Author: stilley2

.travis.yml

@@ -2,9 +2,9 @@
 language: python
 python:
   - 2.7
-  - 3.4
   - 3.5
   - 3.6
+  - 3.7
 
 cache:
   - apt

docs/usage.rst

@@ -43,6 +43,13 @@ DICOMs as an independent ``heudiconv`` execution.
 The first script aggregates the DICOM directories and submits them to
 ``run_heudiconv.sh`` with SLURM as a job array.
 
+If using bids, the ``notop`` bids option suppresses creation of
+top-level files in the bids directory (e.g.,
+``dataset_description.json``) to avoid possible race conditions.
+These files may be generated later with ``populate_templates.sh``
+below (except for ``participants.tsv``, which must be create
+manually).
+
 .. code:: shell
 
     #!/bin/bash
@@ -76,7 +83,22 @@ The second script processes a DICOM directory with ``heudiconv`` using the built
     echo Submitted directory: ${DCMDIR}
 
     IMG="/singularity-images/heudiconv-0.5.4-dev.sif"
-    CMD="singularity run -B ${DCMDIR}:/dicoms:ro -B ${OUTDIR}:/output -e ${IMG} --files /dicoms/ -o /output -f reproin -c dcm2niix -b --minmeta -l ."
+    CMD="singularity run -B ${DCMDIR}:/dicoms:ro -B ${OUTDIR}:/output -e ${IMG} --files /dicoms/ -o /output -f reproin -c dcm2niix -b notop --minmeta -l ."
+
+    printf "Command:\n${CMD}\n"
+    ${CMD}
+    echo "Successful process"
+
+This script creates the top-level bids files (e.g.,
+``dataset_description.json``)
+
+..code:: shell
+    #!/bin/bash
+    set -eu
+
+    OUTDIR=${1}
+    IMG="/singularity-images/heudiconv-0.5.4-dev.sif"
+    CMD="singularity run -B ${OUTDIR}:/output -e ${IMG} --files /output -f reproin --command populate-templates"
 
     printf "Command:\n${CMD}\n"
     ${CMD}

heudiconv/bids.py

@@ -240,6 +240,27 @@ def add_participant_record(studydir, subject, age, sex):
             known_subjects = {l.split('\t')[0] for l in f.readlines()}
         if participant_id in known_subjects:
             return
+    else:
+        # Populate particpants.json (an optional file to describe column names in
+        # participant.tsv). This auto generation will make BIDS-validator happy.
+        participants_json = op.join(studydir, 'participants.json')
+        if not op.lexists(participants_json):
+            save_json(participants_json,
+                OrderedDict([
+                    ("participant_id", OrderedDict([
+                        ("Description", "Participant identifier")])),
+                    ("age", OrderedDict([
+                        ("Description", "Age in years (TODO - verify) as in the initial"
+                            " session, might not be correct for other sessions")])),
+                    ("sex", OrderedDict([
+                        ("Description", "self-rated by participant, M for male/F for "
+                            "female (TODO: verify)")])),
+                    ("group", OrderedDict([
+                        ("Description", "(TODO: adjust - by default everyone is in "
+                            "control group)")])),
+                ]),
+                sort_keys=False,
+                indent=2)
     # Add a new participant
     with open(participants_tsv, 'a') as f:
         f.write(
@@ -311,7 +332,8 @@ def save_scans_key(item, bids_files):
 
 def add_rows_to_scans_keys_file(fn, newrows):
     """
-    Add new rows to file fn for scans key filename
+    Add new rows to file fn for scans key filename and generate accompanying json
+    descriptor to make BIDS validator happy.
 
     Parameters
     ----------
@@ -334,6 +356,25 @@ def add_rows_to_scans_keys_file(fn, newrows):
         os.unlink(fn)
     else:
         fnames2info = newrows
+        # Populate _scans.json (an optional file to describe column names in
+        # _scans.tsv). This auto generation will make BIDS-validator happy.
+        scans_json = '.'.join(fn.split('.')[:-1] + ['json'])
+        if not op.lexists(scans_json):
+            save_json(scans_json,
+                OrderedDict([
+                    ("filename", OrderedDict([
+                        ("Description", "Name of the nifti file")])),
+                    ("acq_time", OrderedDict([
+                        ("LongName", "Acquisition time"),
+                        ("Description", "Acquisition time of the particular scan")])),
+                    ("operator", OrderedDict([
+                        ("Description", "Name of the operator")])),
+                    ("randstr", OrderedDict([
+                        ("LongName", "Random string"),
+                        ("Description", "md5 hash of UIDs")])),
+                ]),
+                sort_keys=False,
+                indent=2)
 
     header = ['filename', 'acq_time', 'operator', 'randstr']
     # prepare all the data rows
@@ -370,7 +411,7 @@ def get_formatted_scans_key_row(dcm_fn):
         time = dcm_data.ContentTime.split('.')[0]
         td = time + date
         acq_time = datetime.strptime(td, '%H%M%S%Y%m%d').isoformat()
-    except AttributeError as exc:
+    except (AttributeError, ValueError) as exc:
         lgr.warning("Failed to get date/time for the content: %s", str(exc))
         acq_time = None
     # add random string

heudiconv/cli/run.py

@@ -16,7 +16,7 @@
 import logging
 lgr = logging.getLogger(__name__)
 
-INIT_MSG = "Running {packname} version {version}".format
+INIT_MSG = "Running {packname} version {version} latest {latest}".format
 
 
 def is_interactive():
@@ -112,6 +112,7 @@ def main(argv=None):
         random.seed(args.random_seed)
         import numpy
         numpy.random.seed(args.random_seed)
+    # Ensure only supported bids options are passed
     if args.debug:
         lgr.setLevel(logging.DEBUG)
     # Should be possible but only with a single subject -- will be used to
@@ -129,7 +130,7 @@ def main(argv=None):
 
 def get_parser():
     docstr = ("""Example:
-             heudiconv -d rawdata/{subject} -o . -f heuristic.py -s s1 s2 s3""")
+             heudiconv -d 'rawdata/{subject}' -o . -f heuristic.py -s s1 s2 s3""")
     parser = ArgumentParser(description=docstr)
     parser.add_argument('--version', action='version', version=__version__)
     group = parser.add_mutually_exclusive_group()
@@ -138,7 +139,12 @@ def get_parser():
                        'subject id {subject} and session {session}. Tarballs '
                        '(can be compressed) are supported in addition to '
                        'directory. All matching tarballs for a subject are '
-                       'extracted and their content processed in a single pass')
+                       'extracted and their content processed in a single '
+                       'pass. If multiple tarballs are found, each is '
+                       'assumed to be a separate session and the --ses '
+                       'argument is ignored. Note that you might need to '
+                       'surround the value with quotes to avoid {...} being '
+                       'considered by shell')
     group.add_argument('--files', nargs='*',
                        help='Files (tarballs, dicoms) or directories '
                        'containing files to process. Cannot be provided if '
@@ -181,8 +187,16 @@ def get_parser():
     parser.add_argument('-ss', '--ses', dest='session', default=None,
                         help='session for longitudinal study_sessions, default '
                         'is none')
-    parser.add_argument('-b', '--bids', action='store_true',
-                        help='flag for output into BIDS structure')
+    parser.add_argument('-b', '--bids', nargs='*',
+                        metavar=('BIDSOPTION1', 'BIDSOPTION2'),
+                        choices=['notop'],
+                        dest='bids_options',
+                        help='flag for output into BIDS structure. Can also '
+                        'take bids specific options, e.g., --bids notop.'
+                        'The only currently supported options is'
+                        '"notop", which skips creation of top-level bids '
+                        'files. This is useful when running in batch mode to '
+                        'prevent possible race conditions.')
     parser.add_argument('--overwrite', action='store_true', default=False,
                         help='flag to allow overwriting existing converted files')
     parser.add_argument('--datalad', action='store_true',
@@ -234,8 +248,16 @@ def process_args(args):
 
     outdir = op.abspath(args.outdir)
 
+    import etelemetry
+    try:
+        latest = etelemetry.get_project("nipy/heudiconv")
+    except Exception as e:
+        lgr.warning("Could not check for version updates: ", e)
+        latest = {"version": 'Unknown'}
+
     lgr.info(INIT_MSG(packname=__packagename__,
-                      version=__version__))
+                      version=__version__,
+                      latest=latest["version"]))
 
     if args.command:
         process_extra_commands(outdir, args)
@@ -302,7 +324,8 @@ def process_args(args):
             from ..external.dlad import prepare_datalad
             dlad_sid = sid if not anon_sid else anon_sid
             dl_msg = prepare_datalad(anon_study_outdir, anon_outdir, dlad_sid,
-                                     session, seqinfo, dicoms, args.bids)
+                                     session, seqinfo, dicoms,
+                                     args.bids_options)
 
         lgr.info("PROCESSING STARTS: {0}".format(
             str(dict(subject=sid, outdir=study_outdir, session=session))))
@@ -316,7 +339,7 @@ def process_args(args):
                         anon_outdir=anon_study_outdir,
                         with_prov=args.with_prov,
                         ses=session,
-                        bids=args.bids,
+                        bids_options=args.bids_options,
                         seqinfo=seqinfo,
                         min_meta=args.minmeta,
                         overwrite=args.overwrite,
@@ -333,7 +356,7 @@ def process_args(args):
             #  also in batch mode might fail since we have no locking ATM
             #  and theoretically no need actually to save entire study
             #  we just need that
-            add_to_datalad(outdir, study_outdir, msg, args.bids)
+            add_to_datalad(outdir, study_outdir, msg, args.bids_options)
 
     # if args.bids:
     #     # Let's populate BIDS templates for folks to take care about