diff --git a/phys2bids/cli/run.py b/phys2bids/cli/run.py index 0527d6155..f2afa7e23 100644 --- a/phys2bids/cli/run.py +++ b/phys2bids/cli/run.py @@ -134,6 +134,12 @@ def _get_parser(): help='full path to file with info needed to generate ' 'participant.tsv file ', default='') + optional.add_argument('-report', '--report', + dest='make_report', + action='store_true', + help='Generate a report with the data and generated folder structure. ' + 'Default is False.', + default=False) optional.add_argument('-debug', '--debug', dest='debug', action='store_true', diff --git a/phys2bids/phys2bids.py b/phys2bids/phys2bids.py index f97323e74..8a95327d6 100644 --- a/phys2bids/phys2bids.py +++ b/phys2bids/phys2bids.py @@ -38,6 +38,7 @@ from phys2bids import utils, viz, _version, bids from phys2bids.cli.run import _get_parser from phys2bids.physio_obj import BlueprintOutput +from phys2bids.reporting.html_report import generate_report from phys2bids.slice4phys import slice4phys from . import __version__ @@ -129,7 +130,8 @@ def print_json(outfile, samp_freq, time_offset, ch_name): cite_module=True) def phys2bids(filename, info=False, indir='.', outdir='.', heur_file=None, sub=None, ses=None, chtrig=0, chsel=None, num_timepoints_expected=None, - tr=None, thr=None, pad=9, ch_name=[], yml='', debug=False, quiet=False): + tr=None, thr=None, pad=9, ch_name=[], yml='', make_report=False, + debug=False, quiet=False): """ Run main workflow of phys2bids. @@ -438,6 +440,9 @@ def phys2bids(filename, info=False, indir='.', outdir='.', heur_file=None, os.path.join(conversion_path, os.path.splitext(os.path.basename(phys_out[key].filename) )[0])) + # Only generate report if specified by the user + if make_report: + generate_report(conversion_path, logname, phys_out[key]) def _main(argv=None): diff --git a/phys2bids/reporting/__init__.py b/phys2bids/reporting/__init__.py new file mode 100644 index 000000000..107d3aba2 --- /dev/null +++ b/phys2bids/reporting/__init__.py @@ -0,0 +1 @@ +"""Visual reporting tools for inspecting phys2bids workflow outputs.""" \ No newline at end of file diff --git a/phys2bids/reporting/assets/apple-icon-180x180.png b/phys2bids/reporting/assets/apple-icon-180x180.png new file mode 100644 index 000000000..4b6e2e4bf Binary files /dev/null and b/phys2bids/reporting/assets/apple-icon-180x180.png differ diff --git a/phys2bids/reporting/assets/main.css b/phys2bids/reporting/assets/main.css new file mode 100644 index 000000000..426336395 --- /dev/null +++ b/phys2bids/reporting/assets/main.css @@ -0,0 +1,102 @@ +html, body { + margin: 0; + padding: 0; + font-family: 'Lato', sans-serif; + overflow-x: hidden; + overflow-y: scroll; +} +* { + box-sizing: border-box; + } + +.header { + background: linear-gradient(90deg, rgba(0,240,141,1) 0%, rgba(0,73,133,1) 100%); + height: 70px; + width: 100%; + position: fixed; + overflow: hidden; + margin: 0; + z-index: 100; +} + +.header a, span { + color: white; + text-decoration: none; + font-weight: 700; +} + +.header_logo { + display: inline-block; + float: left; +} + +.header_logo img{ + height: 50px; + top: 0; + left: 0; + padding-top: 15px; +} + +.header_links { + top: 0; + left: 0; + padding-top: 25px; + margin-left: 20px; + margin-right: 20px; + float: left; + display: inline-block; +} +.clear { + clear: both; +} + +.content { + margin-top: 100px; + display: flex; + width: 100%; +} + +.tree { + margin-left: 50px; + margin-right: 50px; + flex: 0.5; + min-width: 300px; + float: left; +} + +.tree_text { + margin-top: 10px; + margin-bottom: 70px; + width: 100%; +} + +.bk-root { + display: inline-block; + margin-top: 10px; + width: 100%; +} + +.bokeh_plots { + margin-left: 50px; + margin-right: 50px; + flex: 1; + min-width: 500px; + float: left; +} + +@media screen and (max-width: 600px) { + .content { + flex-wrap: wrap; + } + .tree { + flex-basis: 100%; + } + .bokeh_plots { + flex-basis: 100%; + } +} + +.main{ + margin-top: 100px; + margin-left: 100px; +} \ No newline at end of file diff --git a/phys2bids/reporting/assets/phys2bids_negativeReadTheDocs_500x150.png b/phys2bids/reporting/assets/phys2bids_negativeReadTheDocs_500x150.png new file mode 100644 index 000000000..815be7e98 Binary files /dev/null and b/phys2bids/reporting/assets/phys2bids_negativeReadTheDocs_500x150.png differ diff --git a/phys2bids/reporting/html_report.py b/phys2bids/reporting/html_report.py new file mode 100644 index 000000000..12cde4a91 --- /dev/null +++ b/phys2bids/reporting/html_report.py @@ -0,0 +1,234 @@ +"""Reporting functionality for phys2bids.""" +import sys +from distutils.dir_util import copy_tree +from os.path import join +from pathlib import Path +from string import Template +from bokeh.plotting import figure, ColumnDataSource +from bokeh.embed import components +from bokeh.layouts import gridplot + +from phys2bids import _version + + +def _save_as_html(log_html_path, log_content, qc_html_path): + """ + Save an HTML report out to a file. + + Parameters + ---------- + log_html_path : str + Body for HTML report with embedded figures + log_content: str + String containing the logs generated by phys2bids + qc_html_path : str + Path to the quality check section of the report + + Returns + ------- + html: HTML code of the report + + Outcome + ------- + Saves the html file + """ + resource_path = Path(__file__).resolve().parent + head_template_name = 'report_log_template.html' + head_template_path = resource_path.joinpath(head_template_name) + with open(str(head_template_path), 'r') as head_file: + head_tpl = Template(head_file.read()) + + html = head_tpl.substitute(version=_version.get_versions()['version'], + log_html_path=log_html_path, log_content=log_content, + qc_html_path=qc_html_path) + return html + + +def _update_fpage_template(tree_string, bokeh_id, bokeh_js, log_html_path, qc_html_path): + """ + Populate a report with content. + + Parameters + ---------- + tree_string: str + Tree of files in directory. + bokeh_id : str + HTML div created by bokeh.embed.components + bokeh_js : str + Javascript created by bokeh.embed.components + log_html_path : str + Path to the log section of the report + qc_html_path : str + Path to the quality check section of the report + + Returns + ------- + body : Body for HTML report with embedded figures + """ + resource_path = Path(__file__).resolve().parent + + body_template_name = 'report_plots_template.html' + body_template_path = resource_path.joinpath(body_template_name) + with open(str(body_template_path), 'r') as body_file: + body_tpl = Template(body_file.read()) + body = body_tpl.substitute(tree=tree_string, + content=bokeh_id, + javascript=bokeh_js, + version=_version.get_versions()['version'], + log_html_path=log_html_path, + qc_html_path=qc_html_path) + return body + + +def _generate_file_tree(out_dir): + """ + Populate a report with content. + + Parameters + ---------- + outdir : str + Path to the output directory + + Returns + ------- + tree_string: String with the tree of files in directory + """ + # prefix components: + space = ' ' + branch = '│ ' + # pointers: + tee = '├── ' + last = '└── ' + + def tree(dir_path: Path, prefix: str = ''): + """Generate tree structure. + + Given a directory Path object + will yield a visual tree structure line by line + with each line prefixed by the same characters + + from https://stackoverflow.com/questions/9727673/list-directory-tree-structure-in-python + """ + contents = list(dir_path.iterdir()) + # contents each get pointers that are ├── with a final └── : + pointers = [tee] * (len(contents) - 1) + [last] + for pointer, path in zip(pointers, contents): + yield prefix + pointer + path.name + if path.is_dir(): # extend the prefix and recurse: + extension = branch if pointer == tee else space + # i.e. space because last, └── , above so no more | + yield from tree(path, prefix=prefix + extension) + + tree_string = '' + for line in tree(Path(out_dir)): + tree_string += line + '
' + return tree_string + + +def _generate_bokeh_plots(phys_in, figsize=(250, 500)): + """ + Plot all the channels for visualizations as linked line plots for dynamic report. + + Parameters + ---------- + phys_in: BlueprintInput object + Object returned by BlueprintInput class + figsize: tuple + Size of the figure expressed as (size_x, size_y), + Default is 250x750px + + Outcome + ------- + Creates new plot with path specified in outfile. + + See Also + -------- + https://phys2bids.readthedocs.io/en/latest/howto.html + """ + colors = ['#ff7a3c', '#008eba', '#ff96d3', '#3c376b', '#ffd439'] + + time = phys_in.timeseries.T[0] # assumes first phys_in.timeseries is time + ch_num = len(phys_in.ch_name) + if ch_num > len(colors): + colors *= 2 + + downsample = int(phys_in.freq / 100) + plot_list = [] + for row, timeser in enumerate(phys_in.timeseries.T[1:]): + # build a data source for each plot, with only the data + index (time) + # for the purpose of reporting, data is downsampled 10x + # doesn't make much of a difference to the naked eye, fine for reports + source = ColumnDataSource(data=dict( + x=time[::downsample], + y=timeser[::downsample])) + + i = row + 1 + + tools = ['wheel_zoom,pan,reset'] + q = figure(plot_height=figsize[0], plot_width=figsize[1], + tools=tools, + title=f' Channel {i}: {phys_in.ch_name[i]}', + sizing_mode='stretch_both', + x_range=(0, 100)) + q.line('x', 'y', color=colors[i - 1], alpha=0.9, source=source) + q.xaxis.axis_label = 'Time (s)' + # hovertool commented for posterity because I (KB) will be triumphant + # eventually + # q.add_tools(HoverTool(tooltips=[ + # (phys_in.ch_name[i], '@y{0.000} ' + phys_in.units[i]), + # ('HELP', '100 :D') + # ], mode='vline')) + plot_list.append([q]) + p = gridplot(plot_list, toolbar_location='right', + plot_height=250, plot_width=750, + merge_tools=True) + script, div = components(p) + return script, div + + +def generate_report(out_dir, log_path, phys_in): + """ + Plot all the channels for visualizations as linked line plots for dynamic report. + + Parameters + ---------- + out_dir : str + File path to a completed phys2bids output directory + log_path: path + Path to the logged output of phys2bids + phys_in: BlueprintInput object + Object returned by BlueprintInput class + + Outcome + ------- + Creates new plot with path specified in outfile. + + See Also + -------- + https://phys2bids.readthedocs.io/en/latest/howto.html + """ + # Copy assets into output folder + pkgdir = sys.modules['phys2bids'].__path__[0] + assets_path = join(pkgdir, 'reporting', 'assets') + copy_tree(assets_path, join(out_dir, 'assets')) + + # Read log + with open(log_path, 'r') as f: + log_content = f.read() + + log_content = log_content.replace('\n', '
') + log_html_path = join(out_dir, 'phys2bids_report_log.html') + qc_html_path = join(out_dir, 'phys2bids_report.html') + + html = _save_as_html(log_html_path, log_content, qc_html_path) + + with open(log_html_path, 'wb') as f: + f.write(html.encode('utf-8')) + + # Read in output directory structure & create tree + tree_string = _generate_file_tree(out_dir) + bokeh_js, bokeh_div = _generate_bokeh_plots(phys_in, figsize=(250, 750)) + html = _update_fpage_template(tree_string, bokeh_div, bokeh_js, log_html_path, qc_html_path) + + with open(qc_html_path, 'wb') as f: + f.write(html.encode('utf-8')) \ No newline at end of file diff --git a/phys2bids/reporting/report_log_template.html b/phys2bids/reporting/report_log_template.html new file mode 100644 index 000000000..64182d820 --- /dev/null +++ b/phys2bids/reporting/report_log_template.html @@ -0,0 +1,33 @@ + + + + phys2bids report + + + + + +
+ +
+ phys2bids $version +
+
+ Quality Check +
+
+ Log +
+
+ GitHub +
+
+
+

$log_content

+
+ + \ No newline at end of file diff --git a/phys2bids/reporting/report_plots_template.html b/phys2bids/reporting/report_plots_template.html new file mode 100644 index 000000000..c3f56c9ca --- /dev/null +++ b/phys2bids/reporting/report_plots_template.html @@ -0,0 +1,50 @@ + + + + + phys2bids report + + + + + + + + +
+ +
+ phys2bids $version +
+
+ Quality Check +
+
+ Log +
+
+ GitHub +
+
+
+
+

Channel Plots

+

Hold on... It might take a few seconds to load the plots depending on how big your data is.

+ $content +
+
+

phys2BIDS Output Directory

+

$tree

+
+
+ + + +$javascript \ No newline at end of file diff --git a/phys2bids/tests/test_integration.py b/phys2bids/tests/test_integration.py index 6afc15215..e840e7df5 100644 --- a/phys2bids/tests/test_integration.py +++ b/phys2bids/tests/test_integration.py @@ -5,18 +5,23 @@ import shutil import subprocess from os import remove -from os.path import isfile, join, split -from pkg_resources import resource_filename +from os.path import dirname, isfile, join, split + +import phys2bids as p2b import pytest from phys2bids._version import get_versions from phys2bids.phys2bids import phys2bids +from pkg_resources import resource_filename def check_string(str_container, str_to_find, str_expected, is_num=True): - idx = [log_idx for log_idx, log_str in enumerate( - str_container) if str_to_find in log_str] + idx = [ + log_idx + for log_idx, log_str in enumerate(str_container) + if str_to_find in log_str + ] str_found = str_container[idx[0]] if is_num: num_found = re.findall(r"[-+]?\d*\.\d+|\d+", str_found) @@ -31,14 +36,20 @@ def test_integration_acq(skip_integration, samefreq_full_acq_file): """ if skip_integration: - pytest.skip('Skipping integration test') + pytest.skip("Skipping integration test") test_path, test_filename = split(samefreq_full_acq_file) test_chtrig = 3 - conversion_path = join(test_path, 'code', 'conversion') + conversion_path = join(test_path, "code", "conversion") - phys2bids(filename=test_filename, indir=test_path, outdir=test_path, - chtrig=test_chtrig, num_timepoints_expected=60, tr=1.5) + phys2bids( + filename=test_filename, + indir=test_path, + outdir=test_path, + chtrig=test_chtrig, + num_timepoints_expected=60, + tr=1.5, + ) # Check that files are generated for suffix in ['.json', '.tsv.gz']: @@ -68,17 +79,17 @@ def test_integration_acq(skip_integration, samefreq_full_acq_file): json_data = json.load(json_file) # Compares values in json file with ground truth - assert math.isclose(json_data['SamplingFrequency'], 10000.0) - assert math.isclose(json_data['StartTime'], 10.4251) - assert json_data['Columns'] == ['time', 'RESP - RSP100C', 'PULSE - Custom, DA100C', - 'MR TRIGGER - Custom, HLT100C - A 5', 'PPG100C', 'CO2', 'O2'] - - # Remove generated files - for filename in glob.glob(join(conversion_path, 'phys2bids*')): - remove(filename) - for filename in glob.glob(join(test_path, 'Test_belt_pulse_samefreq*')): - remove(filename) - shutil.rmtree(conversion_path) + assert math.isclose(json_data["SamplingFrequency"], 10000.0) + assert math.isclose(json_data["StartTime"], 10.4251) + assert json_data["Columns"] == [ + "time", + "RESP - RSP100C", + "PULSE - Custom, DA100C", + "MR TRIGGER - Custom, HLT100C - A 5", + "PPG100C", + "CO2", + "O2", + ] def test_integration_heuristic(skip_integration, multifreq_lab_file): @@ -100,21 +111,25 @@ def test_integration_heuristic(skip_integration, multifreq_lab_file): heur_path = resource_filename('phys2bids', 'heuristics') test_heur = join(heur_path, 'heur_test_multifreq.py') + shutil.rmtree(conversion_path) + # Move into folder subprocess.run(f'cd {test_path}', shell=True, check=True) # Phys2bids call through terminal - command_str = (f'phys2bids -in {test_full_path} ', - f'-chtrig {test_chtrig} -outdir {test_outdir} ', - f'-tr {test_tr} -ntp {test_ntp} -thr {test_thr} ', - f'-sub 006 -ses 01 -heur {test_heur}') - command_str = ''.join(command_str) + command_str = ( + f"phys2bids -in {test_full_path} ", + f"-chtrig {test_chtrig} -outdir {test_outdir} ", + f"-tr {test_tr} -ntp {test_ntp} -thr {test_thr} ", + f"-sub 006 -ses 01 -heur {test_heur} -report", + ) + command_str = "".join(command_str) subprocess.run(command_str, shell=True, check=True) # Check that call.sh is generated assert isfile(join(conversion_path, 'call.sh')) # Read logger file - logger_file = glob.glob(join(conversion_path, '*phys2bids*'))[0] + logger_file = glob.glob(join(conversion_path, "*phys2bids*.tsv"))[0] with open(logger_file) as logger_info: logger_info = logger_info.readlines() @@ -197,11 +212,14 @@ def test_integration_heuristic(skip_integration, multifreq_lab_file): assert math.isclose(json_data['StartTime'], 3.6960,) assert json_data['Columns'] == ['time', 'Trigger', 'CO2'] - # Remove generated files - shutil.rmtree(test_path_output) - shutil.rmtree(conversion_path) - for filename in glob.glob(join(test_path, 'Test1_multifreq_onescan*')): - remove(filename) + # shutil.copy( + # join(conversion_path, "phys2bids_report.html"), + # join(dirname(p2b.__file__), "reporting", "phys2bids_report.html"), + # ) + # shutil.copy( + # join(conversion_path, "phys2bids_report_log.html"), + # join(dirname(p2b.__file__), "reporting", "phys2bids_report_log.html"), + # ) def test_integration_multirun(skip_integration, multi_run_file): diff --git a/setup.cfg b/setup.cfg index cafcfd943..b47cf026a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,8 @@ doc = sphinx >=2.0 sphinx-argparse sphinx_rtd_theme +reports = + bokeh style = flake8 >=3.7 flake8-docstrings >=1.5 @@ -52,12 +54,15 @@ interfaces = test = pytest >=5.3 pytest-cov + %(doc)s %(interfaces)s + %(reports)s %(style)s all = %(doc)s %(duecredit)s %(interfaces)s + %(reports)s %(style)s %(test)s