From 4cda52c74c3286b7cf0ac5a0c8738f9b3ff5c34e Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Tue, 15 Jan 2019 22:51:52 +0100 Subject: [PATCH] New --sync argument on jupytext CLI #146 --- jupytext/cli.py | 62 +++++++++++++++++++++++++++++++++++++++++++-- jupytext/combine.py | 2 ++ tests/test_cli.py | 35 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/jupytext/cli.py b/jupytext/cli.py index f77afa0e3..6e2c015e6 100644 --- a/jupytext/cli.py +++ b/jupytext/cli.py @@ -246,6 +246,11 @@ def cli_jupytext(args=None): parser.add_argument('--paired-paths', '-p', help='Return the locations of the alternative representations for this notebook.', action='store_true') + parser.add_argument('--sync', '-s', + help='Synchronize the content of the paired representations of the given notebook. Input cells ' + 'are taken from the file that was last modified, and outputs are read from the ipynb file,' + ' if present.', + action='store_true') parser.add_argument('--pre-commit', action='store_true', help="""Run Jupytext on the ipynb files in the git index. Create a pre-commit hook with: @@ -296,6 +301,11 @@ def cli_jupytext(args=None): raise ValueError('--paired-paths applies to a single notebook') return args + if args.sync: + if not args.notebooks: + raise ValueError('Please give the path to at least one notebook') + return args + if not args.input_format: if not args.notebooks and not args.pre_commit: raise ValueError('Please specificy either --from, --pre-commit or notebooks') @@ -314,6 +324,49 @@ def cli_jupytext(args=None): return args +def sync_paired_notebooks(nb_file, nb_fmt): + """Read the notebook from the given file, read the inputs and outputs from the most recent text and ipynb + representation, and update all paired files.""" + notebook = readf(nb_file, nb_fmt) + formats = notebook.metadata.get('jupytext', {}).get('formats') + if not formats: + sys.stderr.write("[jupytext] '{}' is not paired to any other file\n".format(nb_file)) + return + + max_mtime_inputs = None + max_mtime_outputs = None + latest_inputs = None + latest_outputs = None + for path, fmt in paired_paths(nb_file, formats): + if not os.path.isfile(path): + continue + info = os.lstat(path) + if not max_mtime_inputs or info.st_mtime > max_mtime_inputs: + max_mtime_inputs = info.st_mtime + latest_inputs, input_fmt = path, fmt + + if path.endswith('.ipynb'): + if not max_mtime_outputs or info.st_mtime > max_mtime_outputs: + max_mtime_outputs = info.st_mtime + latest_outputs = path + + if latest_outputs and latest_outputs != latest_inputs[0]: + sys.stdout.write("[jupytext] Loading input cells from '{}'\n".format(latest_inputs)) + inputs = notebook if latest_inputs == nb_file else readf(latest_inputs, input_fmt) + sys.stdout.write("[jupytext] Loading output cells from '{}'\n".format(latest_outputs)) + outputs = notebook if latest_outputs == nb_file else readf(latest_outputs) + notebook = combine_inputs_with_outputs(inputs, outputs) + else: + sys.stdout.write("[jupytext] Loading notebook from '{}'\n".format(latest_inputs)) + notebook = notebook if latest_inputs == nb_file else readf(latest_inputs, input_fmt) + + for path, fmt in paired_paths(nb_file, formats): + if path == latest_inputs and (path == latest_outputs or not path.endswith('.ipynb')): + continue + sys.stdout.write("[jupytext] Updating '{}'\n".format(path)) + writef(notebook, path, fmt) + + def jupytext(args=None): """Entry point for the jupytext script""" try: @@ -325,14 +378,19 @@ def jupytext(args=None): if args.paired_paths: main_path = args.notebooks[0] - nb = readf(main_path, args.input_format) - formats = nb.metadata.get('jupytext', {}).get('formats') + notebook = readf(main_path, args.input_format) + formats = notebook.metadata.get('jupytext', {}).get('formats') if formats: for path, _ in paired_paths(main_path, formats): if path != main_path: sys.stdout.write(path + '\n') return + if args.sync: + for nb_file in args.notebooks: + sync_paired_notebooks(nb_file, args.input_format) + return + convert_notebook_files(nb_files=args.notebooks, fmt=args.to, input_format=args.input_format, diff --git a/jupytext/combine.py b/jupytext/combine.py index 72bc436a3..c62a1cf99 100644 --- a/jupytext/combine.py +++ b/jupytext/combine.py @@ -88,3 +88,5 @@ def combine_inputs_with_outputs(nb_source, nb_outputs): output_other_cells = output_other_cells[(i + 1):] break + + return nb_source diff --git a/tests/test_cli.py b/tests/test_cli.py index e3827e729..dd7e97108 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -401,3 +401,38 @@ def test_paired_paths(nb_file, tmpdir, capsys): formats = nb.metadata.get('jupytext', {}).get('formats') assert set(out.splitlines()).union([tmp_ipynb]) == set([path for path, _ in paired_paths(tmp_ipynb, formats)]) + + +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_py')) +def test_sync(nb_file, tmpdir): + tmp_ipynb = str(tmpdir.join('notebook.ipynb')) + tmp_py = str(tmpdir.join('notebook.py')) + tmp_rmd = str(tmpdir.join('notebook.Rmd')) + nb = readf(nb_file) + nb.metadata.setdefault('jupytext', {})['formats'] = 'ipynb,py,Rmd' + writef(nb, tmp_ipynb) + + # Test that missing files are created + jupytext(['--sync', tmp_ipynb]) + + assert os.path.isfile(tmp_py) + compare_notebooks(nb, readf(tmp_py)) + + assert os.path.isfile(tmp_rmd) + compare_notebooks(nb, readf(tmp_rmd), 'Rmd') + + # Now we keep only the first four cells and save to Rmd + nb.cells = nb.cells[:4] + writef(nb, tmp_rmd, 'Rmd') + jupytext(['--sync', tmp_ipynb]) + + nb2 = readf(tmp_ipynb) + compare_notebooks(nb, nb2, 'Rmd', compare_outputs=True) + + # Now we keep only the first two cells and save to py + nb.cells = nb.cells[:4] + writef(nb, tmp_py, 'py') + jupytext(['--sync', tmp_ipynb]) + + nb2 = readf(tmp_ipynb) + compare_notebooks(nb, nb2, compare_outputs=True)