Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/verify flag #192

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Sequential cell ids (#184)
* Add command line argument keep-id, which maintiains randomly generated cell ids. Otherwise cell ids are assigned incrementally (after the removal of cells), which should keep them consistent across runs in version control

* Modify test_cell and test_exception in test_keep_output_tags.py to use the new strip_output signature

* Fix failed test_end_to_end_nbstripout with test_max_size by passing --keep-id for keeping the existing ids

* Add tests for notebooks with and without the --keep-id flag. A new extension expected_id was added for expected output with ordered ids

* Modify the readme to include the --include-id flag

* Add keyword arguments for None inputs in test_keep_output_tags.py

* Rename expected output files to make desired sequential ids more explicit

Co-authored-by: Florian Rathgeber <florian.rathgeber@gmail.com>
  • Loading branch information
JasonJoosteCSIRO and kynan authored May 5, 2023
commit 08bfc679ff51b319f4241f02069b44495153fd7a
4 changes: 4 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,10 @@ Do not strip the output ::

nbstripout --keep-output

Do not reassign the cell ids to be sequential ::

nbstripout --keep-id

To mark special cells so that the output is not stripped, you can either:

1. Set the ``keep_output`` tag on the cell. To do this, enable the tags
Expand Down
8 changes: 5 additions & 3 deletions nbstripout/_nbstripout.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,9 @@ def main():
help='Do not strip the execution count/prompt number')
parser.add_argument('--keep-output', action='store_true',
help='Do not strip output', default=None)
parser.add_argument('--keep-id', action='store_true',
help='Keep the randomly generated cell ids, '
'which will be different after each execution.')
parser.add_argument('--extra-keys', default='',
help='Space separated list of extra keys to strip '
'from metadata, e.g. metadata.foo cell.metadata.bar')
Expand Down Expand Up @@ -409,7 +412,6 @@ def main():

parser.add_argument('files', nargs='*', help='Files to strip output from')
args = parser.parse_args()

git_config = ['git', 'config']

if args._system:
Expand Down Expand Up @@ -487,7 +489,7 @@ def main():
warnings.simplefilter("ignore", category=UserWarning)
nb = read(f, as_version=NO_CONVERT)

nb = strip_output(nb, args.keep_output, args.keep_count, extra_keys, args.drop_empty_cells,
nb = strip_output(nb, args.keep_output, args.keep_count, args.keep_id, extra_keys, args.drop_empty_cells,
args.drop_tagged_cells.split(), args.strip_init_cells, _parse_size(args.max_size))

if args.dry_run:
Expand Down Expand Up @@ -533,7 +535,7 @@ def main():
warnings.simplefilter("ignore", category=UserWarning)
nb = read(input_stream, as_version=NO_CONVERT)

nb = strip_output(nb, args.keep_output, args.keep_count, extra_keys, args.drop_empty_cells,
nb = strip_output(nb, args.keep_output, args.keep_count, args.keep_id, extra_keys, args.drop_empty_cells,
args.drop_tagged_cells.split(), args.strip_init_cells, _parse_size(args.max_size))

if args.dry_run:
Expand Down
8 changes: 5 additions & 3 deletions nbstripout/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def strip_zeppelin_output(nb):
return nb


def strip_output(nb, keep_output, keep_count, extra_keys=[], drop_empty_cells=False, drop_tagged_cells=[],
def strip_output(nb, keep_output, keep_count, keep_id, extra_keys=[], drop_empty_cells=False, drop_tagged_cells=[],
strip_init_cells=False, max_size=0):
"""
Strip the outputs, execution count/prompt number and miscellaneous
Expand Down Expand Up @@ -124,7 +124,7 @@ def strip_output(nb, keep_output, keep_count, extra_keys=[], drop_empty_cells=Fa
for tag_to_drop in drop_tagged_cells:
conditionals.append(lambda c: tag_to_drop not in c.get("metadata", {}).get("tags", []))

for cell in _cells(nb, conditionals):
for i, cell in enumerate(_cells(nb, conditionals)):
keep_output_this_cell = determine_keep_output(cell, keep_output, strip_init_cells)

# Remove the outputs, unless directed otherwise
Expand All @@ -148,7 +148,9 @@ def strip_output(nb, keep_output, keep_count, extra_keys=[], drop_empty_cells=Fa
cell['prompt_number'] = None
if 'execution_count' in cell and not keep_count:
cell['execution_count'] = None

# Replace the cell id with an incremental value that will be consistent across runs
if 'id' in cell and not keep_id:
cell['id'] = str(i)
for field in keys['cell']:
pop_recursive(cell, field)
return nb
90 changes: 90 additions & 0 deletions tests/e2e_notebooks/test_max_size.ipynb.expected_sequential_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "0",
"metadata": {},
"source": [
"This notebook tests that outputs can be cleared based on size"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"aaaaaaaaaa\n"
]
}
],
"source": [
"print(\"a\"*10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2",
"metadata": {},
"outputs": [],
"source": [
"print(\"a\"*100)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.4"
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 5
}
93 changes: 93 additions & 0 deletions tests/e2e_notebooks/test_nbformat45.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "5c42035d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'This is the new Jupyter notebook'"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\"This is the new Jupyter notebook\""
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "886205fa",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"'text2'"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\"text2\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a183d4e9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"f(3) = 4\n"
]
}
],
"source": [
"def f(x):\n",
" \"\"\"My function\n",
" x : parameter\"\"\"\n",
" \n",
" return x+1\n",
"\n",
"print(\"f(3) = \", f(3))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
61 changes: 61 additions & 0 deletions tests/e2e_notebooks/test_nbformat45.ipynb.expected
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "5c42035d",
"metadata": {},
"outputs": [],
"source": [
"\"This is the new Jupyter notebook\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "886205fa",
"metadata": {},
"outputs": [],
"source": [
"\"text2\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a183d4e9",
"metadata": {},
"outputs": [],
"source": [
"def f(x):\n",
" \"\"\"My function\n",
" x : parameter\"\"\"\n",
" \n",
" return x+1\n",
"\n",
"print(\"f(3) = \", f(3))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
61 changes: 61 additions & 0 deletions tests/e2e_notebooks/test_nbformat45.ipynb.expected_sequential_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0",
"metadata": {},
"outputs": [],
"source": [
"\"This is the new Jupyter notebook\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1",
"metadata": {},
"outputs": [],
"source": [
"\"text2\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2",
"metadata": {},
"outputs": [],
"source": [
"def f(x):\n",
" \"\"\"My function\n",
" x : parameter\"\"\"\n",
" \n",
" return x+1\n",
"\n",
"print(\"f(3) = \", f(3))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading