From a499196b55cd0f70e15bd9e1c117bb29f4d7ea42 Mon Sep 17 00:00:00 2001 From: tbepler Date: Mon, 25 Nov 2019 13:07:37 -0500 Subject: [PATCH 1/5] updated topaz gui --- README.md | 1 + topaz/gui/topaz.html | 340 +++++++++++++++++++++++-------------------- 2 files changed, 184 insertions(+), 157 deletions(-) diff --git a/README.md b/README.md index f40ff99..deba8f6 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ A pipeline for particle detection in cryo-electron microscopy images using convo - Improvements to the pretrained denoising models - Topaz now includes pretrained particle picking models - Updated tutorials +- Updated GUI to include denoising commands - Denoising paper preprint is available [here](https://www.biorxiv.org/content/10.1101/838920v1) ## New in v0.2.2 diff --git a/topaz/gui/topaz.html b/topaz/gui/topaz.html index 8826bc1..6dcf1c6 100644 --- a/topaz/gui/topaz.html +++ b/topaz/gui/topaz.html @@ -201,14 +201,13 @@ $('.denoise_micrographs_inputs').each(function() { $(this).on('keyup', function() { var input7 = $('#input7').val() == '' ? '/path/to/input/images/*.mrc' : $('#input7').val(); - var model7 = $('#model7').val() == '' ? 'L2' : $('#model7').val(); + var model7 = $('#model7').val() == '' ? 'unet' : $('#model7').val(); var device7 = $('#device7').val() == '' ? '0' : $('#device7').val(); var format7 = $('#format7').val() == '' ? 'mrc' : $('#format7').val(); - var bin7 = $('#bin7').val() == '' ? '1' : $('#bin7').val(); var patchsize7 = $('#patchsize7').val() == '' ? '1536' : $('#patchsize7').val(); var patchpadding7 = $('#patchpadding7').val() == '' ? '384' : $('#patchpadding7').val(); - var output7 = $('#output7').val() == '' ? '/path/to/output/folder' : $('#output7').val(); - $('#result7').html("topaz denoise " + input7 + " --model " + model7 + " --device " + device7 + " --format " + format7 + " --bin " + bin7 + " --patch-size " + patchsize7 + " --patch-padding " + patchpadding7 + " --normalize --output " + output7); + var output7 = $('#output7').val() == '' ? '/path/to/output/folder/' : $('#output7').val(); + $('#result7').html("topaz denoise " + input7 + " --model " + model7 + " --device " + device7 + " --format " + format7 + " --patch-size " + patchsize7 + " --patch-padding " + patchpadding7 + " --normalize --output " + output7); }); }); }); @@ -216,29 +215,34 @@ $('.denoise_stack_inputs').each(function() { $(this).on('keyup', function() { var input8 = $('#input8').val() == '' ? '/path/to/particle/stack.mrcs' : $('#input8').val(); - var model8 = $('#model8').val() == '' ? 'L2' : $('#model8').val(); + var model8 = $('#model8').val() == '' ? 'unet' : $('#model8').val(); var device8 = $('#device8').val() == '' ? '0' : $('#device8').val(); var format8 = $('#format8').val() == '' ? 'mrc' : $('#format8').val(); - var bin8 = $('#bin8').val() == '' ? '1' : $('#bin8').val(); var output8 = $('#output8').val() == '' ? '/path/to/denoised/stack.mrcs' : $('#output8').val(); - $('#result8').html("topaz denoise " + input8 + " --model " + model8 + " --device " + device8 + " --format " + format8 + " --bin " + bin8 + " --stack --normalize --output " + output8); + $('#result8').html("topaz denoise " + input8 + " --model " + model8 + " --device " + device8 + " --format " + format8 + " --stack --normalize --output " + output8); }); }); }); - /*$( document ).ready(function() { + $( document ).ready(function() { $('.denoise_training_inputs').each(function() { $(this).on('keyup', function() { - var coordinateinput9 = $('#coordinateinput9').val() == '' ? '/path/to/Topaz/coordinate/file.txt' : $('#coordinateinput9').val(); - var imageroot9 = $('#imageroot9').val() == '' ? '/path/to/full/micrographs/' : $('#imageroot9').val(); - var boxsize9 = $('#boxsize9').val() == '' ? '192' : $('#boxsize9').val(); - var threshold9 = $('#threshold9').val() == '' ? '-9999' : $('#threshold9').val(); - var resize9 = $('#resize9').val() == '' ? '-1' : $('#resize9').val(); - var imageext9 = $('#imageext9').val() == '' ? 'mrc' : $('#imageext9').val(); - var output9 = $('#output9').val() == '' ? '/path/to/output/particles.mrcs' : $('#output9').val(); - $('#result9').html("topaz particle_stack " + coordinateinput9 + " --image-root " + imageroot9 + " --size " + boxsize9 + " --threshold " + threshold9 + " --resize " + resize9 + " --image-ext ." + imageext9 + " --output " + output9); + var dira10 = $('#dira10').val() == '' ? '/path/to/input/images/half1/' : $('#dira10').val(); + var dirb10 = $('#dirb10').val() == '' ? '/path/to/input/images/half2/' : $('#dirb10').val(); + var method10 = $('#method10').val() == '' ? 'noise2noise' : $('#method10').val(); + var arch10 = $('#arch10').val() == '' ? 'unet' : $('#arch10').val(); + var optim10 = $('#optim10').val() == '' ? 'adagrad' : $('#optim10').val(); + var lr10 = $('#lr10').val() == '' ? '0.001' : $('#lr10').val(); + var criteria10 = $('#criteria10').val() == '' ? 'L2' : $('#criteria10').val(); + var crop10 = $('#crop10').val() == '' ? '800' : $('#crop10').val(); + var batchsize10 = $('#batchsize10').val() == '' ? '4' : $('#batchsize10').val(); + var numepochs10 = $('#numepochs10').val() == '' ? '100' : $('#numepochs10').val(); + var device10 = $('#device10').val() == '' ? '0' : $('#device10').val(); + var numworkers10 = $('#numworkers10').val() == '' ? '16' : $('#numworkers10').val(); + var output10 = $('#output10').val() == '' ? '/path/to/output/folder/and/file' : $('#output10').val(); + $('#result10').html("topaz denoise --dir-a " + dira10 + " --dir-b " + dirb10 + " --method " + method10 + " --arch " + arch10 + " --optim " + optim10 + " --lr " + lr10 + " --criteria " + criteria10 + " --crop " + crop10 + " --batch-size " + batchsize10 + " --num-epochs " + numepochs10 + " --device " + device10 + " --num-workers " + numworkers10 + " --save-prefix " + output10); }); }); - });*/ + }); @@ -478,7 +482,7 @@ display: block; background-color: inherit; color: black; - padding: 12px 16px; + padding: 9.75px 16px; width: 100%; border: none; outline: none; @@ -808,6 +812,8 @@ .via_logo { background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAPoAAABkBAMAAABdrjA9AAAACXBIWXMAAAsSAAALEgHS3X78AAAAJFBMVEUhISH///8BAScODhX//wDZ2QRMTDaQkA/W1tZzc3eoqKjt7QHwJDbUAAAFdElEQVRo3u2ZTW/bNhjHWVJwrxUCBGlOgizbh1xos3PsnASFVq6Dl8GHXVpby2WXwrHhg0/GPkHQfYGgX6AwmvbrjZREiUokh4/SJhjmJ2kRvZA/8s8/X4XQPvaxj328ZDgII+yi+N/zxxy1UGf0Hv1y8cSMCJX/M1giH1l4cXqOR9Y1nIgvaPZ3Zy5vrB1Q+rA3cfye0wowLX+D7kjdGjkNx0GOI9qtPaY9F6+bxKW9pmEZ8MQNvLnrWtgtT9Eg1XhiWc3FZbC8HHd4j/N5NPL4qM2ny8vAsPJRJ7DwchK1l99Kn3PuVxfdt3A09qNxsHi9bfzW8UVhmHUVLK5M6d7JPPJGF6vFXVjeJwIHuUKX7FeLDufThetHbhA2g/5W5NSM3JUbrB1TD7UWIkXzfOV2ysvbD3DIoy3+6vcux5SPCyW/PpmvXX/pBr4boMBjlsjruxssHWpI74wcaxrNI7Iof97YYr4Ofe/X8N3HO29WEChyOoGgr1x/1BqjoDWPmKh9X9CNXR+iNr9wPB6WJznd4tHJdBLJKk7+9HSB8HeKBT1cuqHHHRQgaTynJVxnTifiRwx3qFn+uL/F4cl8Jao4ccPFt8KIKJJIjXuiIDj2iMwDE/rjxsLTL9OwNV9Hoort8C/Xcp5zIMacBxetbeTxrx7/0uIhfU46IqKT9a8bvfdiPDvH0HG8MtyqeKgtceSAS9r3+iSuzOJxSe9+r4htRYIFv3fnpCqHPx6nj6rSVo2SvftV8qpyuHzUHDioki3Ypa7eLF5VDp4B3ejB61UxPur0KsgPo7/jxfCfl35P1PPnpRf7H97T//N0wiiTv9ktFsdDOkuD7qATRmH0qzRm6lZyOX1AVy9e0Wp6Qz4F0M83dhIH6Z1hevmAbquophORGXoxOhVPP70YfajlY+q6rkiTu44QdmPPSl1H4xdZtesG4oVDCutxZwW94tajpT2OxhXf0eNkPY6BdFbQS14fldPRI3SysdUb5nSZ6FB780xe1aLHtpgB6R90vaR+B6gWvW8f6zIajrRdXS9ZmE/16AP7QLzxhsLog4LtREOgWnTyyj640WQ0pBdtx2JEHfqtLQqQy2hIL9ouNl0tungqZSRAuq5XYro6dCYyoZqMpvO7bjvStT/Xow+E4WSnewOk67YTrTerR5eiidT2WwqjS9v9rbVeLbpovxnSbWdKJ4U57rAmXU4PcSMSIF3Ta5CUA06Ppwcx3mUyGtM123WTxHD6IB7lNBmN17SDbHZITVeDnvRUKWM6dhjTpV6fddPB6cn0oNvOmJ7r1U8NAKdv7GyFQYD0jbLdIB0s4PT0WS6jOV3ZjnRTDcD0MzW1ZjKa76SU7ZTp4HRVbCljUg5z+jDViynLQOnCbYneKLMdYBeZ6kXUZAemb9QQl9nOnK5sd6Y6K5geL/VlDNWUBaALvY5o3npg+tB+q3aZynYA5RPTx/NULfrA1iO2HYCebGiSeaoOvVugHwHpyZIomadq0MmmQLeB9GRlmZkOSmeZ6RhTK0sIPd7QZKaD0uX0QNLop7aDnBrJQYJ9yJZ3QPog38LEU5a8gtDjleVtthgH0rvadkQt0CH02Hb5whpGJ3mx8ykeQpd6HedbGhid6dtQNdaC6Df66Q1U+cIWPLUd6KxS6qXmKShdN122UgLRB4WDB9C5DekWjl7SBTrsnLZw6AI5s2LxKRfNj1rF2HHEGIhOCsdd2Py8jm0KB35UvfcPiH6rnyFA6LcFOqpHf5VvJUvo1LTu9eiIMO18u/qEvKzd05Nz/Ygd2O77LyP/K7rRd9id9Cd8h+VVYU6vzAI9GmZf/3fRn/D13zB2fYf9+WHt+Ab984OyYlC0j33sYx87418ZX9xBzIakRAAAAABJRU5ErkJggg==); display:inline-block; width:250px; height:100px;} + .denoising_logo { background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAE4AAABLCAMAAADpuUGcAAAACXBIWXMAAAsSAAALEgHS3X78AAAAY1BMVEVHcEyAgICCgoKBgYF/f3+CgoKCgoKBgYGEhIR/f3+YmJiWlpaAgICCgoKUlJSNjY2ZmZmqqqqIiIikpKSenp6ysrJ4eHhubm5hYWG9vb3IyMhNTU3Z2dnu7u42NjYhISEQEBDtS1fZAAAADHRSTlMAbLOTHuHN8v1C4scOuIPcAAAHpElEQVRYw62YiUIaSxBFZZsZTO/rLAj+/1e+Uw0+NSCSxE4MGuFOVd1bWz893T7b/Wqz7na96vvderPab5/+/mz3m26n3o9WfbdZ/SXiftX1Wv1+tO42+78A23T6HeIz4u5PAberNzAtRwXLCeEN9w8B9+v+/LEQjHOhliF5TjLGDmdE3a0eRlvt9BnMORN9NKrYFOW4lEy4APabxzjZbppperDt0/xrVTAmgeisSU6cboDrRxzeri8xCy1ayuEhx7TXYGPiZWhB1N3+QTQdVC1FlVqCRMwZgIwJeM2PAcvPlHyLt13LcweTjAYNOJtcIn5a3LWlWPw1w6BD3z+At900tJRzjtaqgsdgETNVghNqk8PUoEMyz42v7i4fq/bM4Od59mIfoc9TtGXEP4NUYjRYZ3TK3jU8vb6Dt981SomQgNQ6GAecH8ZxiAmmHXYSRF69d+lZ3t2v7tOgh/YB52yphAu4nIKy0SmxVWJqJZjIJj+LL7svw9cEh3Yj4kINwfANdmafrBNjqzLgWNG0x8A8/ervuNtcFZdcaj7FCY0Rehxz2euxwnOe+E9M8/x6np67O+4Kq3DmExGyZYg5G62aglPMXo1jCWmasjOScTzVZy/mfcGuGKd7m3gXylXKwJ5VCG1oGimlaBen2QeIwtlkCMod85pxcIlVHpNCSdmqOmpSTYJYCqA4mAKEeAknvKRfuy+it20PEtfkvSnKVxmXY/FImqJSq5umecJirCScwrJ1XfvU/raCh+FiihFdubIcj0uZDi9zQnrVTgchpy4FluT3DgulMurNbV+Hnjy33kvSDxrbTsfT4g4vL5NeFuUPBx/qyEHnXlyFjV/dTTLEV/KcFPKwFw2RH3H19HqqDpjxuNj55SVj8FjGaozI0UtqS8249nYvRlsp31AxeUOsqE7l+Pq66Jjt0pw+xHIEDo4DHE9SJqx4O1xxu2pERBeU48G81LEY7yrOqTKMCzzPL4ccg0utrto8z9lUZdci/vVV6AQuSO1GKVPSwBSXY5wmy7cLJlqNiA/z4XCYM1ZliQlctOBdSaURzqOJHmJNalyqTn6aD7Mb6iIuglnNDC8vLwczqiih4+uZFqqvuGjFBpVYskwEwacrtVNEkgJ4y0gYT4uKDTCOYygS5eSehYursiJM8EUqEiTCYhHJUms+HLI3Q/P3BM3HY00wPAntY0AuoV8PtPHf4VorKUEa1pTznB2fxiQzuSB5J4I7HsXGY0lEcPIAqpTjUIG7Vsq5cBoJHhFDBMPCKQNCW4qTKIJYddSSHhnNeBqaFT3dhJNKQ9eKHqnDGdILRdPKjqfTUstYfOPTUVmEl0n4oFAoxHkTTqigjFDKaPdUSCLGZ8lQYiQnNEnipFpeX49J+JioDLX0N6k415OBItsKGVlr3eTjJMW8sVzIs2GacwDuNGDcTB2jEe+6W3BrYdbQT00r54ZyHqmkM7k0BSghvU6nMdAsRvSSSN/knS6DkQp6rbtN44LGDxlJIphwhTaG8LKtFGALqcsyRl+WUU8vs1SdUgf3a3crK1q5I/TV0KYYwnCyMCVOwgANjIKPefwpxMs6ujpvpRZY0+rx5nankBmHiQIOmCTIEXBCqWMAVeCI2sgzyIaIDitTQky/1K2Kcm7ZqtByYJZKS/LGLPm1jDVML4dkyONlFAjimaREUM3OzedG615J5+FvkdkQWq0MAE6yFQSkGEyWCSXyLSdDaqU8uucvek+rn0xasFGkUJG80xQFTslsZ1Um+Q9yJGcyI6mVAiS83uqMzduBeMmUHr0JwWVxahnb3K7NJNIFTFKaUsfzGPna2HNzTBFutUzqjtYdGWNtlILLYMeIUUeHbFtBR0i+jX8MHGb3RSN7Mw9xuuAiA7oqpAXA6MbUpSYatvXSNDUlneiyZZivjbvMila6ccCcch5OEqNjTkBHKW6iEUYpqSWoOPln9ZVxb1qR6XIoaiwMiCVYTY+RgQQyMVSmMKc1foqaor074J0nMtHJILOcQalVZEa8koSRSGCtpAuPFC4ETX89frZBGzjmHSbZ0hp+kBGP+pmcqoFIMKOwU4EVTBuk7wzH5zUgUKcczZVuj30iE7KEgbbUlFsFrDKhOnVeT+/tZvvuvPEYac6BOi4jp7ArDTO5NpkoWTb4QenvV5XL7A6PbYGQIaBEaahR2rTsZdQt6Xfsjm/L6Orutth2KCMLIv1LMlOyNNMhZ1igPjANEcDhfWX+Hg+9OGle2MBCJRLBOO9o1WKXE5F/OHd30f35NoGBWDIpkHHMe6SniZJwdJFEE/q8zd/Hu6yg5K9tu0m07MTEEm1LKRmu7htUf2+XuizIrZ4aR3mympoVpADz44XR9ebjncg9+RHA/68p2JDZqqTj84/S7+HfXlT1du4KZrvZ6Q93J7rdWHyI/f5dpQ/hyUVKr24c3b/felyirB66vNiu1rvf7nn0sFuv9r9npXpE0OeLo3ahpc9Wdd3VpdY7a+fbmm8vQ7bb/ep89rfuxz7jPXq58m0avZ0fxrt7OfDgldVnvP0/43UfCe5+Fk/9O95nAa6f/hnvo2B2Tz+L9/QDeKv/8bqnnzhvAuxXTz+D1/3JNeZDd7B9R2X9Dzld2TSqoC4RAAAAAElFTkSuQmCC); display:inline-block; width:78px; height:75px;} + .preprocessing_logo { background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADYAAABLBAMAAADJ12nxAAAACXBIWXMAABbqAAAW6gHljkMQAAAAMFBMVEWkpKT9/f39/f3////9/f36+vqJiYl7e3tycnLu7u6Xl5djY2PU1NTFxcW0tLTh4eFzh75wAAAAA3RSTlP6696exOcsAAAETklEQVRIx62V32tbVRzAg4V22haZb/4Xoefch4y0kZ1zhULmYOccobJue/G2S5eQB0lKiqNP7aWXsmKSSRW2giyhIUqf2ot3dkKk0FWkTBMHdb5UM2OLViMD13X4Pefe2x+J88kLgXA+53O+3+/53nNuoJeIFzxnA8UyfgG7ECikX3Wh1s6KnVMNOXz+O619zfWuqSoMh2eaeptXCRczFV2E09ezbV6FnBtNNvVwajSh4BA+5lF0CYbDmb3xOIOh4M/aMY+iufF4JIPup8tQbHDGy0p5AJ+mP8twrTYN9QTTMrzvYUJxbTqGWOgx1BNMyvCeRxDGRKulGohFoJ7g7CU3K+UhRDEBp4oJ1LM3i9ysXE/BCJSJyfBo6n3NzcrzECLSSTKKbo0ksxAedtn3EORDho2srCcdhwj16UafzyQMGVlZz/0M5BbqmnpwyBCWTNYTnsQyfCF5xBCWDLICJuGZdoakB7nR4L8wrBj+D4b+d0Z8RtsZJT6jrUwj2GMckZNsJgu7o9jLskE+g8VwOJVgWNX+OFXFLlQelmw87nrwGkBLqWIj8h/Eg9aqeCxazDQVBJbIqj5o0Fr0RjIg2PDodTkk14T3hEomWwueEOzWaIJBU4GpxaKTGodIm5IJtieDI+htTi4GnuDhYkoxgdR88HJyMckg0siVgHtm4W1XnoDX9JFkEMn1hAjVpxp4EBhESmXUWSEHHpPBq9ITPFpMuueoz2OCwdu+LpkQwxn/jPknGMpMuowd6C0Mgo+4TCD/3B6dffI87bK8OOlxCb9U7HJCMO2Yx+ZlDAo/nfYbzXPVI48/MXPetTXX2Cv11K9phx7fsjoEOwWzeS1WMIzYon7osR3zHgo6SzD7o9KVHzaNDup5RGR3zJXdbnMV2IDxFdLq5XnP+x2FxhzTsc27mqBP42pC6S/lDTjxh8u7fz87Zd/QxWZhUV6YsWJFemwDlCqiKNR9QyOdhZ+ADRbKKh7/0Lauyux5/0r+9o+KDRWabrywveLenGxr2nqvdgf+RWIVN0/efde7FYOO+cVNWfZArKwrj+9/4m3qoGUu1UtwadaNGHPZTsXf8G6r1zASX2/GfuuJcliTXBzP+S3cWo0sbBqGsUB21/IXAvz5xLJ/wfP9Dp4PlUqlBdJVeuVBgI/Zn/tfBv7xdl4XhcW52Defztc/AGYe896FaZ2VwZ61DvZOb4B1Oyt5n22sQM23BacD2yKYCIiLdeswzx1rVZfd52uL6MkfsJ9oo8NjQxOm7SZ88OujwkuyR2/f85J5y1HtlbDW1fhTMS8Zvj9lXHMZfzZQ6VP7Yt1RI+fNJqX+NnC1Z0OWudzEguOHS9rJ7xjsvm05S+v8cs1e1k9+/4SIOvbumG1MOJaZa/X49w30pu1M/mJb37Z6QsfijG1tRyxru9VTldl2nt+cbfneUi6PUL+zqulalgqduw/8ORs4rZ7XO6+ebn1e+wc/BQRd7DBzCAAAAABJRU5ErkJggg==); display:inline-block; width:54px; height:75px;} .picking_logo { background:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEwAAABLBAMAAAAovHygAAAACXBIWXMAAA7DAAAOwwHHb6hkAAAAD1BMVEWBgYFHcEx/f3+AgIB/f3/QWM4IAAAABXRSTlP+AEfMi12fsSIAAAHwSURBVEjHpZbhtYMgDIVTcYCCDCDWAVA7gFr3n+khUEgQ1J6Xf8JHEi5IAvxo/UcDe694CA5Q9wFnbD3Beg3B1iLWA7axgEkXbpomTfxRTGibk4rRVQ5rzcRbfb/2vdQZTOJ0jL3CN8F0WB05plJMfgejzd4d0LEl0UeYlQkm05A+7ECx9ujMSlQTzA+k1lrtAMdcMpjJrsJY65I92LwHASRalcWaPSog50sWE/teAS3iedvDQEytLmD7DFyl5uIEzKldSG6BqJoqYfoWZvJ5QEiAlSgjcAWXG7VzEauK2PMeZhKCGP8e9jjBIGJDEZMYu+lt/L+3X3fa3sVuHVZzilU/3pDz+zbA1f9HL/m+pKyuAn4p3NOkDfiDFx8R4Jd7II+DKP2o9kED+kBlxWX44WoKUTV9BgVkr1xjBQW8jGX3CTwJmnEn7dNLHtVjkbHOFMYaWmdj9ahxlRG+dtPzl9qPAKqJB06EkgjUmUlvobVaYax1zGbru0I9xIirs3BU7WJPW9d12wSxhntsDnnNpMEIAkFsK6yOYs5RDvNTKvQLzmIPYTFJW5jegdOatFI6icC7fus7nnRcDWTPKcG8soyfY/6YhnPMO6v5OfbK3YwjNt9xxuGWMx6P6RzDPd9FbiO/xGTuf8rI+2HLJcX/AKTZty2z1S2iAAAAAElFTkSuQmCC); display:inline-block; width:76px; height:75px;} @@ -905,6 +911,7 @@
+ @@ -925,13 +932,14 @@

Welcome!

This GUI will allow you to pick training particles and make Topaz commands. Here is the general workflow:

-
-
+
+
+ → → → → @@ -939,17 +947,125 @@ + Denoise
images Normalize &
downsample images Pick training
particles, or
analyze picks Train a
neural network Extract particle
coordinates -
-

Citation: Bepler, T., Morin, A., Rapp, M., Brasch, J., Shapiro, L., Noble, A.J., Berger, B. (2019). Positive-unlabeled convolutional neural networks for particle picking in cryo-electron micrographs. Nature Methods

+
+

Citations: Bepler, T., Morin, A., Rapp, M., Brasch, J., Shapiro, L., Noble, A.J., Berger, B. (2019). Positive-unlabeled convolutional neural networks for particle picking in cryo-electron micrographs. Nature Methods +

Bepler, T., Noble, A.J., Berger, B. (2019). Topaz-Denoise: general deep denoising models for cryoEM. bioRxiv

+
+
+
+
    +
    • +
+
+
+

   Topaz-Denoise

 +
+ +
Denoise Micrographs +
+

+

Create a command for denoising micrographs.

+ Parameters:            + Must modify + Might modify + Rarely modify +

+
+
+ + + + + +
Input micrographs
Output folder
Model
GPU/CPU device
Format
+
Advanced options
+
+ + + +
Patch size
Patch padding
+
+ +

Command:

+
+ topaz denoise /path/to/input/images/*.mrc --model unet --device 0 --format mrc --patch-size 1536 --patch-padding 384 --normalize --output /path/to/output/folder/ +
+

+
+ +
Denoise Particle Stack +
+

+

Create a command for denoising a particle or micrograph stack.

+ Parameters:            + Must modify + Might modify + Rarely modify +

+
+
+ + + + + +
Input stack
Output stack
Model
GPU/CPU device
Format
+

Command:

+
+ topaz denoise /path/to/particle/stack.mrcs --model unet --device 0 --format mrc --stack --normalize --output /path/to/denoised/stack.mrcs +
+

+
+ +
Train New Denoising Model +
+

+

Create a command for training a Topaz-Denoise model.

+ Parameters:            + Must modify + Might modify + Rarely modify +

+
+
+ + + + + + + +
Input folder #1
Input folder #2
Output files
Model architecture
Training criteria
GPU/CPU device
Number of CPUs
+
Advanced options
+
+ + + + + + + +
Training method
Optimizer
Learning rate
Crop size
Batch size
Number of epochs
+
+ +

Command:

+
+ topaz denoise --dir-a /path/to/input/images/half1/ --dir-b /path/to/input/images/half2/ --method noise2noise --arch unet --optim adagrad --lr 0.001 --criteria L2 --crop 800 --batch-size 4 --num-epochs 100 --device 0 --num-workers 16 --save-prefix /path/to/output/folder/and/file +
+

+


+
+
@@ -966,16 +1082,16 @@ Rarely modify


- + - - + +
Input micrographs
Input micrographs
Output folder
Number of CPUs
GPU/CPU device
Scale factor
GPU/CPU device
Scale factor

Advanced options
- + @@ -987,7 +1103,7 @@
topaz preprocess /path/to/input/images/*.mrc --scale 4 --sample 1 --num-workers -1 --format mrc,png --device 0 --niters 100 --alpha 900 --beta 1 --verbose --destdir /path/to/output/preprocessed/images/
-


Note: Denoising micrographs before preprocessing will make particle picking even easier. Training on denoised micrographs may not work any better than the originals. You can denoise micrographs in More Tools on the right. +


Note: Denoising micrographs before preprocessing and before binning will make particle picking easier. Training on denoised micrographs may not work any better than the originals. It is recommended that you denoise micrographs for manual picking/inspection by using the Denoise tab on the right.


@@ -1066,6 +1182,7 @@
  • Getting Started (F2)
  • Topaz GitHub Page
  • Topaz Citation
    • +
  • Topaz-Denoise Citation
  • VIA Project Page
  • Report an Issue
    • @@ -1444,14 +1561,19 @@

    File Not Found

      -
    1. Preprocess MRCs: Use Topaz to downsample, normalize, and convert micrographs to PNG (for use in Topaz GUI) and MRC (for use with Topaz training).
      2. +
    2. Denoise MRCs (recommended): Use Topaz-Denoise to denoise full-size raw micrographs.
      3. +
        +
      • Denoising using the pre-trained models works best on K2, K3, Falcon II, and Falcon III micrographs. You may wish to make your own denoising model if the pre-trained models are not optimal. +
        • +
      +
    3. Preprocess MRCs: Use Topaz to downsample, normalize, and convert denoised micrographs to PNG (for use in Topaz picking GUI) and raw micrographs to MRC (for use with Topaz training).
      • Your particle must have a diameter (longest dimension) after downsampling of: 70 pixels or less for resnet8; 30 pixels or less for conv31; 62 pixels or less for conv63; 126 pixels or less for conv127.
      • Topaz normalization uses image statistics for all input micrographs. A Gaussian mixture model is used to effectively make the ice region of each micrograph visible, even if there is gold in the image.
      -
    4. Pick Particles: Add images then pick particles: +
    5. Pick Particles: Add images then pick particles:
      • To add images:
          @@ -1472,15 +1594,13 @@

          File Not Found

      - -
    6. Train a model: Use Topaz to train a neural network model to pick your particles. -
      7. +
    7. Train a model: Use Topaz to train a neural network model to pick your particles.
        8. +
      • Training often works best when the training radius is 3 pixels or less. +
      • Several parameters are optimizable by cross-validation.
      - -
    8. Export Particles: To export particles, click Particle Picks → Export Picks in the top menu. These picks may then be used in Topaz training.
    9. Analyze Topaz Picks: Load Topaz picks by clicking Particle Picks → Import Picks.
        10. @@ -1506,7 +1626,7 @@

        File Not Found

    Output format
    Output format
    Sample
    Number of iterations
    Alpha
    - +
        
    Topaz Version 0.2.2VIA Version 2.0.7 (modified)
    Topaz Version 0.2.3VIA Version 2.0.7 (modified)
    @@ -1560,21 +1680,21 @@

    File Not Found

    1) To start picking, select images or add images from paths.
    2) Pick 100-1,000 particles (more is better) representing all orientations.
    -     â€¢ Pick across several representative images.
    -     â€¢ Not all particles need to be picked from each image.
    +     • Pick across several representative images.
    +     • Not all particles need to be picked from each image.
    3) Export your particle picks and proceed to Topaz training.
    -
    • Save your particles and project regularly. You may load them later to continue.
    -
    • For further help, see the Getting Started page. +
    • Save your particles and project regularly. You may load them later to continue.
    +
    • For further help, see the Getting Started page.

    Analyzing Topaz picks

    - • Import your Topaz particle picks as a CSV file and review the picks.

    -   â—¦ Press L on your keyboard to show the particle number, then the Up ↑ key will toggle the Topaz particle scores. Zoom-in on the image with + to inspect.

    + • Import your Topaz particle picks as a CSV file and review the picks.

    +   ◦ Press L on your keyboard to show the particle number, then the Up ↑ key will toggle the Topaz particle scores. Zoom-in on the image with + to inspect.

    Or

    - • Load your micrographs and particles in .star format into Relion. Picks may be colored based on Topaz score by using the AutopickFigureOfMerit column in the .star file. + • Load your micrographs and particles in .star format into Relion. Picks may be colored based on Topaz score by using the AutopickFigureOfMerit column in the .star file.

    @@ -1594,7 +1714,7 @@

    File Not Found

        Topaz Training Command Generator

     -

    Create a command for training a network using particle picks. +

    Create a command for training a model using particle picks.



    Parameters:            Must modify Might modify @@ -1604,13 +1724,13 @@

    File Not Found


    - - + + - + @@ -1684,7 +1804,7 @@

    File Not Found

    Rarely modify


    Training images folder
    Training particles
    Output
    Particle radius
    Output
    Particle radius
    Autoencoder
    Number of epochs
    GPU/CPU device
    Number of CPUs
    CNN model
    CNN model
    Method
    Num particles/image
    K-fold
    - + @@ -1737,9 +1857,9 @@

    Convert and/or Threshold Pick Coordinates


    Input micrographs
    Input micrographs
    Model
    Output filenames
    Particle radius
    - - - + + +
    Input file
    Output file
    Threshold
    Upscale picks
    Downscale picks
    Threshold
    Upscale picks
    Downscale picks

    Command:

    @@ -1759,9 +1879,9 @@

    Extract Particles as an MRC Stack


    - + - + --> @@ -1775,100 +1895,6 @@

    Extract Particles as an MRC Stack

    --> -
    Denoise Micrographs -
    -

    -

    Create a command for denoising micrographs using an implementation of Noise2Noise.

    - Parameters:            - Must modify - Might modify - Rarely modify -

    -
    -
    Input picks
    Input micrographs
    Output stack
    Output stack
    Boxsize
    Threshold
    Threshold
    Image extension
    Resize particles
    - - - - - - -
    Input micrographs
    Output folder
    Model
    GPU/CPU device
    Format
    Bin micrographs
    -
    Advanced options
    -
    - - - -
    Patch size
    Patch padding
    -
    - -

    Command:

    -
    - topaz denoise /path/to/input/images/*.mrc --model L2 --device 0 --format mrc --bin 1 --patch-size 1536 --patch-padding 384 --normalize --output /path/to/output/folder -
    -
    - - -
    Denoise Particle Stack -
    -

    -

    Create a command for denoising a particle stack using an implementation of Noise2Noise.

    - Parameters:            - Must modify - Might modify - Rarely modify -

    -
    -
    - - - - - - -
    Input stack
    Output stack
    Model
    GPU/CPU device
    Format
    Bin Stack
    -

    Command:

    -
    - topaz denoise /path/to/particle/stack.mrcs --model L2 --device 0 --format mrc --bin 1 --stack --normalize --output /path/to/denoised/stack.mrcs -
    -

    -
    - - -
    Convert | Threshold Picks

    @@ -1882,19 +1908,19 @@

    Extract Particles as an MRC Stack


    - - - + + +
    Input file
    Output file
    Threshold
    Upscale picks
    Downscale picks
    Threshold
    Upscale picks
    Downscale picks

    Specific filetype options
    - - - - - - + + + + + +
    Voltage
    Detector pixelsize
    Magnification
    Amplitude contrast
    Box size
    Image extension
    Voltage
    Detector pixelsize
    Magnification
    Amplitude contrast
    Box size
    Image extension
    @@ -1917,9 +1943,9 @@

    Extract Particles as an MRC Stack


    - + - + From 55ec396ba1ae232769456d67fc62d8a3a34f23e9 Mon Sep 17 00:00:00 2001 From: Tristan Bepler Date: Mon, 25 Nov 2019 13:38:16 -0500 Subject: [PATCH 2/5] manifest updated --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 46ce159..e747191 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include README.md include topaz/gui/topaz.html include topaz/pretrained/denoise/*.sav +include topaz/pretrained/detector/*.sav From fcf999920bd4a3a212b5de7a93e61daa5ed93b0c Mon Sep 17 00:00:00 2001 From: Jay Jaewon Yoo <39362841+JayJaewonYoo@users.noreply.github.com> Date: Tue, 3 Dec 2019 14:34:45 -0500 Subject: [PATCH 3/5] Store trained model in models for denoise_image function (#41) --- topaz/commands/denoise.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/topaz/commands/denoise.py b/topaz/commands/denoise.py index 7b1d40b..5a3d1f1 100644 --- a/topaz/commands/denoise.py +++ b/topaz/commands/denoise.py @@ -431,6 +431,8 @@ def main(args): torch.save(model, path) if use_cuda: model.cuda() + + models = [model] else: # load the saved model(s) models = [] From 6ca7cfd47b343d1f614ce16d4d13de633da4fc1e Mon Sep 17 00:00:00 2001 From: Tristan Bepler Date: Tue, 3 Dec 2019 14:46:22 -0500 Subject: [PATCH 4/5] updated denoise training to allow fraction of image pairs used for validation to be specified on the command line --- setup.py | 2 +- topaz/_version.py | 2 +- topaz/commands/denoise.py | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index b81d848..41ca3fa 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages name = 'topaz-em' -version = '0.2.3' +version = '0.2.4a' description = 'Particle picking with positive-unlabeled CNNs' long_description = 'Particle picking software for single particle cryo-electron microscopy using convolutional neural networks and positive-unlabeled learning. Includes methods for micrograph denoising.' diff --git a/topaz/_version.py b/topaz/_version.py index d31c31e..eb1b266 100644 --- a/topaz/_version.py +++ b/topaz/_version.py @@ -1 +1 @@ -__version__ = "0.2.3" +__version__ = "0.2.4a" diff --git a/topaz/commands/denoise.py b/topaz/commands/denoise.py index 5a3d1f1..86e2cd5 100644 --- a/topaz/commands/denoise.py +++ b/topaz/commands/denoise.py @@ -42,6 +42,7 @@ def add_arguments(parser): parser.add_argument('-b', '--dir-b', nargs='+', help='directory of training images part B') parser.add_argument('--hdf', help='path to HDF5 file containing training image stack as an alternative to dirA/dirB') parser.add_argument('--preload', action='store_true', help='preload micrographs into RAM') + parser.add_argument('--holdout', type=float, default=0.1, help='fraction of training micrograph pairs to holdout for validation (default: 0.1)') parser.add_argument('--lowpass', type=float, default=1, help='lowpass filter micrographs by this amount (in pixels) before applying the denoising filter. uses a hard lowpass filter (i.e. sinc) (default: no lowpass filtering)') parser.add_argument('--gaussian', type=float, default=0, help='Gaussian filter micrographs with this standard deviation (in pixels) before applying the denoising filter (default: 0)') @@ -76,7 +77,7 @@ def add_arguments(parser): from topaz.utils.image import save_image -def make_paired_images_datasets(dir_a, dir_b, crop, random=np.random, preload=False, cutoff=0): +def make_paired_images_datasets(dir_a, dir_b, crop, random=np.random, holdout=0.1, preload=False, cutoff=0): # train denoising model # make the dataset A = [] @@ -87,7 +88,7 @@ def make_paired_images_datasets(dir_a, dir_b, crop, random=np.random, preload=Fa B.append(dir_b + os.sep + name) # randomly hold out some image pairs for validation - n = int(0.1*len(A)) + n = int(holdout*len(A)) order = random.permutation(len(A)) A_train = [] @@ -110,7 +111,7 @@ def make_paired_images_datasets(dir_a, dir_b, crop, random=np.random, preload=Fa return dataset_train, dataset_val -def make_images_datasets(dir_a, dir_b, crop, random=np.random, cutoff=0): +def make_images_datasets(dir_a, dir_b, crop, random=np.random, holdout=0.1, cutoff=0): # train denoising model # make the dataset paths = [] @@ -122,7 +123,7 @@ def make_images_datasets(dir_a, dir_b, crop, random=np.random, cutoff=0): paths.append(path) # randomly hold out some image pairs for validation - n = int(0.1*len(paths)) + n = int(holdout*len(paths)) order = random.permutation(len(paths)) path_train = [] @@ -229,7 +230,7 @@ def __getitem__(self, i): # retrieve the i'th image pair return x -def make_hdf5_datasets(path, paired=True, preload=False, cutoff=0): +def make_hdf5_datasets(path, paired=True, preload=False, holdout=0.1, cutoff=0): # open the hdf5 dataset import h5py @@ -242,7 +243,7 @@ def make_hdf5_datasets(path, paired=True, preload=False, cutoff=0): N = len(dataset) # number of image pairs if paired: N = N//2 - n = int(0.1*N) + n = int(holdout*N) split = 2*(N-n) if paired: @@ -317,6 +318,7 @@ def main(args): method = args.method paired = (method == 'noise2noise') preload = args.preload + holdout = args.holdout # fraction of image pairs to holdout for validation if args.hdf is None: #use dirA/dirB crop = args.crop @@ -331,13 +333,15 @@ def main(args): if paired: dataset_train, dataset_val = make_paired_images_datasets(dir_a, dir_b, crop , random=random + , holdout=holdout , preload=preload , cutoff=cutoff ) else: dataset_train, dataset_val = make_images_datasets(dir_a, dir_b, crop , cutoff=cutoff - , random=random) + , random=random + , holdout=holdout) dset_train.append(dataset_train) dset_val.append(dataset_val) @@ -357,6 +361,7 @@ def main(args): else: # make HDF datasets dataset_train, dataset_val = make_hdf5_datasets(args.hdf, paired=paired , cutoff=cutoff + , holdout=holdout , preload=preload) shuffle = preload From 50edb44ffd2a3921be530331fc42e9865e8ddbd8 Mon Sep 17 00:00:00 2001 From: tbepler Date: Mon, 10 Feb 2020 12:15:17 -0500 Subject: [PATCH 5/5] added error message when topaz does not detect any training particles in the input --- topaz/commands/train.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/topaz/commands/train.py b/topaz/commands/train.py index 6cad0e8..6df8675 100644 --- a/topaz/commands/train.py +++ b/topaz/commands/train.py @@ -266,6 +266,10 @@ def load_data(train_images, train_targets, test_images, test_targets, radius num_micrographs = sum(len(train_images[k]) for k in train_images.keys()) num_particles = len(train_targets) report('Loaded {} training micrographs with {} labeled particles'.format(num_micrographs, num_particles)) + if num_particles == 0: + print('ERROR: no training particles specified. Check that micrograph names in the particles file match those in the micrographs file/directory.', file=sys.stderr) + raise Exception('No training particles.') + train_images, train_targets = match_images_targets(train_images, train_targets, radius)
    Input picks
    Input micrographs
    Output stack
    Output stack
    Boxsize
    Threshold
    Threshold
    Image extension
    Resize particles