From 20faf17df0d82255b4e11c850bbe6b1acafd67b1 Mon Sep 17 00:00:00 2001 From: Jos Verlinde Date: Sun, 4 Aug 2024 22:23:22 +0200 Subject: [PATCH] stubber: Add --no-clean-rst option and notebook to copy stubs. Signed-off-by: Jos Verlinde --- prepare_source_stubs.ipynb | 411 +++++++++++++++++++++++ src/stubber/commands/get_docstubs_cmd.py | 4 +- src/stubber/rst/lookup.py | 13 + src/stubber/rst/reader.py | 44 +-- src/stubber/stubs_from_docs.py | 6 +- src/stubber/update_module_list.py | 4 +- 6 files changed, 451 insertions(+), 31 deletions(-) create mode 100644 prepare_source_stubs.ipynb diff --git a/prepare_source_stubs.ipynb b/prepare_source_stubs.ipynb new file mode 100644 index 00000000..387500c1 --- /dev/null +++ b/prepare_source_stubs.ipynb @@ -0,0 +1,411 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "21:23:01 | ℹ️ micropython-stubber 1.23.2a0\n", + "21:23:01 | ℹ️ fetch updates\n", + "21:23:03 | ℹ️ Switching to v1.23.0\n", + "21:23:07 | ℹ️ repos\\micropython v1.23.0\n", + "21:23:07 | ℹ️ repos\\micropython-lib v1.23.0\n", + "21:23:08 | ℹ️ repos\\micropython\\lib\\micropython-lib v1.23.0\n", + "21:23:08 | ℹ️ Get docstubs for MicroPython v1.23.0\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\array\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\asyncio\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\binascii\\__init__.py\n", + "i\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\bluetooth\\__init__.p\n", + "yi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\btree\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\cmath\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\collections\\__init__\n", + ".pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\cryptolib\\__init__.p\n", + "yi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\deflate\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\errno\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\esp\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\esp32\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\espnow\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\framebuf\\__init__.py\n", + "i\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\gc\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\gzip\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\hashlib\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\heapq\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\io\\__init__.pyi\n", + "21:23:08 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\json\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\lcd160cr\\__init__.py\n", + "i\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\ADC.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\ADCBlock.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\ADCWiPy.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\I2C.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\I2S.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\Pin.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\PWM.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\RTC.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\SD.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\SDCard.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\Signal.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\SPI.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\Timer.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\TimerWiPy.py\n", + "i\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\UART.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\USBDevice.py\n", + "i\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\machine\\WDT.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\math\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\micropython\\__init__\n", + ".pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\neopixel\\__init__.py\n", + "i\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\network\\LAN.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\network\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\network\\WIZNET5K.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\network\\WLAN.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\network\\WLANWiPy.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\openamp\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\os\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\platform\\__init__.py\n", + "i\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\Accel.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\ADC.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\CAN.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\DAC.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\ExtInt.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\Flash.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\I2C.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\LCD.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\LED.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\Pin.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\RTC.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\Servo.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\SPI.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\Switch.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\Timer.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\UART.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\USB_HID.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\pyb\\USB_VCP.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\random\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\rp2\\DMA.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\rp2\\Flash.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\rp2\\PIO.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\rp2\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\rp2\\StateMachine.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\select\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\socket\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\ssl\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\stm\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\struct\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\sys\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\time\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\uctypes\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\vfs\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\wipy\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\wm8960\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\zephyr\\DiskAccess.py\n", + "i\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\zephyr\\FlashArea.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\zephyr\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\zephyr\\zsensor.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\zlib\\__init__.pyi\n", + "21:23:09 | ℹ️ - Writing to: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\\_thread\\__init__.pyi\n", + "21:23:09 | ℹ️ ::group:: start post processing of retrieved stubs\n", + "21:23:14 | ℹ️ Running autoflake on: \n", + "repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs\n", + "21:23:16 | ℹ️ ::group:: Done\n" + ] + } + ], + "source": [ + "# create docstubs without cleaning up the .rst docstrings\n", + "!stubber docstubs --version stable --no-clean-rst\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# # merge the docstubs with boardstubs to get the non-documented methods and functions\n", + "# !stubber merge --version stable\n", + "# !stubber build --version stable" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copying array from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying asyncio from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying binascii from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying cmath from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying collections from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying errno from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying gc from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying gzip from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying hashlib from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying heapq from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying io from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying json from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying math from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying os from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying platform from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying random from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying select from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying socket from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying ssl from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying struct from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying sys from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying time from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying zlib from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying _thread from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying bluetooth from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying btree from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying cryptolib from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying deflate from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying framebuf from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying machine from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying micropython from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying neopixel from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying network from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying openamp from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying uctypes from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying vfs from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying wm8960 from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying pyb from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying stm from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying lcd160cr from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying wipy from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying esp from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying esp32 from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying espnow from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying rp2 from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n", + "Copying zephyr from D:\\mypython\\micropython-stubber\\repos\\micropython-stubs\\stubs\\micropython-v1_23_0-docstubs.\n" + ] + } + ], + "source": [ + "# copy the merged docstubs to the docs folder\n", + "from pathlib import Path\n", + "import shutil\n", + "\n", + "origins = [\n", + " # standard\n", + " (\"array\", \"\"),\n", + " (\"asyncio\", \"\"),\n", + " (\"binascii\", \"\"),\n", + " # builtins not stubbed - needs to be created manually\n", + " # (\"builtin\", \"\"),\n", + " (\"cmath\", \"\"),\n", + " (\"collections\", \"\"), # not on esp32\n", + " (\"errno\", \"\"),\n", + " (\"gc\", \"\"),\n", + " (\"gzip\", \"\"), # Not in stubs\n", + " (\"hashlib\", \"\"),\n", + " (\"heapq\", \"\"),\n", + " (\"io\", \"\"),\n", + " (\"json\", \"\"),\n", + " (\"math\", \"\"),\n", + " (\"os\", \"\"),\n", + " (\"platform\", \"\"),\n", + " (\"random\", \"\"),\n", + " # (\"re\", \"\"), # Not in stubs\n", + " (\"select\", \"\"),\n", + " (\"socket\", \"\"),\n", + " (\"ssl\", \"\"),\n", + " (\"struct\", \"\"),\n", + " (\"sys\", \"\"),\n", + " (\"time\", \"\"),\n", + " (\"zlib\", \"\"), # Not in stubs\n", + " (\"_thread\", \"\"),\n", + " # micropython specific\n", + " (\"bluetooth\", \"\"),\n", + " (\"btree\", \"\"),\n", + " (\"cryptolib\", \"\"),\n", + " (\"deflate\", \"\"),\n", + " (\"framebuf\", \"\"),\n", + " (\"machine\", \"\"),\n", + " (\"micropython\", \"\"),\n", + " (\"neopixel\", \"\"),\n", + " (\"network\", \"\"),\n", + " (\"openamp\", \"\"), # not in default stubs\n", + " (\"uctypes\", \"\"),\n", + " (\"vfs\", \"\"),\n", + " # drivers\n", + " (\"wm8960\", \"\"),\n", + " # stm32\n", + " (\"pyb\", \"publish/micropython-v1_23_0-stm32-stubs\"),\n", + " (\"stm\", \"publish/micropython-v1_23_0-stm32-stubs\"),\n", + " (\"lcd160cr\", \"publish/micropython-v1_23_0-stm32-stubs\"),\n", + " # Wipy\n", + " (\"wipy\", \"stubs/micropython-v1_23_0_preview-docstubs\"),\n", + " # (\"machine.ADCWiPy\", \"stubs/micropython-v1_23_0_preview-docstubs\"),\n", + " # (\"machine.TimerWiPy\", \"stubs/micropython-v1_23_0_preview-docstubs\"),\n", + " # esp\n", + " (\"esp\", \"publish/micropython-v1_23_0-esp32-stubs\"),\n", + " (\"esp32\", \"publish/micropython-v1_23_0-esp32-stubs\"),\n", + " (\"espnow\", \"publish/micropython-v1_23_0-esp32-stubs\"),\n", + " # rp2\n", + " (\"rp2\", \"publish/micropython-v1_23_0-rp2-stubs\"),\n", + " # Zephyr\n", + " (\"zephyr\", \"stubs/micropython-v1_23_0-docstubs\"),\n", + "]\n", + "\n", + "base_path = Path(\"D:\\\\mypython\\\\micropython-stubber\\\\repos\\\\micropython-stubs\")\n", + "target = Path(\"D:\\\\mypython\\\\autodoc201\\\\docs\\\\stubs\")\n", + "if target.exists():\n", + " shutil.rmtree(target)\n", + "target.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "for module, source in origins:\n", + " if not source:\n", + " # default\n", + " source = \"stubs/micropython-v1_23_0-docstubs\" # \"publish/micropython-v1_23_0-esp32-stubs\"\n", + " source = \"stubs/micropython-v1_23_0-docstubs\" # \"publish/micropython-v1_23_0-esp32-stubs\"\n", + " source_path = base_path / source\n", + " target_path = target\n", + " try:\n", + " print(f\"Copying {module} from {source_path}.\")\n", + " if (source_path / module).is_dir():\n", + " shutil.copytree(source_path / module, target_path / module, dirs_exist_ok=True)\n", + " else:\n", + " shutil.copy(source_path / f\"{module}.pyi\", target_path)\n", + " # print(f\"Copied {source}\")\n", + "\n", + " except FileExistsError:\n", + " print(f\"FileExistsError: {source_path} \")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/stubber/commands/get_docstubs_cmd.py b/src/stubber/commands/get_docstubs_cmd.py index acbf9fd1..ff10bf01 100644 --- a/src/stubber/commands/get_docstubs_cmd.py +++ b/src/stubber/commands/get_docstubs_cmd.py @@ -47,12 +47,14 @@ "--version", "--tag", default="", type=str, help="Version number to use. [default: Git tag]" ) @click.option("--black/--no-black", "-b/-nb", default=True, help="Run black", show_default=True) +@click.option("--clean-rst/--no-clean-rst", "-b/-nb", default=True, help="remove .rST constructs from the docstrings", show_default=True) @click.pass_context def cli_docstubs( ctx: click.Context, path: Optional[str] = None, target: Optional[str] = None, black: bool = True, + clean_rst: bool = True, basename: Optional[str] = None, version: str = "", ): @@ -88,5 +90,5 @@ def cli_docstubs( dst_path = Path(target) / f"{basename}-{utils.clean_version(version, flat=True)}-docstubs" log.info(f"Get docstubs for MicroPython {utils.clean_version(version, drop_v=False)}") - generate_from_rst(rst_path, dst_path, version, release=release, suffix=".pyi", black=black) + generate_from_rst(rst_path, dst_path, version, release=release, suffix=".pyi", black=black, clean_rst=clean_rst) log.info("::group:: Done") diff --git a/src/stubber/rst/lookup.py b/src/stubber/rst/lookup.py index 1214a36c..a13aee27 100644 --- a/src/stubber/rst/lookup.py +++ b/src/stubber/rst/lookup.py @@ -12,6 +12,7 @@ "NONE_VERBS", "CHILD_PARENT_CLASS", "PARAM_FIXES", + "PARAM_RE_FIXES", "MODULE_GLUE", "RST_DOC_FIXES", "DOCSTUB_SKIP", @@ -494,6 +495,18 @@ class Fix: ), ] +# and some param fixes that require a regex +PARAM_RE_FIXES = [ + Fix( + r"\[angle, time=0\]", "[angle], time=0", is_re=True + ), # fix: method:: Servo.angle([angle, time=0]) + Fix( + r"\[speed, time=0\]", "[speed], time=0", is_re=True + ), # fix: .. method:: Servo.speed([speed, time=0]) + Fix( + r"\[service_id, key=None, \*, \.\.\.\]", "[service_id], [key], *, ...", is_re=True + ), # fix: network - AbstractNIC.connect +] # List of classes and their parent classes that should be added to the class definition CHILD_PARENT_CLASS = { # machine diff --git a/src/stubber/rst/reader.py b/src/stubber/rst/reader.py index f852861c..e7b29427 100644 --- a/src/stubber/rst/reader.py +++ b/src/stubber/rst/reader.py @@ -73,6 +73,7 @@ CHILD_PARENT_CLASS, MODULE_GLUE, PARAM_FIXES, + PARAM_RE_FIXES, RST_DOC_FIXES, TYPING_IMPORT, ClassSourceDict, @@ -185,6 +186,7 @@ def __init__(self): self.current_module = "" self.current_class = "" self.current_function = "" # function & method + self.clean_rst = True super().__init__() def read_file(self, filename: Path): @@ -223,7 +225,7 @@ def at_anchor(self) -> bool: # return _l.startswith("..") and not any(_l.startswith(a) for a in self.docstring_anchors) # @property - def at_heading(self, large:bool=False) -> bool: + def at_heading(self, large: bool = False) -> bool: "stop at heading" u_line = self.rst_text[min(self.line_no + 1, self.max_line - 1)].rstrip() # Heading ---, ==, ~~~ @@ -294,16 +296,17 @@ def read_docstring(self, large: bool = False) -> List[str]: block[0] = block[0][0].upper() + block[0][1:] return block - @staticmethod - def clean_docstr(block: List[str]): + # @staticmethod + def clean_docstr(self, block: List[str]): """Clean up a docstring""" - # if a Quoted Literal Block , then remove the first character of each line - # https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks - if block and len(block[0]) > 0 and block[0][0] != " ": - q_char = block[0][0] - if all(l.startswith(q_char) for l in block): - # all lines start with the same character, so skip that character - block = [l[1:] for l in block] + if self.clean_rst: + # if a Quoted Literal Block , then remove the first character of each line + # https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks + if block and len(block[0]) > 0 and block[0][0] != " ": + q_char = block[0][0] + if all(l.startswith(q_char) for l in block): + # all lines start with the same character, so skip that character + block = [l[1:] for l in block] # rstrip all lines block = [l.rstrip() for l in block] # remove empty lines at beginning/end of block @@ -311,7 +314,6 @@ def clean_docstr(block: List[str]): block = block[1:] while len(block) and len(block[-1]) == 0: block = block[:-1] - # Clean up Synopsis if len(block) and ":synopsis:" in block[0]: block[0] = re.sub( @@ -321,9 +323,11 @@ def clean_docstr(block: List[str]): ) return block - @staticmethod - def add_link_to_docstr(block: List[str]): + def add_link_to_docstr(self, block: List[str]): """Add clickable hyperlinks to CPython docpages""" + if not self.clean_rst: + return block + for i in range(len(block)): # hyperlink to Cpython doc pages # https://regex101.com/r/5RN8rj/1 @@ -374,18 +378,6 @@ class RSTParser(RSTReader): """ target = ".py" # py/pyi - # TODO: Move to lookup.py - PARAM_RE_FIXES = [ - Fix( - r"\[angle, time=0\]", "[angle], time=0", is_re=True - ), # fix: method:: Servo.angle([angle, time=0]) - Fix( - r"\[speed, time=0\]", "[speed], time=0", is_re=True - ), # fix: .. method:: Servo.speed([speed, time=0]) - Fix( - r"\[service_id, key=None, \*, \.\.\.\]", "[service_id], [key], *, ...", is_re=True - ), # fix: network - AbstractNIC.connect - ] def __init__(self, v_tag: str) -> None: super().__init__() @@ -414,7 +406,7 @@ def fix_parameters(self, params: str, name: str = "") -> str: ## Deal with SQUARE brackets first ( Documentation meaning := [optional]) - for fix in self.PARAM_RE_FIXES: + for fix in PARAM_RE_FIXES: params = self.apply_fix(fix, params, name) # ########################################################################################################### diff --git a/src/stubber/stubs_from_docs.py b/src/stubber/stubs_from_docs.py index c63add11..dbe604d7 100644 --- a/src/stubber/stubs_from_docs.py +++ b/src/stubber/stubs_from_docs.py @@ -22,6 +22,7 @@ def generate_from_rst( pattern: str = "*.rst", suffix: str = ".pyi", black: bool = True, + clean_rst: bool = True, ) -> int: dst_path.mkdir(parents=True, exist_ok=True) if not release: @@ -36,7 +37,7 @@ def generate_from_rst( # files = [f for f in files if "machine" in f.name] clean_destination(dst_path) - make_docstubs(dst_path, v_tag, release, suffix, files) + make_docstubs(dst_path, v_tag, release, suffix, files, clean_rst=clean_rst) log.info("::group:: start post processing of retrieved stubs") # do not run stubgen @@ -72,11 +73,12 @@ def get_rst_sources(rst_path: Path, pattern: str) -> List[Path]: return files -def make_docstubs(dst_path: Path, v_tag: str, release: str, suffix: str, files: List[Path]): +def make_docstubs(dst_path: Path, v_tag: str, release: str, suffix: str, files: List[Path],clean_rst:bool): """Create the docstubs""" for file in files: reader = RSTWriter(v_tag) + reader.clean_rst = clean_rst reader.source_release = release log.debug(f"Reading: {file}") reader.read_file(file) diff --git a/src/stubber/update_module_list.py b/src/stubber/update_module_list.py index c7cef5c0..4f694fe6 100644 --- a/src/stubber/update_module_list.py +++ b/src/stubber/update_module_list.py @@ -23,7 +23,7 @@ def read_modules(path: Optional[Path] = None) -> Set[str]: read text files with modules per firmware. each contains the output of help("modules") - lines starting with # are comments. - - split the other lines at whitespace seperator, + - split the other lines at whitespace separator, - and add each module to a set """ path = Path(path or "./data") @@ -73,7 +73,7 @@ def update_module_list(): "upip_utarfile", "upysh", "uasyncio", - "builtins", + # "builtins", # may be used for type hints "re", } log.info("Update the module list in createstubs.py")