From 4b2a8e9335df2a995c65c32ea700822638a8061b Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 28 Nov 2024 08:32:14 -0800 Subject: [PATCH 1/2] docs --- CHANGELOG.md | 8 +++ CONTRIBUTING.md | 117 +++++++++++++++++++++++++++++++++++++ Docs/FEATURES.md | 4 ++ Docs/Handy_Dandy_Papers.md | 3 + Docs/Issues/ISSUES.md | 13 ++++- 5 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 Docs/FEATURES.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..cf88a547 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# [Changelog](https://keepachangelog.com) + +## 0.1.0 (2025-xx-xx) + +- Initial release + +## Prior changelog from Gist + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..8e2bc60b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,117 @@ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every +little bit helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +### Report Bugs + +Report bugs at . + +If you are reporting a bug, please include: + +- Your operating system name and version. +- Any details about your local setup that might be helpful in + troubleshooting. +- Detailed steps to reproduce the bug. + +### Fix Bugs + +Look through the GitHub issues for bugs. Anything tagged with "bug" or "help wanted" is +open to whoever wants to implement it. + +### Implement Features + +Look through the GitHub issues for features. Anything tagged with +"help wanted", "enhancement" or "Feature-Addition" is open to whoever wants to implement it. + +### Write Documentation + +tldw could definitely use more documentation, +whether as part of the official tldw docs, +in docstrings, or even on the web in blog posts, articles, and such. + +### Submit Feedback + +The best way to send feedback is to file an issue at +. + +If you are proposing a feature: + +- Explain in detail how it would work. +- Keep the scope as narrow as possible, to make it easier to + implement. +- Remember that this is a volunteer-driven project, and that + contributions are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up tldw +for local development. + +1. Fork the tldw repo on GitHub. +1. Clone your fork locally: + + $ git clone git@github.com:your_name_here/tldw.git + $ cd tldw + +1. Check out the `dev` branch, where development happens prior to being merged + into `main`. Your changes should be based on the `dev` branch, and your PR + should eventually be requested against my `dev` branch. + + $ git checkout dev + +1. Install your local copy into a virtualenv (`venv` in modern python). Some + linux distributions will require you to install `python-venv` or + `python3-venv`, other times it will already be bundled with python. There + are many ways to skin a cat, but this is how I usually set up a fork for + local development: + + $ python3 -m venv .venv # set up hidden virtualenv folder: .venv + $ source ./.venv/bin/actiate # activate virtualenv + $ which python + /Users/me/tldw/.venv/bin/python + $ python -m pip install requirements.txt + +1. Create a branch for local development: + + $ git checkout -b name-of-your-bugfix-or-feature # or use e.g. issue_13 + + Now you can make your changes locally. + +1. (WIP, for now, just use pytest ./Tests/ or Github Actions)When you're done making changes, check that your changes pass flake8 + and the tests, including testing other Python versions with tox: + + $ tox + +1. Commit your changes and push your branch to GitHub: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +1. Submit a pull request through the GitHub website against my `dev` branch. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in Docs/FEATURES.md +3. The pull request should work for all Python versions that this project + tests against with tox. Tests are ran automatically using Github Actions. + +Tips +---- + +To run a subset of tests: `pytest Tests/test_your_test.py` + diff --git a/Docs/FEATURES.md b/Docs/FEATURES.md new file mode 100644 index 00000000..0f016b5d --- /dev/null +++ b/Docs/FEATURES.md @@ -0,0 +1,4 @@ +# Features of tldw + + + diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 17ad05c6..ba35c948 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -15,6 +15,9 @@ ### Personalization https://arxiv.org/abs/2411.16034 +### Quantization + https://arxiv.org/abs/2411.17691 + ### RAG https://arxiv.org/html/2407.21059v1 diff --git a/Docs/Issues/ISSUES.md b/Docs/Issues/ISSUES.md index a8365e9f..f4004f1a 100644 --- a/Docs/Issues/ISSUES.md +++ b/Docs/Issues/ISSUES.md @@ -5,6 +5,15 @@ 2. cudnn8dlops.dll or whatever is missing/not in PATH * https://stackoverflow.com/questions/66083545/could-not-load-dynamic-library-cudnn64-8-dll-dlerror-cudnn64-8-dll-not-found + +Inspo + https://github.com/hoarder-app/hoarder + +Testing + https://tox.wiki/en/4.23.2/ + https://flake8.pycqa.org/en/latest/ + + Create a blog post tldwproject.com - Done Make a nicer homepage - https://vitepress.dev/guide/getting-started @@ -56,7 +65,6 @@ PDF Parsing https://pub.towardsai.net/demystifying-pdf-parsing-05-unifying-separate-tasks-into-a-small-model-d3739db021f7 https://ai.gopubby.com/demystifying-pdf-parsing-06-representative-industry-solutions-5d4a1cfe311b - Import HTML Files Manga @@ -79,6 +87,9 @@ CHM: https://github.com/dottedmag/archmage +SQLite studio + https://github.com/pawelsalawa/sqlitestudio + URL Shortener https://realpython.com/fastapi-python-web-apis/ From 88d44b15f2eeb8a894e4d726f38670969ab46f72 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 28 Nov 2024 09:27:21 -0800 Subject: [PATCH 2/2] fixes --- CONTRIBUTING.md | 6 ++++-- summarize.py | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8e2bc60b..6646428b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,10 +85,12 @@ for local development. Now you can make your changes locally. -1. (WIP, for now, just use pytest ./Tests/ or Github Actions)When you're done making changes, check that your changes pass flake8 +1. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox: - $ tox + $ flake8 App_Function_Libraries/.py + $ pytest Tests/test_.py + $ tox (WIP) 1. Commit your changes and push your branch to GitHub: diff --git a/summarize.py b/summarize.py index dca85ba6..1941ad8f 100644 --- a/summarize.py +++ b/summarize.py @@ -547,7 +547,7 @@ def main(input_path, api_name=None, api_key=None, transcription_text = {'audio_file': audio_file, 'transcription': segments} # FIXME rolling summarization - if rolling_summarization == True: + if rolling_summarization: pass # text = extract_text_from_segments(segments) # detail = detail_level @@ -572,7 +572,6 @@ def main(input_path, api_name=None, api_key=None, # max_tokens, summarize_recursively, verbose # ) - elif api_name: summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) else: @@ -718,8 +717,8 @@ def main(input_path, api_name=None, api_key=None, # FIXME if rolling_summarization: - # text = extract_text_from_segments(segments) - # summary = summarize_with_detail_openai(text, detail=detail) + #text = extract_text_from_segments(segments) + #summary = summarize_with_detail_openai(text, detail=detail) pass elif api_name: summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) @@ -751,12 +750,13 @@ def signal_handler(sig, frame): logging.debug(f"Active threads after shutdown: {threading.enumerate()}") sys.exit(0) + signal.signal(signal.SIGINT, signal_handler) -############################## MAIN ############################## -# +#################################################################################################################### # +# MAIN if __name__ == "__main__": # Register signal handlers @@ -1014,9 +1014,14 @@ def signal_handler(sig, frame): logging.error('An error occurred during the transcription process.') logging.error(str(e)) sys.exit(1) + def cleanup(): logging.info("Cleanup function called. Script is exiting.") atexit.register(cleanup) # Register the cleanup function to run on exit - atexit.register(cleanup_downloads) \ No newline at end of file + atexit.register(cleanup_downloads) + +# +# End of summarize.py +#######################################################################################################################