forked from AllYourBot/hostedgpt
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Voice improvements: auto-dismiss, better reinvoking, better speaking …
…of markdown (AllYourBot#413)
Showing
12 changed files
with
346 additions
and
246 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,43 +1,48 @@ | ||
| require "./lib/redcarpet/custom_html_renderer" | ||
| Dir[File.join(File.dirname(__FILE__), "redcarpet", "*.rb")].each { |file| require file } | ||
|
|
||
| class MarkdownRenderer | ||
| class << self | ||
| def render_for_speaking(markdown) | ||
| Redcarpet::CustomSpeakingRenderer.new.render(markdown.to_s) | ||
| end | ||
|
|
||
| def self.render(markdown, options = {}) | ||
| markdown ||= "" | ||
|
|
||
| render_class = Redcarpet::CustomHtmlRenderer | ||
| def render_for_display(markdown, options = {}) | ||
| renderer = create_renderer_for_display(options) | ||
| formatter = Redcarpet::Markdown.new(renderer, | ||
| autolink: true, | ||
| tables: true, | ||
| space_after_headers: true, | ||
| strikethrough: true, | ||
| underline: true, | ||
| no_intra_emphasis: true, | ||
| fenced_code_blocks: true, | ||
| disable_indented_code_blocks: true | ||
| ) | ||
|
|
||
| markdown = ensure_blank_line_before_code_block_start(markdown) | ||
| formatter.render(markdown) | ||
| end | ||
|
|
||
| block_code_proc = options.delete(:block_code) | ||
| private | ||
|
|
||
| if block_code_proc | ||
| def create_renderer_for_display(options) | ||
| render_class = Redcarpet::CustomDisplayRenderer | ||
|
|
||
| render_class = Class.new(Redcarpet::CustomHtmlRenderer) | ||
| render_class.instance_eval do | ||
| define_method(:block_code) do |code, language| | ||
| block_code_proc.call(code.html_safe, language) | ||
| block_code_proc = options.delete(:block_code) | ||
| if block_code_proc | ||
| render_class = Class.new(Redcarpet::CustomDisplayRenderer) | ||
| render_class.instance_eval do | ||
| define_method(:block_code) do |code, language| | ||
| block_code_proc.call(code.html_safe, language) | ||
| end | ||
| end | ||
| end | ||
| end | ||
|
|
||
| renderer = render_class.new(safe_links_only: true) | ||
|
|
||
| formatter = Redcarpet::Markdown.new(renderer, | ||
| autolink: true, | ||
| tables: true, | ||
| space_after_headers: true, | ||
| strikethrough: true, | ||
| underline: true, | ||
| no_intra_emphasis: true, | ||
| fenced_code_blocks: true, | ||
| disable_indented_code_blocks: true | ||
| ) | ||
|
|
||
| markdown = ensure_blank_line_before_code_block_start(markdown) | ||
|
|
||
| formatter.render(markdown) | ||
| end | ||
| render_class.new(safe_links_only: true) | ||
| end | ||
|
|
||
| def self.ensure_blank_line_before_code_block_start(markdown) | ||
| markdown.gsub(/(\n*)( *)(```.*?```)/m, "\n\n\\2\\3") | ||
| def ensure_blank_line_before_code_block_start(markdown) | ||
| markdown.to_s.gsub(/(\n*)( *)(```.*?```)/m, "\n\n\\2\\3") | ||
| end | ||
| end | ||
| end |
2 changes: 1 addition & 1 deletion
2
lib/redcarpet/custom_html_renderer.rb → lib/redcarpet/custom_display_renderer.rb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| class Redcarpet::CustomSpeakingRenderer # can't get redcarpet working, just using regex | ||
| def render(markdown) | ||
| not_needing_suffix = %w[javascript jsx typescript css html json markdown yaml] | ||
|
|
||
| markdown.gsub(/```(\w+)?\s*(.*?)```/m) do |match| # paired ```` | ||
| if $1.to_s.strip.in? not_needing_suffix | ||
| "Here is some #{$1}." | ||
| else | ||
| "Here is some #{$1.to_s + ($1.present? ? " " : "")}code." | ||
| end | ||
| end.gsub(/```.*$/m, '') # opening ``` without closing` | ||
| .gsub(/\[([^\]]+)\]\(([^)]+)\)/, "Here is a link to \\1") # markdown links | ||
| .gsub(/http[s]?:\/\/[^\s]+/, "this link") # naked links | ||
| end | ||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,127 +1,5 @@ | ||
| require 'test_helper' | ||
|
|
||
| class MarkdownRendererTest < ActiveSupport::TestCase | ||
| setup do | ||
| @renderer = MarkdownRenderer | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start adds blank line before code block when zero newlines" do | ||
| markdown = "Text before code block```ruby\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
|
|
||
| markdown = "```ruby\ncode block\n```" | ||
| expected = "\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
|
|
||
| markdown = "Text before code block```ruby\ncode block\n```Text before second```\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start adds blank line before code block when one newline" do | ||
| markdown = "Text before code block\n```ruby\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
|
|
||
| markdown = "\n```ruby\ncode block\n```" | ||
| expected = "\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
|
|
||
| markdown = "Text before code block\n```ruby\ncode block\n```Text before second\n```\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start does not add blank line when one is already present" do | ||
| markdown = "Text before code block\n\n```ruby\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
|
|
||
| markdown = "\n\n```ruby\ncode block\n```" | ||
| expected = "\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
|
|
||
| markdown = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start keeps a block indented if it is indented" do | ||
| # TODO: Notably, if a block is indented right after a bullet such that it's intended to be included within the bullet, | ||
| # redcarpet is not including the block within the bullet: https://github.com/allyourbot/hostedgpt/issues/323 | ||
|
|
||
| markdown = "Text before code block\n\n ``` ruby\n code block\n ```" | ||
| expected = "Text before code block\n\n ``` ruby\n code block\n ```" | ||
| assert_equal expected, MarkdownRenderer.ensure_blank_line_before_code_block_start(markdown) | ||
| end | ||
|
|
||
| test "block_code nicely formatted gets converted" do | ||
| markdown = <<~MD | ||
| This is sql: | ||
| ```sql | ||
| SELECT * FROM users; | ||
| ``` | ||
| Line after. | ||
| MD | ||
|
|
||
| formatted = <<~HTML | ||
| <p>This is sql:</p> | ||
| <pre><code class="sql">SELECT * FROM users; | ||
| </code></pre> | ||
| <p>Line after.</p> | ||
| HTML | ||
|
|
||
| assert_equal formatted.strip, @renderer.render(markdown).strip | ||
| end | ||
|
|
||
| test "block_code missing a blank line before and after gets gets nicely - ensure_blank_line_before_code_block_start" do | ||
| markdown = <<~MD | ||
| This is sql: | ||
| ```sql | ||
| SELECT * FROM users; | ||
| ``` | ||
| Line after. | ||
| MD | ||
|
|
||
| formatted = <<~HTML | ||
| <p>This is sql:</p> | ||
| <pre><code class="sql">SELECT * FROM users; | ||
| </code></pre> | ||
| <p>Line after.</p> | ||
| HTML | ||
|
|
||
| assert_equal formatted.strip, @renderer.render(markdown).strip | ||
| end | ||
|
|
||
| test "block_code can be provided with a proc" do | ||
| markdown = <<~MD | ||
| This is sql: | ||
| ```sql | ||
| SELECT * FROM users; | ||
| ``` | ||
| Line after. | ||
| MD | ||
|
|
||
| formatted = <<~HTML | ||
| <p>This is sql:</p> | ||
| <CODE lang="sql">SELECT * FROM users; | ||
| </CODE> | ||
| <p>Line after.</p> | ||
| HTML | ||
|
|
||
| block_code = -> (code, language) do | ||
| %(<CODE lang="#{language}">#{code}</CODE>) | ||
| end | ||
|
|
||
| assert_equal formatted.strip, @renderer.render(markdown, block_code: block_code).strip | ||
| end | ||
| # See lib/redcarpet/* tests | ||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| require "test_helper" | ||
|
|
||
| class CustomDisplayRendererTest < ActiveSupport::TestCase | ||
| setup do | ||
| @renderer = MarkdownRenderer | ||
| end | ||
|
|
||
| test "code_span" do | ||
| markdown = "This is `code` inline." | ||
| formatted = "<p>This is <code>code</code> inline.</p>\n" | ||
|
|
||
| assert_equal formatted.strip, @renderer.render_for_display(markdown).strip | ||
| end | ||
|
|
||
| test "newlines within paragraphs get converted into BRs" do | ||
| markdown = <<~MD | ||
| This is the first paragraph | ||
| This is a second paragraph | ||
| with a line break. | ||
| This is a third paragraph | ||
| MD | ||
|
|
||
| formatted = <<~HTML | ||
| <p>This is the first paragraph</p> | ||
| <p>This is a second paragraph<br> | ||
| with a line break.</p> | ||
| <p>This is a third paragraph</p> | ||
| HTML | ||
|
|
||
| assert_equal formatted.strip, @renderer.render_for_display(markdown).strip | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start adds blank line before code block when zero newlines" do | ||
| markdown = "Text before code block```ruby\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
|
|
||
| markdown = "```ruby\ncode block\n```" | ||
| expected = "\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
|
|
||
| markdown = "Text before code block```ruby\ncode block\n```Text before second```\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start adds blank line before code block when one newline" do | ||
| markdown = "Text before code block\n```ruby\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
|
|
||
| markdown = "\n```ruby\ncode block\n```" | ||
| expected = "\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
|
|
||
| markdown = "Text before code block\n```ruby\ncode block\n```Text before second\n```\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start does not add blank line when one is already present" do | ||
| markdown = "Text before code block\n\n```ruby\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
|
|
||
| markdown = "\n\n```ruby\ncode block\n```" | ||
| expected = "\n\n```ruby\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
|
|
||
| markdown = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| expected = "Text before code block\n\n```ruby\ncode block\n```Text before second\n\n```\ncode block\n```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
| end | ||
|
|
||
| test "ensure_blank_line_before_code_block_start keeps a block indented if it is indented" do | ||
| # TODO: Notably, if a block is indented right after a bullet such that it's intended to be included within the bullet, | ||
| # redcarpet is not including the block within the bullet: https://github.com/allyourbot/hostedgpt/issues/323 | ||
|
|
||
| markdown = "Text before code block\n\n ``` ruby\n code block\n ```" | ||
| expected = "Text before code block\n\n ``` ruby\n code block\n ```" | ||
| assert_equal expected, MarkdownRenderer.send(:ensure_blank_line_before_code_block_start, markdown) | ||
| end | ||
|
|
||
| test "block_code nicely formatted gets converted" do | ||
| markdown = <<~MD | ||
| This is sql: | ||
| ```sql | ||
| SELECT * FROM users; | ||
| ``` | ||
| Line after. | ||
| MD | ||
|
|
||
| formatted = <<~HTML | ||
| <p>This is sql:</p> | ||
| <pre><code class="sql">SELECT * FROM users; | ||
| </code></pre> | ||
| <p>Line after.</p> | ||
| HTML | ||
|
|
||
| assert_equal formatted.strip, @renderer.render_for_display(markdown).strip | ||
| end | ||
|
|
||
| test "block_code missing a blank line before and after gets gets nicely - ensure_blank_line_before_code_block_start" do | ||
| markdown = <<~MD | ||
| This is sql: | ||
| ```sql | ||
| SELECT * FROM users; | ||
| ``` | ||
| Line after. | ||
| MD | ||
|
|
||
| formatted = <<~HTML | ||
| <p>This is sql:</p> | ||
| <pre><code class="sql">SELECT * FROM users; | ||
| </code></pre> | ||
| <p>Line after.</p> | ||
| HTML | ||
|
|
||
| assert_equal formatted.strip, @renderer.render_for_display(markdown).strip | ||
| end | ||
|
|
||
| test "block_code can be provided with a proc" do | ||
| markdown = <<~MD | ||
| This is sql: | ||
| ```sql | ||
| SELECT * FROM users; | ||
| ``` | ||
| Line after. | ||
| MD | ||
|
|
||
| formatted = <<~HTML | ||
| <p>This is sql:</p> | ||
| <CODE lang="sql">SELECT * FROM users; | ||
| </CODE> | ||
| <p>Line after.</p> | ||
| HTML | ||
|
|
||
| block_code = -> (code, language) do | ||
| %(<CODE lang="#{language}">#{code}</CODE>) | ||
| end | ||
|
|
||
| assert_equal formatted.strip, @renderer.render_for_display(markdown, block_code: block_code).strip | ||
| end | ||
| end |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| require "test_helper" | ||
|
|
||
| class CustomSpeakingRendererTest < ActiveSupport::TestCase | ||
| setup do | ||
| @renderer = MarkdownRenderer | ||
| end | ||
|
|
||
| test "code_span" do | ||
| markdown = "This is `code` inline. This is a bullet:\n\n* Bullet" | ||
| formatted = "This is `code` inline. This is a bullet:\n\n* Bullet" | ||
| assert_equal formatted, @renderer.render_for_speaking(markdown) | ||
| end | ||
|
|
||
| test "block_code with a language that gets the word 'code'" do | ||
| markdown = "This is an example:\n```ruby\ncode\n```\n" | ||
| formatted = "This is an example:\nHere is some ruby code.\n" | ||
| assert_equal formatted, @renderer.render_for_speaking(markdown) | ||
| end | ||
|
|
||
| test "block_code with a language that stands on its own'" do | ||
| markdown = "This is an example:\n```html\ncode\n```\n" | ||
| formatted = "This is an example:\nHere is some html.\n" | ||
| assert_equal formatted, @renderer.render_for_speaking(markdown) | ||
| end | ||
|
|
||
| test "block_code with no language rewrites properly" do | ||
| markdown = "This is an example:\n```\ncode\n```\n" | ||
| formatted = "This is an example:\nHere is some code.\n" | ||
| assert_equal formatted, @renderer.render_for_speaking(markdown) | ||
| end | ||
|
|
||
| test "block_code that is incomplete is hidden, because we are parsing a partially streamed response" do | ||
| markdown = "This is an example:\n```\nthis is a partially completed response" | ||
| formatted = "This is an example:\n" | ||
| assert_equal formatted, @renderer.render_for_speaking(markdown) | ||
| end | ||
|
|
||
| test "markdown links are caught" do | ||
| markdown = "Try visiting [Google](https://google.com)" | ||
| formatted = "Try visiting Here is a link to Google" | ||
| assert_equal formatted, @renderer.render_for_speaking(markdown) | ||
| end | ||
|
|
||
| test "regular URLs are caught" do | ||
| markdown = "Try visiting https://google.com" | ||
| formatted = "Try visiting this link" | ||
| assert_equal formatted, @renderer.render_for_speaking(markdown) | ||
| end | ||
| end |