Skip to content

Commit

Permalink
feat(twoslash): expose core module, independent from twoslash (#52)
Browse files Browse the repository at this point in the history
  • Loading branch information
antfu authored Dec 16, 2023
1 parent 02e386a commit 1c0d415
Show file tree
Hide file tree
Showing 10 changed files with 221 additions and 183 deletions.
6 changes: 6 additions & 0 deletions docs/.vitepress/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ export default defineConfig({
options.transformers?.splice(options.transformers.indexOf(cleanup), 1)
},
},
{
name: 'shikiji:remove-escape',
postprocess(code) {
return code.replace(/\[\\\!code/g, '[!code')
},
},
],
},

Expand Down
4 changes: 3 additions & 1 deletion docs/guide/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ outline: deep

No custom RegEx to maintain, no custom CSS to maintain, no custom HTML to maintain. And as your favorite languages and themes in VS Code evolve - your syntax highlighting will evolve too.

Shikiji is a ESM-rewrite of [Shiki](https://github.com/shikijs/shiki) with quite many improvements. We aim to [merge this project back to Shiki as a milestone update](https://github.com/shikijs/shiki/issues/510).
Shikiji is a ESM-rewrite of [Shiki](https://github.com/shikijs/shiki) with quite many improvements. We aim to [merge this project back to Shiki as a milestone update](https://github.com/shikijs/shiki/issues/510). [Breaking changes from Shiki and the compatibility build](/guide/compat) if you are migrating.

About the name, <ruby text-lg text-brand-yellow>式<rt>shiki</rt></ruby><ruby text-lg text-brand-red>辞<rt>ji</rt></ruby> is a Japanese word meaing ["Ceremonial Speech"](https://jisho.org/word/%E5%BC%8F%E8%BE%9E). <ruby text-brand-yellow text-lg>式<rt>shiki</rt></ruby> is inherited from [shiki](https://github.com/shikijs/shiki) means ["Style"](https://jisho.org/word/%E5%BC%8F) and <ruby text-brand-red text-lg>辞<rt>ji</rt></ruby> means ["Word"](https://jisho.org/word/%E8%BE%9E).

Expand All @@ -36,6 +36,8 @@ Here is a little playground for you to try out how Shikiji highlights your code.

<ShikijiMiniPlayground />

[Install Shikiji](/guide/install) to use it in your project.

## Who is using?

Projects that depend on Shikiji (sort alphabetically):
Expand Down
13 changes: 11 additions & 2 deletions docs/packages/transformers.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,24 @@ Use `[!code ++]` and `[!code --]` to mark added and removed lines.

For example, the following code

````md
```ts
export function foo() {
console.log('hewwo') // [!code --]
console.log('hello') // [!code ++]
console.log('hewwo') // [\!code --]
console.log('hello') // [\!code ++]
}
```
````

will be transformed to

```ts
export function foo() {
console.log('hewwo') // [!code --]
console.log('hello') // [!code ++]
}
```

```html
<!-- Output (stripped of `style` attributes for clarity) -->
<pre class="shiki has-diff"> <!-- Notice `has-diff` -->
Expand Down
1 change: 1 addition & 0 deletions packages/shikiji-twoslash/build.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { defineBuildConfig } from 'unbuild'
export default defineBuildConfig({
entries: [
'src/index.ts',
'src/core.ts',
],
declaration: true,
rollup: {
Expand Down
9 changes: 8 additions & 1 deletion packages/shikiji-twoslash/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@
"types": "./dist/index.d.mts",
"default": "./dist/index.mjs"
},
"./core": {
"types": "./dist/core.d.mts",
"default": "./dist/core.mjs"
},
"./style-rich.css": "./style-rich.css",
"./style-classic.css": "./style-classic.css",
"./*": "./dist/*"
Expand All @@ -31,6 +35,9 @@
"types": "./dist/index.d.mts",
"typesVersions": {
"*": {
"./core": [
"./dist/core.d.mts"
],
"*": [
"./dist/*",
"./*"
Expand All @@ -49,7 +56,7 @@
},
"dependencies": {
"@typescript/twoslash": "^3.2.4",
"shikiji": "workspace:*"
"shikiji-core": "workspace:*"
},
"devDependencies": {
"@iconify-json/carbon": "^1.1.26",
Expand Down
184 changes: 184 additions & 0 deletions packages/shikiji-twoslash/src/core.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
/**
* This file is the core of the shikiji-twoslash package,
* Decoupled from twoslash's implementation and allowing to introduce custom implementation or cache system.
*/
import type { twoslasher } from '@typescript/twoslash'
import type { ShikijiTransformer } from 'shikiji-core'
import type { Element, ElementContent, Text } from 'hast'
import type { ModuleKind, ScriptTarget } from 'typescript'

import { addClassToHast } from 'shikiji-core'
import { rendererClassic } from './renderer-classic'
import type { TransformerTwoSlashOptions } from './types'

export * from './types'
export * from './renderer-classic'
export * from './renderer-rich'
export * from './icons'

export function defaultTwoSlashOptions() {
return {
customTags: ['annotate', 'log', 'warn', 'error'],
defaultCompilerOptions: {
module: 99 satisfies ModuleKind.ESNext,
target: 99 satisfies ScriptTarget.ESNext,
},
}
}

export function createTransformer(runTwoslasher: typeof twoslasher) {
return function transformerTwoSlash(options: TransformerTwoSlashOptions = {}): ShikijiTransformer {
const {
langs = ['ts', 'tsx'],
twoslashOptions = defaultTwoSlashOptions(),
langAlias = {
typescript: 'ts',
json5: 'json',
yml: 'yaml',
},
explicitTrigger = false,
renderer = rendererClassic(),
throws = true,
} = options
const filter = options.filter || ((lang, _, options) => langs.includes(lang) && (!explicitTrigger || /\btwoslash\b/.test(options.meta?.__raw || '')))
return {
preprocess(code, shikijiOptions) {
let lang = shikijiOptions.lang
if (lang in langAlias)
lang = langAlias[shikijiOptions.lang]

if (filter(lang, code, shikijiOptions)) {
shikijiOptions.mergeWhitespaces = false
const twoslash = runTwoslasher(code, lang, twoslashOptions)
this.meta.twoslash = twoslash
return twoslash.code
}
},
pre(pre) {
if (this.meta.twoslash)
addClassToHast(pre, 'twoslash lsp')
},
code(codeEl) {
const twoslash = this.meta.twoslash
if (!twoslash)
return

const insertAfterLine = (line: number, nodes: ElementContent[]) => {
if (!nodes.length)
return
let index: number
if (line >= this.lines.length) {
index = codeEl.children.length
}
else {
const lineEl = this.lines[line]
index = codeEl.children.indexOf(lineEl)
if (index === -1) {
if (throws)
throw new Error(`[shikiji-twoslash] Cannot find line ${line} in code element`)
return
}
}

// If there is a newline after this line, remove it because we have the error element take place.
const nodeAfter = codeEl.children[index + 1]
if (nodeAfter && nodeAfter.type === 'text' && nodeAfter.value === '\n')
codeEl.children.splice(index + 1, 1)
codeEl.children.splice(index + 1, 0, ...nodes)
}

const locateTextToken = (
line: number,
character: number,
) => {
const lineEl = this.lines[line]
if (!lineEl) {
if (throws)
throw new Error(`[shikiji-twoslash] Cannot find line ${line} in code element`)
}
const textNodes = lineEl.children.flatMap(i => i.type === 'element' ? i.children || [] : []) as (Text | Element)[]
let index = 0
for (const token of textNodes) {
if ('value' in token && typeof token.value === 'string')
index += token.value.length

if (index > character)
return token
}
if (throws)
throw new Error(`[shikiji-twoslash] Cannot find token at L${line}:${character}`)
}

const skipTokens = new Set<Element | Text>()

for (const error of twoslash.errors) {
if (error.line == null || error.character == null)
return
const token = locateTextToken(error.line, error.character)
if (!token)
continue

skipTokens.add(token)

if (renderer.nodeError) {
const clone = { ...token }
Object.assign(token, renderer.nodeError.call(this, error, clone))
}

if (renderer.lineError)
insertAfterLine(error.line, renderer.lineError.call(this, error))
}

for (const query of twoslash.queries) {
if (query.kind === 'completions') {
const token = locateTextToken(query.line - 1, query.offset)
if (!token)
throw new Error(`[shikiji-twoslash] Cannot find token at L${query.line}:${query.offset}`)
skipTokens.add(token)

if (renderer.nodeCompletions) {
const clone = { ...token }
Object.assign(token, renderer.nodeCompletions.call(this, query, clone))
}

if (renderer.lineCompletions)
insertAfterLine(query.line, renderer.lineCompletions.call(this, query))
}
else if (query.kind === 'query') {
const token = locateTextToken(query.line - 1, query.offset)
if (!token)
throw new Error(`[shikiji-twoslash] Cannot find token at L${query.line}:${query.offset}`)

skipTokens.add(token)

if (renderer.nodeQuery) {
const clone = { ...token }
Object.assign(token, renderer.nodeQuery.call(this, query, clone))
}

if (renderer.lineQuery)
insertAfterLine(query.line, renderer.lineQuery.call(this, query, token))
}
}

for (const info of twoslash.staticQuickInfos) {
const token = locateTextToken(info.line, info.character)
if (!token || token.type !== 'text')
continue

// If it's already rendered as popup or error, skip it
if (skipTokens.has(token))
continue

const clone = { ...token }
Object.assign(token, renderer.nodeStaticInfo.call(this, info, clone))
}

if (renderer.lineCustomTag) {
for (const tag of twoslash.tags)
insertAfterLine(tag.line, renderer.lineCustomTag.call(this, tag))
}
},
}
}
}
2 changes: 1 addition & 1 deletion packages/shikiji-twoslash/src/icons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ import completionIcons from './icons-completions.json'
import tagIcons from './icons-tags.json'

export type CompletionItem = NonNullable<TwoSlashReturn['queries'][0]['completions']>[0]
export const defaultCompletionIcons: Record<CompletionItem['kind'], Element | undefined> = completionIcons as any

export const defaultCompletionIcons: Record<CompletionItem['kind'], Element | undefined> = completionIcons as any
export const defaultCustomTagIcons: Record<string, Element | undefined> = tagIcons as any
Loading

0 comments on commit 1c0d415

Please sign in to comment.