Extract isomorphic @getpaseo/highlight package with syntax-highlighted file preview
- Create packages/highlight with shared Lezer-based syntax highlighting (types, parsers, highlighter, color maps) - Add syntax highlighting and line numbers to file-pane.tsx with memoized CodeLine component - Import shared highlight colors in git-diff-pane.tsx instead of local definitions - Delete duplicated syntax-highlighter.ts from both app and server - Fix TSX dialect to "ts jsx" (correct per Lezer docs)
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,65 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { darkHighlightColors, lightHighlightColors } from '../colors.js'
|
||||
import type { HighlightStyle } from '../types.js'
|
||||
|
||||
const allStyles: HighlightStyle[] = [
|
||||
'keyword',
|
||||
'comment',
|
||||
'string',
|
||||
'number',
|
||||
'literal',
|
||||
'function',
|
||||
'definition',
|
||||
'class',
|
||||
'type',
|
||||
'tag',
|
||||
'attribute',
|
||||
'property',
|
||||
'variable',
|
||||
'operator',
|
||||
'punctuation',
|
||||
'regexp',
|
||||
'escape',
|
||||
'meta',
|
||||
'heading',
|
||||
'link',
|
||||
]
|
||||
|
||||
describe('darkHighlightColors', () => {
|
||||
it('covers all HighlightStyle values', () => {
|
||||
for (const style of allStyles) {
|
||||
expect(darkHighlightColors[style]).toBeDefined()
|
||||
expect(typeof darkHighlightColors[style]).toBe('string')
|
||||
}
|
||||
})
|
||||
|
||||
it('has valid hex color values', () => {
|
||||
for (const style of allStyles) {
|
||||
expect(darkHighlightColors[style]).toMatch(/^#[0-9a-fA-F]{6}$/)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('lightHighlightColors', () => {
|
||||
it('covers all HighlightStyle values', () => {
|
||||
for (const style of allStyles) {
|
||||
expect(lightHighlightColors[style]).toBeDefined()
|
||||
expect(typeof lightHighlightColors[style]).toBe('string')
|
||||
}
|
||||
})
|
||||
|
||||
it('has valid hex color values', () => {
|
||||
for (const style of allStyles) {
|
||||
expect(lightHighlightColors[style]).toMatch(/^#[0-9a-fA-F]{6}$/)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('color map completeness', () => {
|
||||
it('dark and light maps have the same keys', () => {
|
||||
const darkKeys = Object.keys(darkHighlightColors).sort()
|
||||
const lightKeys = Object.keys(lightHighlightColors).sort()
|
||||
|
||||
expect(darkKeys).toEqual(lightKeys)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,125 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { highlightCode, highlightLine } from '../highlighter.js'
|
||||
|
||||
describe('highlightCode', () => {
|
||||
it('highlights JavaScript code with correct token styles', () => {
|
||||
const code = 'const x = 42;'
|
||||
const result = highlightCode(code, 'test.js')
|
||||
|
||||
expect(result).toHaveLength(1)
|
||||
const tokens = result[0]
|
||||
expect(tokens.length).toBeGreaterThan(1)
|
||||
|
||||
const keywordToken = tokens.find((t) => t.text === 'const')
|
||||
expect(keywordToken?.style).toBe('keyword')
|
||||
|
||||
const numberToken = tokens.find((t) => t.text === '42')
|
||||
expect(numberToken?.style).toBe('number')
|
||||
})
|
||||
|
||||
it('highlights Python code', () => {
|
||||
const code = 'def hello():\n print("world")'
|
||||
const result = highlightCode(code, 'test.py')
|
||||
|
||||
expect(result).toHaveLength(2)
|
||||
|
||||
const defToken = result[0].find((t) => t.text === 'def')
|
||||
expect(defToken?.style).toBe('keyword')
|
||||
|
||||
const stringToken = result[1].find((t) => t.text.includes('world'))
|
||||
expect(stringToken?.style).toBe('string')
|
||||
})
|
||||
|
||||
it('highlights TSX code with correct dialect', () => {
|
||||
const code = 'const el = <div className="test">hello</div>;'
|
||||
const result = highlightCode(code, 'test.tsx')
|
||||
|
||||
expect(result).toHaveLength(1)
|
||||
const tokens = result[0]
|
||||
|
||||
const constToken = tokens.find((t) => t.text === 'const')
|
||||
expect(constToken?.style).toBe('keyword')
|
||||
|
||||
const tagToken = tokens.find((t) => t.text === 'div')
|
||||
expect(tagToken).toBeDefined()
|
||||
})
|
||||
|
||||
it('returns unhighlighted tokens for unsupported extensions', () => {
|
||||
const code = 'hello world\nsecond line'
|
||||
const result = highlightCode(code, 'test.xyz')
|
||||
|
||||
expect(result).toHaveLength(2)
|
||||
expect(result[0]).toEqual([{ text: 'hello world', style: null }])
|
||||
expect(result[1]).toEqual([{ text: 'second line', style: null }])
|
||||
})
|
||||
|
||||
it('splits multi-line code into separate line arrays', () => {
|
||||
const code = 'line1\nline2\nline3'
|
||||
const result = highlightCode(code, 'test.txt')
|
||||
|
||||
expect(result).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('handles empty lines', () => {
|
||||
const code = 'const a = 1;\n\nconst b = 2;'
|
||||
const result = highlightCode(code, 'test.js')
|
||||
|
||||
expect(result).toHaveLength(3)
|
||||
expect(result[1]).toEqual([{ text: '', style: null }])
|
||||
})
|
||||
|
||||
it('returns a single empty line for empty input', () => {
|
||||
expect(highlightCode('', 'test.ts')).toEqual([[{ text: '', style: null }]])
|
||||
})
|
||||
|
||||
it('returns valid HighlightStyle values for all tokens', () => {
|
||||
const validStyles = new Set([
|
||||
'keyword',
|
||||
'comment',
|
||||
'string',
|
||||
'number',
|
||||
'literal',
|
||||
'function',
|
||||
'definition',
|
||||
'class',
|
||||
'type',
|
||||
'tag',
|
||||
'attribute',
|
||||
'property',
|
||||
'variable',
|
||||
'operator',
|
||||
'punctuation',
|
||||
'regexp',
|
||||
'escape',
|
||||
'meta',
|
||||
'heading',
|
||||
'link',
|
||||
null,
|
||||
])
|
||||
|
||||
const code = '// comment\nconst x: number = 42;\nfunction foo() { return "bar"; }'
|
||||
const result = highlightCode(code, 'test.ts')
|
||||
|
||||
for (const line of result) {
|
||||
for (const token of line) {
|
||||
expect(validStyles.has(token.style)).toBe(true)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('highlightLine', () => {
|
||||
it('highlights a single line', () => {
|
||||
const tokens = highlightLine('const x = 1;', 'test.js')
|
||||
|
||||
expect(tokens.length).toBeGreaterThan(1)
|
||||
const keywordToken = tokens.find((t) => t.text === 'const')
|
||||
expect(keywordToken?.style).toBe('keyword')
|
||||
})
|
||||
|
||||
it('returns unhighlighted token for unsupported files', () => {
|
||||
const tokens = highlightLine('hello', 'test.xyz')
|
||||
|
||||
expect(tokens).toEqual([{ text: 'hello', style: null }])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,74 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { isLanguageSupported, getSupportedExtensions, getParserForFile } from '../parsers.js'
|
||||
|
||||
describe('isLanguageSupported', () => {
|
||||
it('returns true for supported file extensions', () => {
|
||||
expect(isLanguageSupported('test.js')).toBe(true)
|
||||
expect(isLanguageSupported('test.ts')).toBe(true)
|
||||
expect(isLanguageSupported('test.tsx')).toBe(true)
|
||||
expect(isLanguageSupported('test.py')).toBe(true)
|
||||
expect(isLanguageSupported('test.go')).toBe(true)
|
||||
expect(isLanguageSupported('test.rs')).toBe(true)
|
||||
expect(isLanguageSupported('test.json')).toBe(true)
|
||||
expect(isLanguageSupported('test.css')).toBe(true)
|
||||
expect(isLanguageSupported('test.html')).toBe(true)
|
||||
expect(isLanguageSupported('test.java')).toBe(true)
|
||||
expect(isLanguageSupported('test.ex')).toBe(true)
|
||||
})
|
||||
|
||||
it('returns false for unsupported file extensions', () => {
|
||||
expect(isLanguageSupported('test.xyz')).toBe(false)
|
||||
expect(isLanguageSupported('test.txt')).toBe(false)
|
||||
expect(isLanguageSupported('test.csv')).toBe(false)
|
||||
})
|
||||
|
||||
it('returns false for files without extensions', () => {
|
||||
expect(isLanguageSupported('Makefile')).toBe(false)
|
||||
})
|
||||
|
||||
it('handles nested paths', () => {
|
||||
expect(isLanguageSupported('src/utils/test.ts')).toBe(true)
|
||||
expect(isLanguageSupported('deep/nested/path/file.py')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getSupportedExtensions', () => {
|
||||
it('returns an array of extension strings', () => {
|
||||
const extensions = getSupportedExtensions()
|
||||
|
||||
expect(Array.isArray(extensions)).toBe(true)
|
||||
expect(extensions.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('includes common extensions', () => {
|
||||
const extensions = getSupportedExtensions()
|
||||
|
||||
expect(extensions).toContain('js')
|
||||
expect(extensions).toContain('ts')
|
||||
expect(extensions).toContain('tsx')
|
||||
expect(extensions).toContain('py')
|
||||
expect(extensions).toContain('go')
|
||||
expect(extensions).toContain('rs')
|
||||
expect(extensions).toContain('json')
|
||||
})
|
||||
})
|
||||
|
||||
describe('getParserForFile', () => {
|
||||
it('returns a parser for supported files', () => {
|
||||
expect(getParserForFile('test.js')).not.toBeNull()
|
||||
expect(getParserForFile('test.py')).not.toBeNull()
|
||||
})
|
||||
|
||||
it('returns null for unsupported files', () => {
|
||||
expect(getParserForFile('test.xyz')).toBeNull()
|
||||
})
|
||||
|
||||
it('returns null for files without extension', () => {
|
||||
expect(getParserForFile('noext')).toBeNull()
|
||||
})
|
||||
|
||||
it('is case-insensitive for extensions', () => {
|
||||
expect(getParserForFile('test.JS')).not.toBeNull()
|
||||
expect(getParserForFile('test.Py')).not.toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { HighlightStyle } from './types.js'
|
||||
|
||||
export const darkHighlightColors: Record<HighlightStyle, string> = {
|
||||
keyword: '#ff7b72',
|
||||
comment: '#8b949e',
|
||||
string: '#a5d6ff',
|
||||
number: '#79c0ff',
|
||||
literal: '#79c0ff',
|
||||
function: '#d2a8ff',
|
||||
definition: '#d2a8ff',
|
||||
class: '#ffa657',
|
||||
type: '#ff7b72',
|
||||
tag: '#7ee787',
|
||||
attribute: '#79c0ff',
|
||||
property: '#79c0ff',
|
||||
variable: '#c9d1d9',
|
||||
operator: '#79c0ff',
|
||||
punctuation: '#c9d1d9',
|
||||
regexp: '#a5d6ff',
|
||||
escape: '#79c0ff',
|
||||
meta: '#8b949e',
|
||||
heading: '#79c0ff',
|
||||
link: '#a5d6ff',
|
||||
}
|
||||
|
||||
export const lightHighlightColors: Record<HighlightStyle, string> = {
|
||||
keyword: '#cf222e',
|
||||
comment: '#6e7781',
|
||||
string: '#0a3069',
|
||||
number: '#0550ae',
|
||||
literal: '#0550ae',
|
||||
function: '#8250df',
|
||||
definition: '#8250df',
|
||||
class: '#953800',
|
||||
type: '#cf222e',
|
||||
tag: '#116329',
|
||||
attribute: '#0550ae',
|
||||
property: '#0550ae',
|
||||
variable: '#24292f',
|
||||
operator: '#0550ae',
|
||||
punctuation: '#24292f',
|
||||
regexp: '#0a3069',
|
||||
escape: '#0550ae',
|
||||
meta: '#6e7781',
|
||||
heading: '#0550ae',
|
||||
link: '#0a3069',
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { highlightTree, tagHighlighter, tags } from '@lezer/highlight'
|
||||
import type { HighlightStyle, HighlightToken } from './types.js'
|
||||
import { getParserForFile } from './parsers.js'
|
||||
|
||||
const highlighter = tagHighlighter([
|
||||
{ tag: tags.keyword, class: 'keyword' },
|
||||
{ tag: tags.controlKeyword, class: 'keyword' },
|
||||
{ tag: tags.operatorKeyword, class: 'keyword' },
|
||||
{ tag: tags.definitionKeyword, class: 'keyword' },
|
||||
{ tag: tags.moduleKeyword, class: 'keyword' },
|
||||
{ tag: tags.comment, class: 'comment' },
|
||||
{ tag: tags.lineComment, class: 'comment' },
|
||||
{ tag: tags.blockComment, class: 'comment' },
|
||||
{ tag: tags.docComment, class: 'comment' },
|
||||
{ tag: tags.string, class: 'string' },
|
||||
{ tag: tags.special(tags.string), class: 'string' },
|
||||
{ tag: tags.number, class: 'number' },
|
||||
{ tag: tags.integer, class: 'number' },
|
||||
{ tag: tags.float, class: 'number' },
|
||||
{ tag: tags.bool, class: 'literal' },
|
||||
{ tag: tags.null, class: 'literal' },
|
||||
{ tag: tags.function(tags.variableName), class: 'function' },
|
||||
{ tag: tags.function(tags.propertyName), class: 'function' },
|
||||
{ tag: tags.definition(tags.variableName), class: 'definition' },
|
||||
{ tag: tags.definition(tags.propertyName), class: 'definition' },
|
||||
{ tag: tags.definition(tags.function(tags.variableName)), class: 'definition' },
|
||||
{ tag: tags.className, class: 'class' },
|
||||
{ tag: tags.definition(tags.className), class: 'class' },
|
||||
{ tag: tags.typeName, class: 'type' },
|
||||
{ tag: tags.tagName, class: 'tag' },
|
||||
{ tag: tags.attributeName, class: 'attribute' },
|
||||
{ tag: tags.attributeValue, class: 'string' },
|
||||
{ tag: tags.propertyName, class: 'property' },
|
||||
{ tag: tags.variableName, class: 'variable' },
|
||||
{ tag: tags.local(tags.variableName), class: 'variable' },
|
||||
{ tag: tags.special(tags.variableName), class: 'variable' },
|
||||
{ tag: tags.operator, class: 'operator' },
|
||||
{ tag: tags.punctuation, class: 'punctuation' },
|
||||
{ tag: tags.bracket, class: 'punctuation' },
|
||||
{ tag: tags.separator, class: 'punctuation' },
|
||||
{ tag: tags.regexp, class: 'regexp' },
|
||||
{ tag: tags.escape, class: 'escape' },
|
||||
{ tag: tags.meta, class: 'meta' },
|
||||
{ tag: tags.heading, class: 'heading' },
|
||||
{ tag: tags.link, class: 'link' },
|
||||
{ tag: tags.url, class: 'link' },
|
||||
])
|
||||
|
||||
export function highlightCode(code: string, filename: string): HighlightToken[][] {
|
||||
const parser = getParserForFile(filename)
|
||||
|
||||
if (!parser) {
|
||||
return code.split('\n').map((line) => [{ text: line, style: null }])
|
||||
}
|
||||
|
||||
const tree = parser.parse(code)
|
||||
const lines = code.split('\n')
|
||||
const result: HighlightToken[][] = []
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
result.push([])
|
||||
}
|
||||
|
||||
// Build a map of character positions to styles
|
||||
const styleMap: Array<HighlightStyle | null> = new Array(code.length).fill(null)
|
||||
|
||||
highlightTree(tree, highlighter, (from, to, classes) => {
|
||||
for (let i = from; i < to && i < styleMap.length; i++) {
|
||||
styleMap[i] = classes as HighlightStyle
|
||||
}
|
||||
})
|
||||
|
||||
// Convert style map to tokens per line
|
||||
let pos = 0
|
||||
for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
|
||||
const line = lines[lineIndex]
|
||||
|
||||
if (line.length === 0) {
|
||||
result[lineIndex].push({ text: '', style: null })
|
||||
pos++ // skip newline
|
||||
continue
|
||||
}
|
||||
|
||||
let currentToken: HighlightToken = { text: '', style: styleMap[pos] }
|
||||
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const charStyle = styleMap[pos + i]
|
||||
if (charStyle === currentToken.style) {
|
||||
currentToken.text += line[i]
|
||||
} else {
|
||||
if (currentToken.text) {
|
||||
result[lineIndex].push(currentToken)
|
||||
}
|
||||
currentToken = { text: line[i], style: charStyle }
|
||||
}
|
||||
}
|
||||
|
||||
if (currentToken.text) {
|
||||
result[lineIndex].push(currentToken)
|
||||
}
|
||||
|
||||
pos += line.length + 1 // +1 for newline
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export function highlightLine(line: string, filename: string): HighlightToken[] {
|
||||
const result = highlightCode(line, filename)
|
||||
return result[0] ?? [{ text: line, style: null }]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export type { HighlightStyle, HighlightToken } from './types.js'
|
||||
export { getParserForFile, isLanguageSupported, getSupportedExtensions } from './parsers.js'
|
||||
export { highlightCode, highlightLine } from './highlighter.js'
|
||||
export { darkHighlightColors, lightHighlightColors } from './colors.js'
|
||||
@@ -0,0 +1,78 @@
|
||||
import { parser as jsParser } from '@lezer/javascript'
|
||||
import { parser as jsonParser } from '@lezer/json'
|
||||
import { parser as cssParser } from '@lezer/css'
|
||||
import { parser as cppParser } from '@lezer/cpp'
|
||||
import { parser as goParser } from '@lezer/go'
|
||||
import { parser as htmlParser } from '@lezer/html'
|
||||
import { parser as javaParser } from '@lezer/java'
|
||||
import { parser as pythonParser } from '@lezer/python'
|
||||
import { parser as markdownParser } from '@lezer/markdown'
|
||||
import { parser as phpParser } from '@lezer/php'
|
||||
import { parser as rustParser } from '@lezer/rust'
|
||||
import { parser as xmlParser } from '@lezer/xml'
|
||||
import { parser as yamlParser } from '@lezer/yaml'
|
||||
import { parser as elixirParser } from 'lezer-elixir'
|
||||
import type { Parser } from '@lezer/common'
|
||||
|
||||
const parsersByExtension: Record<string, Parser> = {
|
||||
// JavaScript/TypeScript
|
||||
js: jsParser,
|
||||
jsx: jsParser.configure({ dialect: 'jsx' }),
|
||||
ts: jsParser.configure({ dialect: 'ts' }),
|
||||
tsx: jsParser.configure({ dialect: 'ts jsx' }),
|
||||
mjs: jsParser,
|
||||
cjs: jsParser,
|
||||
// C / C++ / Objective-C
|
||||
c: cppParser,
|
||||
h: cppParser,
|
||||
cc: cppParser,
|
||||
cpp: cppParser,
|
||||
cxx: cppParser,
|
||||
hpp: cppParser,
|
||||
hxx: cppParser,
|
||||
m: cppParser,
|
||||
mm: cppParser,
|
||||
// JSON
|
||||
json: jsonParser,
|
||||
// CSS
|
||||
css: cssParser,
|
||||
scss: cssParser,
|
||||
// HTML
|
||||
html: htmlParser,
|
||||
htm: htmlParser,
|
||||
// XML
|
||||
xml: xmlParser,
|
||||
// Java
|
||||
java: javaParser,
|
||||
// Python
|
||||
py: pythonParser,
|
||||
// Go
|
||||
go: goParser,
|
||||
// PHP
|
||||
php: phpParser,
|
||||
// YAML
|
||||
yaml: yamlParser,
|
||||
yml: yamlParser,
|
||||
// Rust
|
||||
rs: rustParser,
|
||||
// Elixir
|
||||
ex: elixirParser,
|
||||
exs: elixirParser,
|
||||
// Markdown
|
||||
md: markdownParser,
|
||||
mdx: markdownParser,
|
||||
}
|
||||
|
||||
export function getParserForFile(filename: string): Parser | null {
|
||||
const ext = filename.split('.').pop()?.toLowerCase()
|
||||
if (!ext) return null
|
||||
return parsersByExtension[ext] ?? null
|
||||
}
|
||||
|
||||
export function isLanguageSupported(filename: string): boolean {
|
||||
return getParserForFile(filename) !== null
|
||||
}
|
||||
|
||||
export function getSupportedExtensions(): string[] {
|
||||
return Object.keys(parsersByExtension)
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
export type HighlightStyle =
|
||||
| 'keyword'
|
||||
| 'comment'
|
||||
| 'string'
|
||||
| 'number'
|
||||
| 'literal'
|
||||
| 'function'
|
||||
| 'definition'
|
||||
| 'class'
|
||||
| 'type'
|
||||
| 'tag'
|
||||
| 'attribute'
|
||||
| 'property'
|
||||
| 'variable'
|
||||
| 'operator'
|
||||
| 'punctuation'
|
||||
| 'regexp'
|
||||
| 'escape'
|
||||
| 'meta'
|
||||
| 'heading'
|
||||
| 'link'
|
||||
|
||||
export interface HighlightToken {
|
||||
text: string
|
||||
style: HighlightStyle | null
|
||||
}
|
||||
Reference in New Issue
Block a user