feat: add list content token

This commit is contained in:
moonrailgun 2024-07-20 01:39:10 +08:00
parent 03904d26e0
commit 7736bf89dc
5 changed files with 36 additions and 2 deletions

View File

@ -5,6 +5,7 @@ import { TelegramContentTokenizer } from './tokenizer/telegram';
import { import {
ContentToken, ContentToken,
ImageContentToken, ImageContentToken,
ListContentToken,
NewlineContentToken, NewlineContentToken,
ParagraphContentToken, ParagraphContentToken,
TextContentToken, TextContentToken,
@ -40,6 +41,10 @@ export const token = {
url, url,
title, title,
}), }),
list: (items: ContentToken[]): ListContentToken => ({
type: 'list',
items,
}),
}; };
export const baseContentTokenizer = new BaseContentTokenizer(); export const baseContentTokenizer = new BaseContentTokenizer();

View File

@ -1,6 +1,7 @@
import { import {
ContentToken, ContentToken,
ImageContentToken, ImageContentToken,
ListContentToken,
NewlineContentToken, NewlineContentToken,
ParagraphContentToken, ParagraphContentToken,
TextContentToken, TextContentToken,
@ -33,7 +34,11 @@ export class BaseContentTokenizer {
return token.url; return token.url;
} }
parse(tokens: ContentToken[]) { parseList(token: ListContentToken) {
return token.items.map((item) => this.parse([item])).join('\n');
}
parse(tokens: ContentToken[]): string {
return tokens return tokens
.map((token) => { .map((token) => {
if (token.type === 'text') { if (token.type === 'text') {
@ -56,6 +61,10 @@ export class BaseContentTokenizer {
return this.parseNewline(token); return this.parseNewline(token);
} }
if (token.type === 'list') {
return this.parseList(token);
}
return ''; return '';
}) })
.join('') .join('')

View File

@ -1,5 +1,6 @@
import { import {
ImageContentToken, ImageContentToken,
ListContentToken,
NewlineContentToken, NewlineContentToken,
ParagraphContentToken, ParagraphContentToken,
TitleContentToken, TitleContentToken,
@ -37,4 +38,8 @@ export class HTMLContentTokenizer extends BaseContentTokenizer {
parseUrl(token: UrlContentToken): string { parseUrl(token: UrlContentToken): string {
return `<a href="${token.url}">${token.title ?? token.url}</a>`; return `<a href="${token.url}">${token.title ?? token.url}</a>`;
} }
parseList(token: ListContentToken) {
return `<ul>${token.items.map((item) => `<li>${this.parse([item])}</li>`).join('')}</ul>`;
}
} }

View File

@ -1,5 +1,6 @@
import { import {
ImageContentToken, ImageContentToken,
ListContentToken,
ParagraphContentToken, ParagraphContentToken,
TitleContentToken, TitleContentToken,
UrlContentToken, UrlContentToken,
@ -32,4 +33,12 @@ export class MarkdownContentTokenizer extends BaseContentTokenizer {
parseUrl(token: UrlContentToken): string { parseUrl(token: UrlContentToken): string {
return `[${token.title ?? ''}](${token.url})`; return `[${token.title ?? ''}](${token.url})`;
} }
parseList(token: ListContentToken) {
return (
'\n' +
token.items.map((item) => `- ${this.parse([item])}`).join('\n') +
'\n'
);
}
} }

View File

@ -29,10 +29,16 @@ export type UrlContentToken = {
title?: string; title?: string;
}; };
export type ListContentToken = {
type: 'list';
items: ContentToken[];
};
export type ContentToken = export type ContentToken =
| TextContentToken | TextContentToken
| ImageContentToken | ImageContentToken
| TitleContentToken | TitleContentToken
| ParagraphContentToken | ParagraphContentToken
| NewlineContentToken | NewlineContentToken
| UrlContentToken; | UrlContentToken
| ListContentToken;