24 lines
430 B
Go
24 lines
430 B
Go
package metadata
|
|
|
|
import (
|
|
"github.com/ikawaha/kagome-dict/uni"
|
|
"github.com/ikawaha/kagome/v2/tokenizer"
|
|
"golang.org/x/text/unicode/norm"
|
|
"log"
|
|
)
|
|
|
|
func GenerateAlternateFromJapanese(text string) []Name {
|
|
normalized := norm.NFC.String(text)
|
|
|
|
t, err := tokenizer.New(uni.Dict(), tokenizer.OmitBosEos())
|
|
for _, token := range t.Tokenize(normalized) {
|
|
token.Features()
|
|
}
|
|
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
|
|
return nil
|
|
}
|