S.O.N.G/wikitext-parser
S.O.N.G/wikitext-parser
3
0
Fork 0
Code Issues Pull requests Releases Activity

Full parsing of templates and lists on main body

Browse source
This commit is contained in:
DataHoarder 2022-02-20 22:10:24 +01:00
parent a0cd37f626
commit 5c99a0ad20
2 changed files with 100 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
template.go
View file

@ -157,7 +157,7 @@ func ParseTemplate(text string, index int, depth int, startCharacter byte) (i in
} }
lastToken = scanIndex lastToken = scanIndex
i = scanIndex - 1 i = scanIndex - 1
} else if c == ';' { } else if afterNewLine && c == ';' {
addValue() addValue()
var list *DescriptionList var list *DescriptionList
var scanIndex int var scanIndex int

140
wikitext.go
View file

@ -5,37 +5,37 @@ import (
) )
func NormalizeWikiTitle(title string) string { func NormalizeWikiTitle(title string) string {
return strings.Replace(title, " ", "_", -1) return strings.ReplaceAll(title, " ", "_")
} }
type NewLineToken struct { type NewLineToken struct {
} }
type GetInterfaceSliceStringValueOptions struct { type WikiStringValueOptions struct {
PageName string PageName string
Trim bool Trim bool
StringHandler func(value string, opt *GetInterfaceSliceStringValueOptions) []string StringHandler func(value string, opt *WikiStringValueOptions) []string
HTMLHandler func(value *HTML, opt *GetInterfaceSliceStringValueOptions) []string HTMLHandler func(value *HTML, opt *WikiStringValueOptions) []string
LinkHandler func(value *Link, opt *GetInterfaceSliceStringValueOptions) []string LinkHandler func(value *Link, opt *WikiStringValueOptions) []string
TemplateLinkHandler func(value *Template, opt *GetInterfaceSliceStringValueOptions) []string TemplateLinkHandler func(value *Template, opt *WikiStringValueOptions) []string
TemplateHandler func(value *Template, opt *GetInterfaceSliceStringValueOptions) []string TemplateHandler func(value *Template, opt *WikiStringValueOptions) []string
UnorderedListHandler func(value *UnorderedList, opt *GetInterfaceSliceStringValueOptions) []string UnorderedListHandler func(value *UnorderedList, opt *WikiStringValueOptions) []string
DescriptionListHandler func(value *DescriptionList, opt *GetInterfaceSliceStringValueOptions) []string DescriptionListHandler func(value *DescriptionList, opt *WikiStringValueOptions) []string
NewLineHandler func(opt *GetInterfaceSliceStringValueOptions) []string NewLineHandler func(opt *WikiStringValueOptions) []string
} }
func (o *GetInterfaceSliceStringValueOptions) Default() { func (o *WikiStringValueOptions) Default() {
o.Trim = true o.Trim = true
o.StringHandler = func(value string, opt *GetInterfaceSliceStringValueOptions) []string { o.StringHandler = func(value string, opt *WikiStringValueOptions) []string {
return []string{value} return []string{value}
} }
o.HTMLHandler = func(value *HTML, opt *GetInterfaceSliceStringValueOptions) []string { o.HTMLHandler = func(value *HTML, opt *WikiStringValueOptions) []string {
return []string{value.Tag.String()} return []string{value.Tag.String()}
} }
o.NewLineHandler = func(opt *GetInterfaceSliceStringValueOptions) []string { o.NewLineHandler = func(opt *WikiStringValueOptions) []string {
return []string{"\n"} return []string{"\n"}
} }
o.LinkHandler = func(value *Link, opt *GetInterfaceSliceStringValueOptions) (result []string) { o.LinkHandler = func(value *Link, opt *WikiStringValueOptions) (result []string) {
if len(value.Name) > 0 { if len(value.Name) > 0 {
result = append(result, GetWikiStringValue(value.Name, opt)...) result = append(result, GetWikiStringValue(value.Name, opt)...)
} else { } else {
@ -44,7 +44,7 @@ func (o *GetInterfaceSliceStringValueOptions) Default() {
result = append(result, GetWikiStringValue(value.Name, opt)...) result = append(result, GetWikiStringValue(value.Name, opt)...)
return return
} }
o.TemplateLinkHandler = func(value *Template, opt *GetInterfaceSliceStringValueOptions) (result []string) { o.TemplateLinkHandler = func(value *Template, opt *WikiStringValueOptions) (result []string) {
output := 0 output := 0
for _, vv := range value.Parameters { for _, vv := range value.Parameters {
for _, vvv := range GetWikiStringValue(vv, opt) { for _, vvv := range GetWikiStringValue(vv, opt) {
@ -61,7 +61,7 @@ func (o *GetInterfaceSliceStringValueOptions) Default() {
return return
} }
o.TemplateHandler = func(value *Template, opt *GetInterfaceSliceStringValueOptions) (result []string) { o.TemplateHandler = func(value *Template, opt *WikiStringValueOptions) (result []string) {
switch strings.ToUpper(value.Name) { switch strings.ToUpper(value.Name) {
case "PAGENAME", "SUBPAGENAME": case "PAGENAME", "SUBPAGENAME":
result = append(result, opt.PageName) result = append(result, opt.PageName)
@ -71,15 +71,15 @@ func (o *GetInterfaceSliceStringValueOptions) Default() {
return return
} }
o.UnorderedListHandler = func(value *UnorderedList, opt *GetInterfaceSliceStringValueOptions) []string { o.UnorderedListHandler = func(value *UnorderedList, opt *WikiStringValueOptions) []string {
return GetWikiStringValue(value.Entries, opt) return GetWikiStringValue(value.Entries, opt)
} }
o.DescriptionListHandler = func(value *DescriptionList, opt *GetInterfaceSliceStringValueOptions) []string { o.DescriptionListHandler = func(value *DescriptionList, opt *WikiStringValueOptions) []string {
return []string{strings.Join(GetWikiStringValue(value.Name, opt), ", ") + ": " + strings.Join(GetWikiStringValue(value.Entries, opt), ", ")} return []string{strings.Join(GetWikiStringValue(value.Name, opt), ", ") + ": " + strings.Join(GetWikiStringValue(value.Entries, opt), ", ")}
} }
} }
func GetWikiStringValue(v []interface{}, opts *GetInterfaceSliceStringValueOptions) (r []string) { func GetWikiStringValue(v []interface{}, opts *WikiStringValueOptions) (r []string) {
var result []string var result []string
for _, value := range v { for _, value := range v {
@ -120,35 +120,93 @@ func GetWikiStringValue(v []interface{}, opts *GetInterfaceSliceStringValueOptio
//ParseWikiText small WikiText parser that extracts text, Templates, and its arguments/parameters //ParseWikiText small WikiText parser that extracts text, Templates, and its arguments/parameters
func ParseWikiText(text string) (result []interface{}) { func ParseWikiText(text string) (result []interface{}) {
index := 0 index := 0
var i int
for index < len(text) { var c byte
templateIndex := strings.Index(text[index:], "{{") lastToken := index
linkIndex := strings.Index(text[index:], "[[")
if templateIndex == -1 && linkIndex == -1 { addValue := func() int {
t := strings.TrimSpace(text[index:]) if lastToken < len(text) && i-lastToken > 0 {
t := strings.TrimSpace(text[lastToken:i])
if len(t) > 0 { if len(t) > 0 {
result = append(result, text[index:]) result = append(result, t)
}
break
} else {
bestIndex := templateIndex
if templateIndex == -1 {
bestIndex = linkIndex
} else {
if linkIndex != -1 && linkIndex < bestIndex {
bestIndex = linkIndex
}
} }
t := strings.TrimSpace(text[index : index+bestIndex]) return len(t)
if len(t) > 0 { }
result = append(result, text[index:index+bestIndex])
} return 0
}
afterNewLine := false
for i = index; i < len(text); i++ {
c = text[i]
if (c == '{' && i < len(text)-1 && text[i+1] == '{') || (c == '[' && i < len(text)-1 && text[i+1] == '[') {
addValue()
var tpl *Template var tpl *Template
index, tpl = ParseTemplate(text, index+bestIndex+2, 0, text[index+bestIndex]) var scanIndex int
scanIndex, tpl = ParseTemplate(text, i+2, 1, c)
if tpl != nil { if tpl != nil {
result = append(result, tpl) result = append(result, tpl)
} }
lastToken = scanIndex
i = scanIndex - 1
} else if (c == '{' && i < len(text)-1 && text[i+1] != '{' && text[i+1] != '[') || (c == '[' && i < len(text)-1 && text[i+1] != '[' && text[i+1] != '{') {
addValue()
var link *Link
var scanIndex int
scanIndex, link = ParseLink(text, i+1, 1, c)
if link != nil {
result = append(result, link)
}
lastToken = scanIndex
i = scanIndex - 1
} else if c == '<' { //html trigger
addValue()
var html *HTML
var scanIndex int
scanIndex, html = ParseHTML(text, i, 1)
if html != nil {
result = append(result, html)
}
lastToken = scanIndex
i = scanIndex - 1
} else if c == '\n' {
addValue()
lastToken = i + 1
afterNewLine = true
result = append(result, NewLineToken{})
} else if afterNewLine && (c == '*' || c == '#') {
addValue()
var list *UnorderedList
var scanIndex int
scanIndex, list = ParseUnorderedList(text, i, 1, 1, c)
if list != nil {
result = append(result, list)
}
lastToken = scanIndex
i = scanIndex - 1
} else if afterNewLine && c == ';' {
addValue()
var list *DescriptionList
var scanIndex int
scanIndex, list = ParseDescriptionList(text, i+1, 1)
if list != nil {
result = append(result, list)
}
lastToken = scanIndex
i = scanIndex - 1
} else if afterNewLine && c == ':' {
addValue()
lastToken = i + 1
}
if afterNewLine && c != '\n' && c != ' ' && c != '\t' {
afterNewLine = false
} }
} }