touhouwiki-mirror/server.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"git.gammaspectra.live/S.O.N.G/touhouwiki-mirror/utilities"
	wikitext_parser "git.gammaspectra.live/S.O.N.G/wikitext-parser"
	"golang.org/x/text/cases"
	"golang.org/x/text/language"
	"golang.org/x/text/runes"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"io/fs"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode"
)

var cdIndex = make(map[int]*albumEntry)
var cdIndexLock sync.Mutex

var albumTitleLookup = make(map[string][]*albumEntry)

//TODO: make this work with discs
var discTracksLookup = make(map[int][]*albumEntry)

type categoryPageIndex struct {
	PageId    int    `json:"pageid"`
	Namespace int    `json:"ns"`
	Title     string `json:"title"`
	Timestamp string `json:"timestamp"`
}

func parseCategoryPageIndex(filePath string) []categoryPageIndex {
	s := &struct {
		Query struct {
			Members []categoryPageIndex `json:"categorymembers"`
		} `json:"query"`
	}{}

	fileData, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil
	}

	err = json.Unmarshal(fileData, s)
	if err != nil {
		return nil
	}
	return s.Query.Members
}

type JSONTime struct {
	time.Time
}

func (t JSONTime) MarshalJSON() ([]byte, error) {
	if t.IsZero() {
		return nil, nil
	}
	return []byte(fmt.Sprintf("\"%s\"", t.Format("2006-01-02"))), nil
}

type albumEntry struct {
	Id             int               `json:"pageid"`
	Type           string            `json:"type"`
	MainTitle      string            `json:"pagetitle"`
	Titles         map[string]string `json:"titles"`
	Image          string            `json:"image,omitempty"`
	CatalogNumbers []string          `json:"catalognumbers,omitempty"`
	Genre          []string          `json:"genre,omitempty"`
	TrackCount     int               `json:"trackcount,omitempty"`
	Duration       int               `json:"duration,omitempty"`
	ReleaseDate    JSONTime          `json:"releasedate,omitempty"`
	ReleaseEvent   string            `json:"releaseevent,omitempty"`
	Links          []string          `json:"links,omitempty"`
	Artists        []artistEntry     `json:"artists,omitempty"`
	Discs          []discEntry       `json:"discs,omitempty"`
}
type discEntry struct {
	Name       string       `json:"name,omitempty"`
	Duration   int          `json:"duration,omitempty"`
	TrackCount int          `json:"trackcount,omitempty"`
	Tracks     []trackEntry `json:"tracks"`
}
type trackEntry struct {
	Duration  int               `json:"duration,omitempty"`
	MainTitle string            `json:"title,omitempty"`
	Titles    map[string]string `json:"titles,omitempty"`
	Artists   []artistEntry     `json:"artists,omitempty"`
	Lyrics    string            `json:"lyrics,omitempty"`
	Links     []string          `json:"links,omitempty"`
	Original  []string          `json:"original,omitempty"`
}
type artistEntry struct {
	Position string   `json:"position"`
	Names    []string `json:"names"`
}

func getArtistEntries(kind string, entries []interface{}, opts *wikitext_parser.WikiStringValueOptions) (artists []artistEntry) {

	artist := artistEntry{
		Position: kind,
	}

	recreateArtist := func(kinds ...string) {
		var names []string
		for _, n := range artist.Names {
			//TODO
			//n = strings.Trim(n, " ()[]（）")
			if opts.Trim {
				n = strings.TrimSpace(n)
			}
			if len(n) > 0 {
				var curName string
				if len(names) > 0 {
					curName = names[len(names)-1]
				}
				if len(curName) > 0 && curName[len(curName)-1] == '(' {
					names[len(names)-1] += n
				} else if len(curName) > 0 && len(n) > 1 && n[len(n)-1] == ')' && strings.Index(n, "(") <= 0 {
					names[len(names)-1] += " " + n
				} else if len(curName) > 0 && n[0] == '(' && (strings.Index(n, ")") == -1 || n[len(n)-1] == ')') {
					names[len(names)-1] += " " + n
				} else if len(curName) > 0 && n == ")" {
					names[len(names)-1] += n
				} else if len(curName) > 0 && curName[len(curName)-1] == '/' {
					names[len(names)-1] += n
				} else if len(curName) > 0 && n == "/" {
					names[len(names)-1] += n
				} else {
					names = append(names, n)
				}
			}
		}

		artist.Names = names
		if len(artist.Names) > 0 {
			artists = append(artists, artist)
			artist = artistEntry{
				Position: strings.Join(kinds, ", "),
			}
		}
	}

	for _, value := range entries {
		if text, ok := value.(string); ok {
			if i := strings.Index(text, " & "); i != -1 {
				for _, vv := range strings.Split(text, " & ") {
					vv = normalizeStringCharacters(vv)
					if len(vv) > 0 {
						artist.Names = append(artist.Names, vv)
					}
					recreateArtist(kind)
				}
			} else if i := strings.Index(text, "& "); i == 0 {
				for _, vv := range strings.Split(text, "& ") {
					vv = normalizeStringCharacters(vv)
					if len(vv) > 0 {
						artist.Names = append(artist.Names, vv)
					}
					recreateArtist(kind)
				}
			} else if i := strings.Index(text, " &"); i != -1 && i == len(text)-2 {
				for _, vv := range strings.Split(text, " &") {
					vv = normalizeStringCharacters(vv)
					if len(vv) > 0 {
						artist.Names = append(artist.Names, vv)
					}
					recreateArtist(kind)
				}
			} else if i := strings.Index(text, " and "); i != -1 {
				for _, vv := range strings.Split(text, " and ") {
					vv = normalizeStringCharacters(vv)
					if len(vv) > 0 {
						artist.Names = append(artist.Names, vv)
					}
					recreateArtist(kind)
				}
			} else if i := strings.Index(text, "and "); i == 0 {
				for _, vv := range strings.Split(text, "and ") {
					vv = normalizeStringCharacters(vv)
					if len(vv) > 0 {
						artist.Names = append(artist.Names, vv)
					}
					recreateArtist(kind)
				}
			} else if i := strings.Index(text, " and"); i != -1 && i == len(text)-4 {
				for _, vv := range strings.Split(text, " and") {
					vv = normalizeStringCharacters(vv)
					if len(vv) > 0 {
						artist.Names = append(artist.Names, vv)
					}
					recreateArtist(kind)
				}
			} else {
				for _, vv := range strings.Split(text, " and") {
					vv = normalizeStringCharacters(vv)
					if len(vv) > 0 {
						artist.Names = append(artist.Names, vv)
					}
				}
			}
		} else if _, ok := value.(wikitext_parser.NewLineToken); ok {
			recreateArtist(kind)
		} else if tpl, ok := value.(*wikitext_parser.Template); ok {
			if tpl.IsLink {
				var result []string
				for _, vv := range tpl.Parameters {
					result = append(result, getStringValue(vv, opts)...)
				}
				if len(result) == 0 {
					result = append(result, tpl.Name)
				}
				artist.Names = append(artist.Names, strings.Join(result, " "))
			} else {
				artist.Names = append(artist.Names, strings.Join(getStringValue([]interface{}{tpl}, opts), " "))
			}
		} else if link, ok := value.(*wikitext_parser.Link); ok {
			if len(link.Name) > 0 {
				artist.Names = append(artist.Names, strings.Join(getStringValue(link.Name, opts), " "))
			} else if !link.IsExternal {
				artist.Names = append(artist.Names, link.URL)
			}
		} else if unorderedList, ok := value.(*wikitext_parser.UnorderedList); ok {
			for _, val := range unorderedList.Entries {
				recreateArtist(kind)

				if text, ok := val.(string); ok {
					artist.Names = append(artist.Names, text)
				} else if tpl, ok := val.(*wikitext_parser.Template); ok {
					if tpl.IsLink {
						artist.Names = append(artist.Names, tpl.Name)
						for _, val := range tpl.Parameters {
							artist.Names = append(artist.Names, getStringValue(val, opts)...)
						}
					} else {
						artist.Names = append(artist.Names, getStringValue([]interface{}{tpl}, opts)...)
					}
				}
			}
			recreateArtist(kind)
		} else if descriptionList, ok := value.(*wikitext_parser.DescriptionList); ok {
			for _, val := range descriptionList.Entries {
				recreateArtist(kind, strings.Join(getStringValue(descriptionList.Name, opts), ", "))

				if text, ok := val.(string); ok {
					artist.Names = append(artist.Names, text)
				} else if tpl, ok := val.(*wikitext_parser.Template); ok {
					if tpl.IsLink {
						artist.Names = append(artist.Names, tpl.Name)
						for _, val := range tpl.Parameters {
							artist.Names = append(artist.Names, getStringValue(val, opts)...)
						}
					} else {
						artist.Names = append(artist.Names, getStringValue([]interface{}{tpl}, opts)...)
					}
				}

				recreateArtist(kind)
			}
		}
	}

	recreateArtist(kind)

	return
}

func getWikiStringOptions(title string, trim bool) *wikitext_parser.WikiStringValueOptions {

	opts := &wikitext_parser.WikiStringValueOptions{}
	opts.Default()
	opts.PageName = title
	opts.TemplateHandler = func(template *wikitext_parser.Template, opt *wikitext_parser.WikiStringValueOptions) (result []string) {
		switch strings.ToUpper(template.Name) {
		case "H:TITLE":
			if val, ok := template.Parameters["0"]; ok && len(val) > 0 {
				result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0])
			}
		case "LANG":
			if val, ok := template.Parameters["1"]; ok && len(val) > 0 {
				result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0])
			}
		case "GENRE":
			if val, ok := template.Parameters["0"]; ok && len(val) > 0 {
				result = append(result, wikitext_parser.GetWikiStringValue(val, opt)[0])
			}
		case "MUSICARTICLE/DIVIDER":
		case "PAGENAME", "SUBPAGENAME":
			result = append(result, opt.PageName)
		default:
			result = append(result, template.Name)
		}
		return
	}
	opts.Trim = trim
	return opts
}

func getStringValue(v []interface{}, opt *wikitext_parser.WikiStringValueOptions) []string {
	return wikitext_parser.GetWikiStringValue(v, opt)
}

func processIndexDirectory(filePath, indexPath, kind string, wg *sync.WaitGroup) {
	entries, err := ioutil.ReadDir(indexPath)
	if err != nil {
		return
	}

	for _, e := range entries {
		if path.Ext(e.Name()) == ".json" {
			for _, v := range parseCategoryPageIndex(path.Join(indexPath, e.Name())) {
				wg.Add(1)
				go func(entry *albumEntry) {
					defer wg.Done()

					contents, err := ioutil.ReadFile(path.Join(filePath, "pages", fmt.Sprintf("%d.wiki", entry.Id)))
					if err != nil {
						return
					}
					result := wikitext_parser.ParseWikiText(string(contents))

					if len(result) == 0 {
						return
					}

					tpl, ok := result[0].(*wikitext_parser.Template)
					if !ok {
						return
					}

					if tpl.Name != "MusicArticle" {
						return
					}

					opts := getWikiStringOptions(entry.MainTitle, true)

					//var discLengths []int
					//var discTrackNumbers []int

					var val []interface{}
					var stringVal []string

					if val, ok = tpl.Parameters["titleen"]; ok {
						if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
							entry.Titles["english"] = stringVal[0]
						}
					}

					if val, ok = tpl.Parameters["titlejp"]; ok {
						if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
							entry.Titles["japanese"] = stringVal[0]
						}
					}

					if val, ok = tpl.Parameters["titlejprom"]; ok {
						if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
							entry.Titles["romaji"] = stringVal[0]
						}
					}

					if val, ok = tpl.Parameters["catalogno"]; ok {
						if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
							for _, catnos := range stringVal {
								for _, catno := range strings.Split(catnos, " ") {
									if strings.ToLower(catno) != "n/a" {
										catno = strings.ReplaceAll(catno, "(", "")
										catno = strings.ReplaceAll(catno, ")", "")
										entry.CatalogNumbers = append(entry.CatalogNumbers, catno)
									}
								}
							}
						}
					}

					if val, ok = tpl.Parameters["genre"]; ok {
						if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
							for _, genre := range strings.Split(strings.Join(stringVal, " "), ",") {
								entry.Genre = append(entry.Genre, strings.ToLower(strings.TrimSpace(genre)))
							}
						}
					}

					if val, ok = tpl.Parameters["image"]; ok && len(val) > 0 {
						entry.Image = strings.TrimSpace(strings.Join(getStringValue(val, opts), ""))
					}
					if val, ok = tpl.Parameters["group"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("group", val, opts)...)
					}
					if val, ok = tpl.Parameters["masterer"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("mastering", val, opts)...)
					}
					if val, ok = tpl.Parameters["illustrator"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("illustration", val, opts)...)
					}
					if val, ok = tpl.Parameters["arranger"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("arranger", val, opts)...)
					}
					if val, ok = tpl.Parameters["lyricist"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("lyrics", val, opts)...)
					}
					if val, ok = tpl.Parameters["vocalist"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("vocals", val, opts)...)
					}
					if val, ok = tpl.Parameters["producer"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("producer", val, opts)...)
					}
					if val, ok = tpl.Parameters["designer"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("design", val, opts)...)
					}
					if val, ok = tpl.Parameters["other"]; ok {
						entry.Artists = append(entry.Artists, getArtistEntries("other", val, opts)...)
					}
					if val, ok = tpl.Parameters["convention"]; ok && len(val) > 0 {
						value := getStringValue(val, opts)
						if len(value) > 0 {
							entry.ReleaseEvent = value[0]
						}
					}

					if val, ok = tpl.Parameters["released"]; ok {
						if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
							if releaseDate, err := time.ParseInLocation("2006-01-02", stringVal[0], time.UTC); err == nil {
								entry.ReleaseDate = JSONTime{Time: releaseDate}
							}
						}
					}

					if val, ok = tpl.Parameters["website"]; ok {
						for _, value := range val {
							if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal {
								entry.Links = append(entry.Links, linkVal.URL)
							}
						}
					}

					if val, ok = tpl.Parameters["digital"]; ok {
						for _, value := range val {
							if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal {
								entry.Links = append(entry.Links, linkVal.URL)
							}
						}
					}

					if val, ok = tpl.Parameters["tracklist"]; ok {
						var disc discEntry
						createDisc := func() {
							if len(disc.Tracks) > 0 {
								disc.TrackCount = len(disc.Tracks)
								entry.Duration += disc.Duration
								entry.TrackCount += disc.TrackCount
								entry.Discs = append(entry.Discs, disc)
							}
						}
						for _, listEntry := range val {
							if strVal, ok := listEntry.(string); ok && len(strVal) > 3 && strVal[0:3] == "===" {
								//title header
								if disc.Name != "" {
									createDisc()
								}
								disc.Name = strings.Trim(strVal, "= ")
							} else if listVal, ok := listEntry.(*wikitext_parser.UnorderedList); ok && len(listVal.Entries) > 0 && len(listVal.Entries) > 0 {
								if sliceVal, ok := listVal.Entries[0].([]interface{}); ok && len(sliceVal) > 0 {

									if trackTpl, ok := sliceVal[0].(*wikitext_parser.Template); ok && strings.ToUpper(trackTpl.Name) == "TRACK" && len(trackTpl.Parameters) >= 3 {
										track := trackEntry{
											Titles: make(map[string]string),
										}

										if mainTitleValue := getStringValue(trackTpl.Parameters["1"], opts); len(mainTitleValue) > 0 {
											track.MainTitle = strings.Join(mainTitleValue, "")
											track.Titles["original"] = track.MainTitle
										}
										if durations := getStringValue(trackTpl.Parameters["2"], opts); len(durations) > 0 {
											split := strings.Split(durations[0], ":")
											numbers := make([]int, len(split))
											for i, numVal := range split {
												numbers[i], _ = strconv.Atoi(numVal)
											}

											var duration int
											if len(numbers) == 3 {
												duration = 3600*numbers[0] + 60*numbers[1] + numbers[2]
											} else if len(numbers) == 2 {
												duration = 60*numbers[0] + numbers[1]
											} else if len(numbers) == 1 {
												duration = numbers[0]
											}
											track.Duration = duration
										}

										if lyrics, ok := trackTpl.Parameters["lyrics"]; ok {
											if stringVal = getStringValue(lyrics, opts); ok && len(stringVal) > 0 {
												track.Lyrics = wikitext_parser.NormalizeWikiTitle(strings.TrimSpace(strings.TrimPrefix(stringVal[0], "Lyrics:")))
											}
										}

										if len(listVal.Entries) > 1 {
											if extraListData, ok := listVal.Entries[1].(*wikitext_parser.UnorderedList); ok && len(extraListData.Entries) > 0 {
												for i, entryValue := range extraListData.Entries {
													if descVal, ok := entryValue.([]interface{}); ok && len(descVal) > 0 {

														keyValue := strings.Split(strings.Join(getStringValue(descVal, opts), " "), ":")
														if len(keyValue) > 0 {
															if i == 0 && len(keyValue[0]) > 2 && keyValue[0][0:2] == "''" {
																track.Titles["english"] = strings.Trim(strings.Join(keyValue, " "), "'' ")
																continue
															}
															keyEntry := strings.ToLower(keyValue[0])
															var values []interface{}
															for j, fullValue := range descVal {
																if strVal, ok = fullValue.(string); j == 0 && ok {
																	values = append(values, strings.Join(strings.Split(strVal, ":")[1:], ":"))
																} else {
																	values = append(values, fullValue)
																}
															}
															switch keyEntry {
															case "original arrangement":
																track.Artists = append(track.Artists, getArtistEntries("original arranger", values, opts)...)
															case "original title":
																track.Original = append(track.Original, strings.TrimSpace(strings.Join(getStringValue(values, opts), " ")))
															case "original album":
																var clean []string
																for _, albumValue := range getStringValue(values, opts) {
																	albumValue = strings.Trim(albumValue, " \t'\"")
																	if len(albumValue) > 0 {
																		clean = append(clean, albumValue)
																	}
																}
																track.Original = append(track.Original, "Album: "+strings.TrimSpace(strings.Join(clean, " ")))
															case "lyrics", "vocals", "chorus", "arranger", "composer", "producer", "remix":
																track.Artists = append(track.Artists, getArtistEntries(keyEntry, values, opts)...)
															case "arrangement":
																track.Artists = append(track.Artists, getArtistEntries("arranger", values, opts)...)
															case "composition":
																track.Artists = append(track.Artists, getArtistEntries("composer", values, opts)...)
															case "promotional video":
																for _, value := range values {
																	if linkVal, ok := value.(*wikitext_parser.Link); ok && linkVal.IsExternal {
																		track.Links = append(track.Links, linkVal.URL)
																	}
																}

															}
														}
													}
												}

											}
										}

										disc.Duration += track.Duration
										disc.Tracks = append(disc.Tracks, track)

									}
								}

							}
						}

						createDisc()

					}

					cdIndexLock.Lock()
					cdIndex[entry.Id] = entry
					for _, d := range entry.Discs {
						if _, ok := discTracksLookup[d.TrackCount]; !ok {
							discTracksLookup[d.TrackCount] = []*albumEntry{entry}
						} else {
							for _, val := range discTracksLookup[d.TrackCount] {
								if val.Id == entry.Id {
									goto exit2
								}
							}
							discTracksLookup[d.TrackCount] = append(discTracksLookup[d.TrackCount], entry)

						exit2:
						}
					}

					if len(entry.CatalogNumbers) > 0 {
						for _, catno := range entry.CatalogNumbers {
							normalized := normalizeSearchTitle(catno)
							if _, ok := albumTitleLookup[normalized]; !ok {
								albumTitleLookup[normalized] = []*albumEntry{entry}
							} else {
								albumTitleLookup[normalized] = append(albumTitleLookup[normalized], entry)
							}
						}
					}
					for _, title := range entry.Titles {
						normalized := normalizeSearchTitle(title)
						if _, ok := albumTitleLookup[normalized]; !ok {
							albumTitleLookup[normalized] = []*albumEntry{entry}
						} else {
							for _, val := range albumTitleLookup[normalized] {
								if val.Id == entry.Id {
									goto exit
								}
							}
							albumTitleLookup[normalized] = append(albumTitleLookup[normalized], entry)

						exit:
						}
					}
					defer cdIndexLock.Unlock()
				}(&albumEntry{
					Id:        v.PageId,
					MainTitle: v.Title,
					Type:      kind,
					Titles:    make(map[string]string),
				})
			}
		}
	}

}

func processIndex(filePath string) {

	var wg sync.WaitGroup
	processIndexDirectory(filePath, path.Join(filePath, "pageindex", "Official_CDs"), "official", &wg)
	processIndexDirectory(filePath, path.Join(filePath, "pageindex", "Arrangement_CDs"), "arrangement", &wg)

	wg.Wait()
}

type Lyrics struct {
	MainTitle string        `json:"pagetitle"`
	Titles    []string      `json:"titles"`
	Links     []string      `json:"links,omitempty"`
	Duration  int           `json:"duration,omitempty"`
	Artists   []artistEntry `json:"artists"`

	Entries struct {
		Kanji   []string `json:"kanji,omitempty"`
		Romaji  []string `json:"romaji,omitempty"`
		English []string `json:"english,omitempty"`
	} `json:"entries"`
}

func parseLyrics(filePath, pageName string) (lyrics *Lyrics) {
	if strings.Index(pageName, "/") != -1 {
		return
	}

	var err error
	var contents []byte
	if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:_"+pageName+".wiki")); err != nil {
		if contents, err = ioutil.ReadFile(path.Join(filePath, "pages_by_name", "Lyrics:"+pageName+".wiki")); err != nil {
			if contents, err = ioutil.ReadFile(path.Join(filePath, "pages", pageName+".wiki")); err != nil {
				return
			}
		}
	}

	result := wikitext_parser.ParseWikiText(string(contents))

	if len(result) == 0 {
		return
	}

	tpl, ok := result[0].(*wikitext_parser.Template)
	if !ok {
		return
	}

	if tpl.Name != "Lyrics" {
		return
	}

	lyrics = &Lyrics{
		MainTitle: strings.ReplaceAll(pageName, "_", " "),
	}

	opts := getWikiStringOptions(lyrics.MainTitle, true)

	//var discLengths []int
	//var discTrackNumbers []int

	var val []interface{}
	var stringVal []string

	if val, ok = tpl.Parameters["titleen"]; ok {
		if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
			lyrics.Titles = append(lyrics.Titles, stringVal[0])
		}
	}

	if val, ok = tpl.Parameters["titlejp"]; ok {
		if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
			lyrics.Titles = append(lyrics.Titles, stringVal[0])
		}
	}

	if val, ok = tpl.Parameters["titlerom"]; ok {
		if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
			lyrics.Titles = append(lyrics.Titles, stringVal[0])
		}
	}

	if val, ok = tpl.Parameters["group"]; ok {
		lyrics.Artists = append(lyrics.Artists, getArtistEntries("group", val, opts)...)
	}
	if val, ok = tpl.Parameters["arranger"]; ok {
		lyrics.Artists = append(lyrics.Artists, getArtistEntries("arranger", val, opts)...)
	}
	if val, ok = tpl.Parameters["lyricist"]; ok {
		lyrics.Artists = append(lyrics.Artists, getArtistEntries("lyrics", val, opts)...)
	}
	if val, ok = tpl.Parameters["vocalist"]; ok {
		lyrics.Artists = append(lyrics.Artists, getArtistEntries("vocals", val, opts)...)
	}

	if val, ok = tpl.Parameters["length"]; ok {
		if stringVal = getStringValue(val, opts); len(stringVal) > 0 {
			split := strings.Split(stringVal[0], ":")
			numbers := make([]int, len(split))
			for i, numVal := range split {
				numbers[i], _ = strconv.Atoi(numVal)
			}

			var duration int
			if len(numbers) == 3 {
				duration = 3600*numbers[0] + 60*numbers[1] + numbers[2]
			} else if len(numbers) == 2 {
				duration = 60*numbers[0] + numbers[1]
			} else if len(numbers) == 1 {
				duration = numbers[0]
			}

			lyrics.Duration = duration
		}
	}

	noTrimOpts := getWikiStringOptions(lyrics.MainTitle, false)

	var kan []interface{}
	var okKan bool
	var rom []interface{}
	var okRom bool
	var eng []interface{}
	var okEng bool
	for i := 1; ; i++ {
		if kan, okKan = tpl.Parameters[fmt.Sprintf("kan%d", i)]; okKan {
			lyrics.Entries.Kanji = append(lyrics.Entries.Kanji, normalizeStringCharacters(strings.Join(getStringValue(kan, noTrimOpts), "")))
		}
		if rom, okRom = tpl.Parameters[fmt.Sprintf("rom%d", i)]; okRom {
			lyrics.Entries.Romaji = append(lyrics.Entries.Romaji, normalizeStringCharacters(strings.Join(getStringValue(rom, noTrimOpts), "")))
		}
		if eng, okEng = tpl.Parameters[fmt.Sprintf("eng%d", i)]; okEng {
			lyrics.Entries.English = append(lyrics.Entries.English, normalizeStringCharacters(strings.Join(getStringValue(eng, noTrimOpts), "")))
		}

		if !okKan && !okRom && !okEng {
			break
		}
	}

	return
}

var normalizeSearchTitleTransformer = transform.Chain(
	norm.NFKD,
	//width.Narrow,
	runes.Remove(runes.In(unicode.Cc)),
	runes.Remove(runes.In(unicode.Cf)),
	runes.Remove(runes.In(unicode.Mn)),
	runes.Remove(runes.In(unicode.Me)),
	runes.Remove(runes.In(unicode.Mc)),
	runes.Remove(runes.In(unicode.Po)),
	runes.Remove(runes.In(unicode.Pe)),
	runes.Remove(runes.In(unicode.Ps)),
	runes.Remove(runes.In(unicode.Pf)),
	runes.Remove(runes.In(unicode.Pi)),
	runes.Remove(runes.In(unicode.Pd)),
	runes.Remove(runes.In(unicode.Pc)),
	runes.Remove(runes.In(unicode.Sc)),
	runes.Remove(runes.In(unicode.Sk)),
	runes.Remove(runes.In(unicode.Sm)),
	runes.Remove(runes.In(unicode.So)),
	runes.Remove(runes.In(unicode.Space)),
	cases.Lower(language.Und),
	norm.NFC,
)

func normalizeSearchTitle(title string) (normalized string) {
	normalized, _, _ = transform.String(normalizeSearchTitleTransformer, title)
	return
}

func normalizeStringCharacters(text string) (normalized string) {
	normalized = strings.TrimSpace(norm.NFKC.String(text))

	return
}

type findByDurationResult struct {
	albumEntry *albumEntry
	cddb1      utilities.CDDB1
	toc        utilities.TOC
	discIndex  int
}

func findByTOC(toc utilities.TOC, trackThreshold, discThreshold int) (results []findByDurationResult) {
	for _, result := range findByCDDB1(toc.GetCDDB1(), discThreshold) {
		func() {
			for i, track := range result.albumEntry.Discs[result.discIndex].Tracks {
				diff := math.Abs(float64(track.Duration) - toc.GetTrackDuration(i).Seconds())
				if diff > float64(trackThreshold) { //too much variation
					return
				}
			}
			results = append(results, result)
		}()
	}

	return
}

func findByCDDB1(cddb1 utilities.CDDB1, discThreshold int) []findByDurationResult {
	return findByTracksAndDuration(cddb1.GetTrackNumber(), int(cddb1.GetDuration().Seconds()), discThreshold)
}

func findByTracksAndDuration(tracks, duration, threshold int) (results []findByDurationResult) {
	for _, r := range discTracksLookup[tracks] {
		for i, d := range r.Discs {

			diff := d.Duration - duration
			if diff < 0 {
				diff = -diff
			}

			if diff <= threshold {
				results = append(results, findByDurationResult{
					albumEntry: r,
					discIndex:  i,
				})
			}
		}
	}

	return
}

func main() {
	wd, _ := os.Getwd()
	servePath := flag.String("path", wd, "Path that will be served by default")
	flag.Parse()

	processIndex(*servePath)

	server := &http.Server{
		Addr: ":8777",
		Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
			if request.URL.Path == "/cddb" {
				//cddb emulator
				writer.Header().Set("Content-Type", "text/plain; charset=UTF-8")
				writer.WriteHeader(http.StatusOK)

				cmd := request.URL.Query().Get("cmd")
				if len(cmd) > 4 && cmd[0:4] == "cddb" {
					splits := strings.Split(cmd, " ")
					if len(splits) < 3 {
						writer.Write([]byte("500 Unrecognized command\n.\n"))
						return
					}
					switch splits[1] {
					case "hello":
						writer.Write([]byte("200 hello and welcome\n.\n"))
						return

					case "query":
						formatEntry := func(cddb1 utilities.CDDB1, result findByDurationResult) (out string) {
							var group string
							for _, a := range result.albumEntry.Artists {
								if a.Position == "group" {
									group = a.Names[0]
									break
								}
							}
							if len(result.albumEntry.CatalogNumbers) > result.discIndex {
								out = fmt.Sprintf("%sSoundtrack%d_%d %s [%s] %s / %s", strings.ToUpper(result.albumEntry.Type[0:1])+result.albumEntry.Type[1:], result.albumEntry.Id, result.discIndex, cddb1.String(), result.albumEntry.CatalogNumbers[result.discIndex], group, result.albumEntry.MainTitle)
							} else {
								out = fmt.Sprintf("%sSoundtrack%d_%d %s %s / %s", strings.ToUpper(result.albumEntry.Type[0:1])+result.albumEntry.Type[1:], result.albumEntry.Id, result.discIndex, cddb1.String(), group, result.albumEntry.MainTitle)
							}

							if len(result.albumEntry.Discs) > 1 {
								out += fmt.Sprintf(" (Disc %d)", result.discIndex+1)
							}
							return
						}

						cddb1 := utilities.NewCDDB1FromString(splits[2])

						toc := utilities.NewTOCFromCDDBString(strings.Join(splits[3:], " "))
						if cddb1 == 0 {
							writer.Write([]byte("500 Command syntax error\n.\n"))
							return
						}

						var entries []findByDurationResult
						if toc != nil {
							cddb1 = toc.GetCDDB1()
							entries = findByTOC(toc, 3, 10)
						} else {
							entries = findByCDDB1(cddb1, 10)
						}

						if len(entries) == 0 {
							writer.Write([]byte("202 No match found\n.\n"))
							return
						} else if len(entries) == 1 {
							writer.Write([]byte(fmt.Sprintf("200 %s\n", formatEntry(cddb1, entries[0]))))
							return
						} else {
							writer.Write([]byte("211 Found inexact matches list follows (until terminating marker `.')\n"))
							for _, e := range entries {
								writer.Write([]byte(fmt.Sprintf("%s\n", formatEntry(cddb1, e))))
							}
							writer.Write([]byte(".\n"))
							return
						}
					case "read":
						if len(splits) < 4 {
							writer.Write([]byte("500 Command syntax error\n.\n"))
							return
						}
						cddb1 := utilities.NewCDDB1FromString(splits[3])
						if cddb1 == 0 {
							writer.Write([]byte("500 Command syntax error\n.\n"))
							return
						}
						firstDigit := strings.IndexFunc(splits[2], unicode.IsNumber)
						if firstDigit == -1 {
							writer.Write([]byte("500 Command syntax error\n.\n"))
							return
						}

						params := strings.Split(splits[2][firstDigit:], "_")
						pageid, err := strconv.Atoi(params[0])
						var discIndex int
						if err != nil {
							writer.Write([]byte("500 Command syntax error\n.\n"))
							return
						}

						if len(params) > 1 {
							discIndex, err = strconv.Atoi(params[1])
							if err != nil {
								writer.Write([]byte("500 Command syntax error\n.\n"))
								return
							}
						}

						entry, ok := cdIndex[pageid]
						if !ok || len(entry.Discs) <= discIndex {
							writer.Write([]byte("401 Entry not found\n.\n"))
							return
						}

						var group string
						for _, a := range entry.Artists {
							if a.Position == "group" {
								group = a.Names[0]
								break
							}
						}
						writer.Write([]byte(fmt.Sprintf("210 %sSoundtrack %s\n", strings.ToUpper(entry.Type[0:1])+entry.Type[1:], cddb1.String())))
						writer.Write([]byte("# xmcd\n# Track frame offsets:\n"))
						for i := 0; i < entry.Discs[discIndex].TrackCount; i++ {
							writer.Write([]byte("#\t0\n"))
						}
						writer.Write([]byte(fmt.Sprintf("# Disc length: %d seconds\n#\n", entry.Duration)))
						writer.Write([]byte(fmt.Sprintf("DISCID=%s\n", cddb1.String())))
						writer.Write([]byte(fmt.Sprintf("DNUM=%d\n", len(entry.Discs))))
						writer.Write([]byte(fmt.Sprintf("DINDEX=%d\n", discIndex+1)))
						if len(entry.CatalogNumbers) > discIndex {
							writer.Write([]byte(fmt.Sprintf("DTITLE=%s / [%s] %s\n", group, entry.CatalogNumbers[discIndex], entry.MainTitle)))
						} else {
							writer.Write([]byte(fmt.Sprintf("DTITLE=%s / %s\n", group, entry.MainTitle)))
						}
						if !entry.ReleaseDate.IsZero() {
							writer.Write([]byte(fmt.Sprintf("DYEAR=%d\n", entry.ReleaseDate.Year())))
						}

						var genres []string
						if entry.Type == "arrangement" {
							genres = append(genres, entry.Type)
						}
						genres = append(genres, entry.Genre...)
						if len(genres) == 0 {
							genres = append(genres, "soundtrack")
						}
						writer.Write([]byte(fmt.Sprintf("DGENRE=%s\n", strings.Join(genres, ", "))))

						for i, t := range entry.Discs[discIndex].Tracks {
							writer.Write([]byte(fmt.Sprintf("TTITLE%d=%s\n", i, t.MainTitle)))
						}

						writer.Write([]byte(fmt.Sprintf("EXTD=https://en.touhouwiki.net/index.php?curid=%d\n", entry.Id)))

						for i, t := range entry.Discs[discIndex].Tracks {
							result := make(map[string][]string)
							for _, a := range t.Artists {
								if _, ok := result[a.Position]; !ok {
									result[a.Position] = []string{}
								}
								result[a.Position] = append(result[a.Position], strings.Join(a.Names, "/"))
							}
							var resultLines []string
							for k, v := range result {
								resultLines = append(resultLines, k+": "+strings.Join(v, ", "))
							}
							sort.SliceStable(resultLines, func(i, j int) bool {
								return resultLines[i] < resultLines[j]
							})
							writer.Write([]byte(fmt.Sprintf("EXTT%d=%s\n", i, strings.Join(resultLines, "\\n"))))
						}

						writer.Write([]byte("PLAYORDER=\n.\n"))
						return
					}
				}
				writer.Write([]byte("500 Unrecognized command\n.\n"))
			} else if request.URL.Path == "/search" {
				writer.Header().Set("Content-Type", "application/json")

				switch request.URL.Query().Get("type") {
				case "album": //search by title or catalog number, "exact" after normalization
					entries, ok := albumTitleLookup[normalizeSearchTitle(request.URL.Query().Get("query"))]

					if !ok {
						writer.Write([]byte("[]"))
					} else {
						jsonBytes, _ := json.Marshal(entries)

						writer.Write(jsonBytes)
					}
				case "loosealbum": //search by title or catalog number, loosely

					var entries []*albumEntry
					normalized := normalizeSearchTitle(request.URL.Query().Get("query"))
					for k, v := range albumTitleLookup {
						if strings.Index(k, normalized) != -1 {
							entries = append(entries, v...)
						}
					}

					sort.SliceStable(entries, func(i, j int) bool {
						return entries[i].Id < entries[j].Id
					})

					jsonBytes, _ := json.Marshal(entries)

					writer.Write(jsonBytes)
				default:
					writer.WriteHeader(http.StatusNotFound)
					writer.Write([]byte("[]"))
				}

			} else if strings.Index(request.URL.Path, "/lyrics/") == 0 {
				writer.Header().Set("Content-Type", "application/json")

				lyrics := parseLyrics(*servePath, wikitext_parser.NormalizeWikiTitle(strings.TrimPrefix(request.URL.Path, "/lyrics/")))
				if lyrics == nil {

					writer.WriteHeader(http.StatusNotFound)
					writer.Write([]byte("{}"))
					return
				}

				jsonBytes, _ := json.Marshal(lyrics)

				writer.Write(jsonBytes)
			} else if strings.Index(request.URL.Path, "/album/") == 0 {
				writer.Header().Set("Content-Type", "application/json")

				if pageid, err := strconv.Atoi(strings.TrimPrefix(request.URL.Path, "/album/")); err == nil {
					if album, ok := cdIndex[pageid]; ok {
						jsonBytes, _ := json.Marshal(album)

						writer.Write(jsonBytes)
						return
					}
				}

				writer.WriteHeader(http.StatusNotFound)
				writer.Write([]byte("{}"))
				return
			} else {
				filePath := path.Join(*servePath, strings.TrimLeft(request.URL.Path, "/"))
				if request.URL.Path == "/" {
					filePath = path.Join(*servePath, "README.md")
				}

				stat, err := os.Stat(filePath)
				if err != nil {
					http.NotFound(writer, request)
					return
				}

				if stat.IsDir() {
					writer.Header().Set("Content-Type", "application/json")
					type dirEntry struct {
						Name  string `json:"name"`
						Size  int64  `json:"size"`
						Mtime int64  `json:"mtime"`
						Link  string `json:"link,omitempty"`
						Type  string `json:"type"`
					}
					dirEntries := make([]dirEntry, 0, 1024)

					entries, _ := ioutil.ReadDir(filePath)
					for _, e := range entries {
						entry := dirEntry{
							Name:  e.Name(),
							Size:  e.Size(),
							Mtime: e.ModTime().UTC().Unix(),
						}
						if e.Mode()&fs.ModeSymlink > 0 {
							entry.Link, _ = os.Readlink(path.Join(filePath, entry.Name))
							entry.Type = "l"
						} else if e.IsDir() {
							entry.Type = "d"
						} else {
							entry.Type = "f"
						}
						dirEntries = append(dirEntries, entry)
					}

					writer.WriteHeader(http.StatusOK)
					byteEntries, _ := json.Marshal(dirEntries)
					writer.Write(byteEntries)
				} else {
					if path.Ext(filePath) == ".wiki" {
						writer.Header().Set("Content-Type", "text/plain")
					} else if path.Ext(filePath) == ".md" {
						writer.Header().Set("Content-Type", "text/markdown")
					}
					http.ServeFile(writer, request, filePath)
				}
			}
		}),
	}
	err := server.ListenAndServe()
	if err != nil {
		log.Panic(err)
	}
}