Added golang server with index files, querying, search by name+catalog, and CDDB emulator

This commit is contained in:
DataHoarder 2022-02-19 03:04:14 +01:00
parent 223ece654a
commit ff0f1e4758
6 changed files with 611 additions and 0 deletions

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
/.mirror.tmp.wiki
/.listing.tmp.json
/.idea

5
go.mod Normal file
View file

@ -0,0 +1,5 @@
module git.gammaspectra.live/S.O.N.G/touhouwiki-mirror
go 1.18
require golang.org/x/text v0.3.7

437
server.go Normal file
View file

@ -0,0 +1,437 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"git.gammaspectra.live/S.O.N.G/touhouwiki-mirror/utilities"
"git.gammaspectra.live/S.O.N.G/touhouwiki-mirror/wikiparser"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"golang.org/x/text/runes"
"golang.org/x/text/transform"
"golang.org/x/text/unicode/norm"
"io/fs"
"io/ioutil"
"log"
"net/http"
"os"
"path"
"strconv"
"strings"
"sync"
"time"
"unicode"
)
var arrangeCDIndex = make(map[int]*arrangeCdEntry)
var arrangeCDIndexLock sync.Mutex
var titleLookup = make(map[string][]*arrangeCdEntry)
var tracksLookup = make(map[int][]*arrangeCdEntry)
type categoryPageIndex struct {
PageId int `json:"pageid"`
Namespace int `json:"ns"`
Title string `json:"title"`
Timestamp string `json:"timestamp"`
}
func parseCategoryPageIndex(filePath string) []categoryPageIndex {
s := &struct {
Query struct {
Members []categoryPageIndex `json:"categorymembers"`
} `json:"query"`
}{}
fileData, err := ioutil.ReadFile(filePath)
if err != nil {
return nil
}
err = json.Unmarshal(fileData, s)
if err != nil {
return nil
}
return s.Query.Members
}
type arrangeCdEntry struct {
Id int `json:"pageid"`
MainTitle string `json:"pagetitle"`
Group string `json:"pagetitle,omitempty"`
Titles []string `json:"titles"`
CatalogNumber string `json:"catalognumber,omitempty"`
TrackCount int `json:"trackcount,omitempty"`
Duration int `json:"duration,omitempty"`
Year int `json:"year,omitempty"`
}
func processIndex(filePath string) {
arrangeIndexPath := path.Join(filePath, "pageindex", "Arrangement_CDs")
entries, err := ioutil.ReadDir(arrangeIndexPath)
if err != nil {
return
}
var wg sync.WaitGroup
for _, e := range entries {
if path.Ext(e.Name()) == ".json" {
for _, v := range parseCategoryPageIndex(path.Join(arrangeIndexPath, e.Name())) {
wg.Add(1)
go func(entry *arrangeCdEntry) {
defer wg.Done()
contents, err := ioutil.ReadFile(path.Join(filePath, "pages", fmt.Sprintf("%d.wiki", entry.Id)))
if err != nil {
return
}
result := wikiparser.ParseWikiText(string(contents))
if len(result) == 0 {
return
}
tpl, ok := result[0].(*wikiparser.Template)
if !ok {
return
}
if tpl.Name != "MusicArticle" {
return
}
var stringName string
title, ok := tpl.Parameters["titleen"]
if ok && len(title) > 0 {
stringName, ok = title[0].(string)
if ok && len(stringName) > 0 {
entry.Titles = append(entry.Titles, stringName)
}
}
title, ok = tpl.Parameters["titlejp"]
if ok && len(title) > 0 {
stringName, ok = title[0].(string)
if ok && len(stringName) > 0 {
entry.Titles = append(entry.Titles, stringName)
}
}
title, ok = tpl.Parameters["titlejprom"]
if ok && len(title) > 0 {
stringName, ok = title[0].(string)
if ok && len(stringName) > 0 {
entry.Titles = append(entry.Titles, stringName)
}
}
catalogNo, ok := tpl.Parameters["catalogno"]
if ok && len(catalogNo) > 0 {
stringName, ok = catalogNo[0].(string)
if ok && len(stringName) > 0 {
entry.CatalogNumber = stringName
}
}
groupCat, ok := tpl.Parameters["groupCat"]
if ok && len(groupCat) > 0 {
stringName, ok = groupCat[0].(string)
if ok && len(stringName) > 0 {
entry.Group = stringName
}
}
released, ok := tpl.Parameters["released"]
if ok && len(released) > 0 {
stringName, ok = released[0].(string)
if ok && len(released) > 0 {
releaseDate, err := time.ParseInLocation("2006-01-02", stringName, time.UTC)
if err == nil {
entry.Year = releaseDate.Year()
}
}
}
tracks, ok := tpl.Parameters["tracks"]
if ok && len(tracks) > 0 {
stringName, ok = tracks[0].(string)
if ok && len(stringName) > 0 {
entry.TrackCount, _ = strconv.Atoi(stringName)
}
}
length, ok := tpl.Parameters["length"]
if ok && len(length) > 0 {
stringName, ok = length[0].(string)
if ok && len(stringName) > 0 {
split := strings.Split(stringName, ":")
numbers := make([]int, len(split))
for i, val := range split {
numbers[i], _ = strconv.Atoi(val)
}
if len(numbers) == 3 {
entry.Duration = 3600*numbers[0] + 60*numbers[1] + numbers[2]
} else if len(numbers) == 2 {
entry.Duration = 60*numbers[0] + numbers[1]
} else if len(numbers) == 1 {
entry.Duration = numbers[0]
}
}
}
arrangeCDIndexLock.Lock()
arrangeCDIndex[entry.Id] = entry
if entry.TrackCount > 0 {
if _, ok := tracksLookup[entry.TrackCount]; !ok {
tracksLookup[entry.TrackCount] = []*arrangeCdEntry{entry}
} else {
tracksLookup[entry.TrackCount] = append(tracksLookup[entry.TrackCount], entry)
}
}
if len(entry.CatalogNumber) > 0 {
normalized := normalizeTitle(entry.CatalogNumber)
if _, ok := titleLookup[normalized]; !ok {
titleLookup[normalized] = []*arrangeCdEntry{entry}
} else {
titleLookup[normalized] = append(titleLookup[normalized], entry)
}
}
for _, title := range entry.Titles {
normalized := normalizeTitle(title)
if _, ok := titleLookup[normalized]; !ok {
titleLookup[normalized] = []*arrangeCdEntry{entry}
} else {
for _, val := range titleLookup[normalized] {
if val.Id == entry.Id {
goto exit
}
}
titleLookup[normalized] = append(titleLookup[normalized], entry)
exit:
}
}
defer arrangeCDIndexLock.Unlock()
}(&arrangeCdEntry{
Id: v.PageId,
MainTitle: v.Title,
})
}
}
}
wg.Wait()
}
var normalizeTransformer = transform.Chain(
norm.NFKD,
//width.Narrow,
runes.Remove(runes.In(unicode.Cc)),
runes.Remove(runes.In(unicode.Cf)),
runes.Remove(runes.In(unicode.Mn)),
runes.Remove(runes.In(unicode.Me)),
runes.Remove(runes.In(unicode.Mc)),
runes.Remove(runes.In(unicode.Po)),
runes.Remove(runes.In(unicode.Pe)),
runes.Remove(runes.In(unicode.Ps)),
runes.Remove(runes.In(unicode.Pf)),
runes.Remove(runes.In(unicode.Pi)),
runes.Remove(runes.In(unicode.Pd)),
runes.Remove(runes.In(unicode.Sc)),
runes.Remove(runes.In(unicode.Sk)),
runes.Remove(runes.In(unicode.Sm)),
runes.Remove(runes.In(unicode.So)),
runes.Remove(runes.In(unicode.Space)),
cases.Lower(language.Und),
norm.NFC,
)
func normalizeTitle(title string) (normalized string) {
normalized, _, _ = transform.String(normalizeTransformer, title)
return
}
func findByTracksAndDuration(tracks, duration, threshold int) (results []*arrangeCdEntry) {
for _, r := range tracksLookup[tracks] {
diff := r.Duration - duration
if diff < 0 {
diff = -diff
}
if diff <= threshold {
results = append(results, r)
}
}
return
}
func main() {
wd, _ := os.Getwd()
servePath := flag.String("path", wd, "Path that will be served by default")
flag.Parse()
processIndex(*servePath)
server := &http.Server{
Addr: ":8777",
Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path == "/cddb" {
//cddb emulator
writer.Header().Set("Content-Type", "text/plain; charset=UTF-8")
writer.WriteHeader(http.StatusOK)
cmd := request.URL.Query().Get("cmd")
if len(cmd) > 4 && cmd[0:4] == "cddb" {
splits := strings.Split(cmd, " ")
if len(splits) < 3 {
writer.Write([]byte("500 Unrecognized command\n.\n"))
return
}
switch splits[1] {
case "query":
cddb1 := utilities.NewCDDB1FromString(splits[2])
if cddb1 == 0 {
writer.Write([]byte("500 Command syntax error\n.\n"))
return
}
entries := findByTracksAndDuration(cddb1.GetTrackNumber(), int(cddb1.GetDuration().Seconds()), 10)
if len(entries) == 0 {
writer.Write([]byte("202 No match found\n.\n"))
return
} else if len(entries) == 1 {
if len(entries[0].CatalogNumber) > 0 {
writer.Write([]byte(fmt.Sprintf("200 Soundtrack%d %s [%s] %s / %s\n", entries[0].Id, cddb1.String(), entries[0].CatalogNumber, entries[0].Group, entries[0].MainTitle)))
} else {
writer.Write([]byte(fmt.Sprintf("200 Soundtrack%d %s %s / %s\n", entries[0].Id, cddb1.String(), entries[0].Group, entries[0].MainTitle)))
}
return
} else {
writer.Write([]byte("211 Found inexact matches list follows (until terminating marker `.')\n"))
for _, e := range entries {
if len(e.CatalogNumber) > 0 {
writer.Write([]byte(fmt.Sprintf("Soundtrack%d %s [%s] %s / %s\n", e.Id, cddb1.String(), e.CatalogNumber, e.Group, e.MainTitle)))
} else {
writer.Write([]byte(fmt.Sprintf("Soundtrack%d %s %s / %s\n", e.Id, cddb1.String(), e.Group, e.MainTitle)))
}
}
writer.Write([]byte(".\n"))
return
}
case "read":
if len(splits) < 4 {
writer.Write([]byte("500 Command syntax error\n.\n"))
return
}
cddb1 := utilities.NewCDDB1FromString(splits[3])
if cddb1 == 0 {
writer.Write([]byte("500 Command syntax error\n.\n"))
return
}
pageid, err := strconv.Atoi(strings.ReplaceAll(splits[2], "Soundtrack", ""))
if err != nil {
writer.Write([]byte("500 Command syntax error\n.\n"))
return
}
entry, ok := arrangeCDIndex[pageid]
if !ok {
writer.Write([]byte("401 Entry not found\n.\n"))
return
}
writer.Write([]byte(fmt.Sprintf("210 Soundtrack %s\n", cddb1.String())))
writer.Write([]byte("# xmcd\n# Track frame offsets:\n"))
for i := 0; i < entry.TrackCount; i++ {
writer.Write([]byte("#\t0\n"))
}
writer.Write([]byte(fmt.Sprintf("# Disc length: %d seconds\n#\n", entry.Duration)))
writer.Write([]byte(fmt.Sprintf("DISCID=%s\n", cddb1.String())))
if len(entry.CatalogNumber) > 0 {
writer.Write([]byte(fmt.Sprintf("DTITLE=%s / [%s] %s\n", entry.Group, entry.CatalogNumber, entry.MainTitle)))
} else {
writer.Write([]byte(fmt.Sprintf("DTITLE=%s / %s\n", entry.Group, entry.MainTitle)))
}
if entry.Year > 0 {
writer.Write([]byte(fmt.Sprintf("DYEAR=%d\nDGENRE=Soundtrack\n", entry.Year)))
}
for i := 0; i < entry.TrackCount; i++ {
writer.Write([]byte(fmt.Sprintf("TTITLE%d=\n", i)))
}
writer.Write([]byte(fmt.Sprintf("EXTD=https://en.touhouwiki.net/index.php?curid=%d\n", entry.Id)))
for i := 0; i < entry.TrackCount; i++ {
writer.Write([]byte(fmt.Sprintf("EXTT%d=\n", i)))
}
writer.Write([]byte("PLAYORDER=\n.\n"))
return
}
}
writer.Write([]byte("500 Unrecognized command\n.\n"))
} else if request.URL.Path == "/search" {
writer.Header().Set("Content-Type", "application/json")
switch request.URL.Query().Get("type") {
case "title": //search by title or catalog number
entries, ok := titleLookup[normalizeTitle(request.URL.Query().Get("query"))]
if !ok {
writer.Write([]byte("[]"))
} else {
jsonBytes, _ := json.MarshalIndent(entries, "", " ")
writer.Write(jsonBytes)
}
default:
writer.WriteHeader(http.StatusNotFound)
writer.Write([]byte("[]"))
}
} else {
filePath := path.Join(*servePath, strings.TrimLeft(request.URL.Path, "/"))
stat, err := os.Stat(filePath)
if err != nil {
http.NotFound(writer, request)
return
}
if stat.IsDir() {
writer.Header().Set("Content-Type", "application/json")
type dirEntry struct {
Name string `json:"name"`
Size int64 `json:"size"`
Mtime int64 `json:"mtime"`
Link string `json:"link,omitempty"`
Type string `json:"type"`
}
dirEntries := make([]dirEntry, 0, 1024)
entries, _ := ioutil.ReadDir(filePath)
for _, e := range entries {
entry := dirEntry{
Name: e.Name(),
Size: e.Size(),
Mtime: e.ModTime().UTC().Unix(),
}
if e.Mode()&fs.ModeSymlink > 0 {
entry.Link, _ = os.Readlink(path.Join(filePath, entry.Name))
entry.Type = "l"
} else if e.IsDir() {
entry.Type = "d"
} else {
entry.Type = "f"
}
dirEntries = append(dirEntries, entry)
}
writer.WriteHeader(http.StatusOK)
byteEntries, _ := json.MarshalIndent(dirEntries, "", " ")
writer.Write(byteEntries)
} else {
http.ServeFile(writer, request, filePath)
}
}
}),
}
err := server.ListenAndServe()
if err != nil {
log.Panic(err)
}
}

35
utilities/cddb1.go Normal file
View file

@ -0,0 +1,35 @@
package utilities
import (
"encoding/binary"
"encoding/hex"
"time"
)
type CDDB1 uint32
func (c CDDB1) GetStartTimeChecksum() int {
return int((c >> 24) & 0xFF)
}
func (c CDDB1) GetDuration() time.Duration {
return time.Second * time.Duration((c>>8)&0xFFFF)
}
func (c CDDB1) GetTrackNumber() int {
return int(c & 0xFF)
}
func NewCDDB1FromString(cddb1 string) CDDB1 {
b, err := hex.DecodeString(cddb1)
if err != nil {
return 0
}
return CDDB1(binary.BigEndian.Uint32(b))
}
func (c CDDB1) String() string {
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, uint32(c))
return hex.EncodeToString(b)
}

107
wikiparser/parser.go Normal file
View file

@ -0,0 +1,107 @@
package wikiparser
import "strings"
func NormalizeWikiTitle(title string) string {
return strings.Replace(title, " ", "_", -1)
}
//ParseWikiText small WikiText parser that extracts text, Templates, and its arguments/parameters
func ParseWikiText(text string) (result []interface{}) {
index := 0
for index < len(text) {
templateIndex := strings.Index(text[index:], "{{")
if templateIndex == -1 {
t := strings.TrimSpace(text[index:])
if len(t) > 0 {
result = append(result, t)
}
break
} else {
t := strings.TrimSpace(text[index : index+templateIndex])
if len(t) > 0 {
result = append(result, t)
}
var tpl *Template
index, tpl = ParseTemplate(text, index+templateIndex+2, 0)
if tpl != nil {
result = append(result, tpl)
}
}
}
return
}
func ParseTemplate(text string, index int, depth int) (i int, template *Template) {
var c byte
lastToken := index
var key string
addValue := func() int {
if lastToken < len(text) && i-lastToken > 0 {
t := strings.TrimSpace(text[lastToken:i])
if len(t) > 0 {
if template == nil {
template = NewTemplate(t)
} else {
if key == "" {
template.AddParameterUnkeyed(t)
} else {
template.AddParameter(key, t)
}
}
}
return len(t)
}
return 0
}
addKey := func() {
if lastToken < len(text) && i-lastToken > 0 {
t := strings.TrimSpace(text[lastToken:i])
if len(t) > 0 {
key = t
}
}
}
for i = index; i < len(text); i++ {
c = text[i]
if c == '}' && i < len(text)-1 && text[i+1] == '}' {
addValue()
i += 2
break
} else if c == '{' && i < len(text)-1 && text[i+1] == '{' {
addValue()
var tpl *Template
i, tpl = ParseTemplate(text, i+2, depth+1)
if tpl != nil {
if key == "" {
template.AddParameterUnkeyed(tpl)
} else {
template.AddParameter(key, tpl)
}
}
lastToken = i
} else if c == '|' {
addValue()
lastToken = i + 1
key = ""
} else if c == '\n' {
addValue()
lastToken = i + 1
} else if c == '=' {
if key == "" {
addKey()
lastToken = i + 1
}
}
}
return
}

26
wikiparser/template.go Normal file
View file

@ -0,0 +1,26 @@
package wikiparser
import "fmt"
type Template struct {
Name string
Parameters map[string][]interface{}
}
func NewTemplate(name string) *Template {
return &Template{
Name: name,
Parameters: make(map[string][]interface{}),
}
}
func (t *Template) AddParameterUnkeyed(value interface{}) {
t.Parameters[fmt.Sprintf("%d", len(t.Parameters))] = []interface{}{value}
}
func (t *Template) AddParameter(key string, value interface{}) {
if _, ok := t.Parameters[key]; !ok {
t.Parameters[key] = make([]interface{}, 0, 1)
}
t.Parameters[key] = append(t.Parameters[key], value)
}