package en_touhouwiki_net import ( "encoding/json" "fmt" "git.gammaspectra.live/S.O.N.G/METANOIA/metadata" "git.gammaspectra.live/S.O.N.G/METANOIA/utilities" wikitext_parser "git.gammaspectra.live/S.O.N.G/wikitext-parser" "github.com/oriser/regroup" "io/ioutil" "log" "net/http" "net/url" "strings" "time" ) var baseURL = "https://en.touhouwiki.net/" var baseAPIURL = "https://touhouwiki-mirror.gammaspectra.live/" type Source struct { client *metadata.CachingClient } func NewSource() *Source { s := &Source{} s.client = metadata.NewCachingClient(s.GetURL(), time.Second/10) return s } func (s *Source) GetName() string { return "Touhou Wiki" } func (s *Source) GetURL() string { return baseURL } func (s *Source) GetLicense() metadata.License { return metadata.License{ Code: metadata.CC_BY_SA_40, URL: baseURL + "wiki/Touhou_Wiki:Copyrights#Content_licensing", Attribution: fmt.Sprintf("%s (%s)", s.GetName(), s.GetURL()), } } func (s *Source) FindByTOC(toc metadata.TOC) []*metadata.Album { return s.QueryCDDB(toc, toc.GetCDDB1()) } func (s *Source) FindByCDDB1(cddb metadata.CDDB1) []*metadata.Album { return s.QueryCDDB(nil, cddb) } func (s *Source) QueryCDDB(toc metadata.TOC, cddb metadata.CDDB1) (results []*metadata.Album) { uri, _ := url.Parse(baseAPIURL) uri.Path += "cddb" query := uri.Query() if len(toc) > 1 { query.Add("cmd", fmt.Sprintf("cddb query %s", toc.CDDBString())) } else { query.Add("cmd", fmt.Sprintf("cddb query %s", cddb.String())) } query.Add("hello", "anonymous localhost METANOIA "+utilities.Version) query.Add("proto", "6") uri.RawQuery = query.Encode() response, err := s.client.Request(&http.Request{ Method: "GET", URL: uri, }, time.Hour*24*14) if err != nil { return nil } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { return nil } matchLineRE := regroup.MustCompile(`(?m)^(200 )?(?P[A-Za-z][A-Za-z0-9_]+) (?P[a-f0-9]{8}) (?P.+)$`) matchURLRE := regroup.MustCompile(`(?m)^EXTD=https://en.touhouwiki.net/index.php\?curid=(?P[0-9]+)$`) type matchResult struct { Group string `regroup:"group"` CDDB1 string `regroup:"cddb1"` Match string `regroup:"match"` } m := &matchResult{} rets, err := matchLineRE.MatchAllToTarget(string(body), -1, m) if err == nil { for _, elem := range rets { match := elem.(*matchResult) uri, _ := url.Parse(baseAPIURL) uri.Path += "cddb" query := uri.Query() query.Add("cmd", fmt.Sprintf("cddb read %s %s", match.Group, match.CDDB1)) query.Add("hello", "anonymous localhost METANOIA "+utilities.Version) query.Add("proto", "6") uri.RawQuery = query.Encode() response, err := s.client.Request(&http.Request{ Method: "GET", URL: uri, }, time.Hour*24*30) if err != nil { continue } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { continue } t := &struct { AlbumId int `regroup:"album"` }{} if matchURLRE.MatchToTarget(string(body), t) == nil { a := s.GetAlbumInformation(t.AlbumId) if a != nil { results = append(results, s.GetAlbumInformation(t.AlbumId)) } } } } return } func (s *Source) FindByAlbumNames(names []metadata.Name) (r []*metadata.Album) { for _, n := range names { for _, result := range s.FindQueryArguments(n.Name, false) { func() { for _, rr := range r { if rr.SourceUniqueIdentifier == result.SourceUniqueIdentifier { return } } r = append(r, result) }() } } return } func (s *Source) FindByCatalogNumber(catalog metadata.CatalogNumber) (r []*metadata.Album) { return s.FindQueryArguments(string(catalog), false) } type albumEntry struct { Id int `json:"pageid"` Type string `json:"type"` MainTitle string `json:"pagetitle"` Titles map[string]string `json:"titles"` Image string `json:"image"` CatalogNumbers []string `json:"catalognumbers"` Genre []string `json:"genre"` TrackCount int `json:"trackcount"` Duration int `json:"duration"` ReleaseDate string `json:"releasedate"` ReleaseEvent string `json:"releaseevent"` Links []string `json:"links"` Artists []artistEntry `json:"artists"` Discs []discEntry `json:"discs"` } type discEntry struct { Name string `json:"name"` Duration int `json:"duration"` TrackCount int `json:"trackcount"` Tracks []trackEntry `json:"tracks"` } type trackEntry struct { Duration int `json:"duration"` MainTitle string `json:"title"` Titles map[string]string `json:"titles"` Artists []artistEntry `json:"artists"` Lyrics string `json:"lyrics"` Links []string `json:"links"` Original []string `json:"original"` } type artistEntry struct { Position string `json:"position"` Names []string `json:"names"` } func (s *Source) FindQueryArguments(queryArgs string, loose bool) (albums []*metadata.Album) { uri, _ := url.Parse(baseAPIURL) uri.Path += "search" query := uri.Query() if loose { query.Add("type", "loosealbum") } else { query.Add("type", "album") } query.Add("query", queryArgs) uri.RawQuery = query.Encode() response, err := s.client.Request(&http.Request{ Method: "GET", URL: uri, }, time.Hour*24*14) if err != nil { return nil } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { return nil } var result []*albumEntry err = json.Unmarshal(body, &result) if err != nil { return nil } for _, r := range result { album := s.ParseAlbumInformation(r) if album != nil { albums = append(albums, album) } } return } type lyricsEntry struct { MainTitle string `json:"pagetitle"` Titles []string `json:"titles"` Links []string `json:"links"` Duration int `json:"duration"` Artists []artistEntry `json:"artists"` Entries struct { Kanji []string `json:"kanji"` Romaji []string `json:"romaji"` English []string `json:"english"` } `json:"entries"` } func (s *Source) GetSongLyrics(lyricsName string) (result []*metadata.TextLyrics) { uri, _ := url.Parse(baseAPIURL) uri.Path += "lyrics/" + wikitext_parser.NormalizeWikiTitle(lyricsName) response, err := s.client.Request(&http.Request{ Method: "GET", URL: uri, }, time.Hour*24*120) if err != nil { return nil } defer response.Body.Close() if response.StatusCode != http.StatusOK { return nil } body, err := ioutil.ReadAll(response.Body) if err != nil { return nil } l := &lyricsEntry{} err = json.Unmarshal(body, &l) if err != nil || l.MainTitle == "" { return nil } identifiers := []metadata.Name{ {Kind: "url", Name: baseURL + "wiki/" + wikitext_parser.NormalizeWikiTitle("Lyrics: "+l.MainTitle)}, } if len(l.Entries.Kanji) > 0 { result = append(result, &metadata.TextLyrics{ Language: "original", //TODO: detect original language Identifiers: identifiers, Entries: l.Entries.Kanji, }) } if len(l.Entries.Romaji) > 0 { result = append(result, &metadata.TextLyrics{ Language: "romaji", Identifiers: identifiers, Entries: l.Entries.Romaji, }) } if len(l.Entries.English) > 0 { result = append(result, &metadata.TextLyrics{ Language: "english", Identifiers: identifiers, Entries: l.Entries.English, }) } return } var staffNameGroupRE = regroup.MustCompile(`(?P.+)\((?P.+)\)`) func (s *Source) GetAlbumInformation(id int) *metadata.Album { uri, _ := url.Parse(baseAPIURL) uri.Path += fmt.Sprintf("album/%d", id) response, err := s.client.Request(&http.Request{ Method: "GET", URL: uri, }, time.Hour*24*14) if err != nil { return nil } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { return nil } result := &albumEntry{} err = json.Unmarshal(body, &result) if err != nil { return nil } return s.ParseAlbumInformation(result) } func (s *Source) ParseAlbumInformation(a *albumEntry) (album *metadata.Album) { if a.MainTitle == "" { return } album = &metadata.Album{ License: s.GetLicense(), SourceUniqueIdentifier: baseURL + "wiki/" + wikitext_parser.NormalizeWikiTitle(a.MainTitle), Identifiers: []metadata.Name{ { Kind: "url", Name: baseURL + "wiki/" + wikitext_parser.NormalizeWikiTitle(a.MainTitle), }, { Kind: "url", Name: fmt.Sprintf("%salbum/%d", baseAPIURL, a.Id), }, }, } if a.Type == "arrangement" { album.Tags = append(album.Tags, metadata.Name{ Kind: "genre", Name: "remix", }) } for k, t := range a.Titles { album.Name = append(album.Name, metadata.Name{ Kind: k, Name: t, }) } if len(a.Image) > 0 { album.Art = append(album.Art, metadata.Name{ Kind: "front", Name: baseAPIURL + "pages_by_name/File:" + wikitext_parser.NormalizeWikiTitle(a.Image), }) } for _, tag := range a.Genre { album.Tags = append(album.Tags, metadata.Name{ Kind: "genre", Name: tag, }) } for _, catno := range a.CatalogNumbers { album.Identifiers = append(album.Identifiers, metadata.Name{ Kind: "catalog", Name: catno, }) } if len(a.ReleaseEvent) > 0 { album.Links = append(album.Links, metadata.Link{ Kind: "release", Name: []metadata.Name{ {Kind: "name", Name: a.ReleaseEvent}, }, }) } for _, l := range a.Links { album.Links = append(album.Links, metadata.Link{ Kind: "official", Name: []metadata.Name{{Kind: "url", Name: l}}, }) } for _, artist := range a.Artists { role := metadata.Role{ Kind: artist.Position, } for _, sValue := range artist.Names { m := &struct { Position string `regroup:"position"` Name string `regroup:"name"` }{} if staffNameGroupRE.MatchToTarget(strings.TrimSpace(sValue), m) == nil { //Is a group role.Name = []metadata.Name{{Kind: "original", Name: strings.TrimSpace(m.Position)}} role.Group = strings.TrimSpace(m.Name) } else { role.Name = []metadata.Name{{Kind: "original", Name: sValue}} } } album.Roles = append(album.Roles, role) } for _, d := range a.Discs { disc := metadata.Disc{} if len(d.Name) > 0 { disc.Name = []metadata.Name{{Kind: "original", Name: d.Name}} } for _, t := range d.Tracks { track := metadata.Track{} for k, title := range t.Titles { track.Name = append(track.Name, metadata.Name{ Kind: k, Name: title, }) } track.Duration = time.Second * time.Duration(t.Duration) for _, artist := range t.Artists { role := metadata.Role{ Kind: artist.Position, } for _, sValue := range artist.Names { m := &struct { Position string `regroup:"position"` Name string `regroup:"name"` }{} if staffNameGroupRE.MatchToTarget(strings.TrimSpace(sValue), m) == nil { //Is a group role.Name = []metadata.Name{{Kind: "original", Name: strings.TrimSpace(m.Position)}} role.Group = strings.TrimSpace(m.Name) } else { role.Name = []metadata.Name{{Kind: "original", Name: sValue}} } } track.Roles = append(track.Roles, role) } if len(t.Original) > 0 { for _, o := range t.Original { if strings.Index(o, "Album: ") == 0 { track.Links = append(track.Links, metadata.Link{ Kind: "original release album", Name: []metadata.Name{{Kind: "original", Name: strings.TrimPrefix(o, "Album: ")}}, }) } else { track.Links = append(track.Links, metadata.Link{ Kind: "original release title", Name: []metadata.Name{{Kind: "original", Name: o}}, }) } } } for _, l := range t.Links { track.Links = append(track.Links, metadata.Link{ Kind: "official", Name: []metadata.Name{{Kind: "url", Name: l}}, }) } if len(t.Lyrics) > 0 { lyricsPage := t.Lyrics track.Lyrics = func() (result []metadata.Lyrics) { for _, l := range s.GetSongLyrics(lyricsPage) { result = append(result, l) } return } } else { track.Lyrics = func() []metadata.Lyrics { return nil } } disc.Tracks = append(disc.Tracks, track) } album.Discs = append(album.Discs, disc) } album.ReleaseDate, _ = time.ParseInLocation("2006-01-02", a.ReleaseDate, time.UTC) return } func (s *Source) Test() { albumC := s.FindByCDDB1(0x730dec08) albums := s.FindByAlbumNames([]metadata.Name{{Name: "Bayside Beat"}}) album := s.GetAlbumInformation(54742) lyrics := album.Discs[0].Tracks[3].Lyrics() log.Print(album) log.Print(albumC) log.Print(lyrics) log.Print(albums) }