586 lines
14 KiB
Go
586 lines
14 KiB
Go
package vgmdb_net
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"git.gammaspectra.live/S.O.N.G/METANOIA/metadata"
|
|
"git.gammaspectra.live/S.O.N.G/METANOIA/utilities"
|
|
"github.com/oriser/regroup"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
var baseURL = "https://vgmdb.net/"
|
|
var baseAPIURL = "https://vgmdb.info/"
|
|
|
|
type Source struct {
|
|
client *metadata.CachingClient
|
|
}
|
|
|
|
func NewSource() *Source {
|
|
s := &Source{}
|
|
s.client = metadata.NewCachingClient(s.GetURL(), time.Second/10)
|
|
return s
|
|
}
|
|
|
|
func (s *Source) GetName() string {
|
|
return "VGMdb"
|
|
}
|
|
|
|
func (s *Source) GetURL() string {
|
|
return baseURL
|
|
}
|
|
|
|
func (s *Source) GetLicense() metadata.License {
|
|
return metadata.License{
|
|
Code: metadata.Unknown,
|
|
URL: "",
|
|
Attribution: fmt.Sprintf("%s (%s)", s.GetName(), s.GetURL()),
|
|
}
|
|
}
|
|
|
|
func (s *Source) FindByTOC(toc metadata.TOC) (results []*metadata.Album) {
|
|
uri, _ := url.Parse(baseURL)
|
|
uri.Path += "cddb"
|
|
|
|
query := uri.Query()
|
|
query.Add("cmd", fmt.Sprintf("cddb query %s", toc.CDDBString()))
|
|
query.Add("hello", "anonymous localhost METANOIA "+utilities.Version)
|
|
query.Add("proto", "6")
|
|
|
|
uri.RawQuery = query.Encode()
|
|
response, err := s.client.Request(&http.Request{
|
|
Method: "GET",
|
|
URL: uri,
|
|
}, time.Hour*24*14)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer response.Body.Close()
|
|
body, err := ioutil.ReadAll(response.Body)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
matchLineRE := regroup.MustCompile(`(?m)^(200 )?(?P<group>[A-Za-z][A-Za-z0-9]+) (?P<cddb1>[a-f0-9]{8}) (?P<match>.+)$`)
|
|
matchURLRE := regroup.MustCompile(`(?m)^EXTD=https://vgmdb.net/album/(?P<album>[0-9]+)$`)
|
|
type matchResult struct {
|
|
Group string `regroup:"group"`
|
|
CDDB1 string `regroup:"cddb1"`
|
|
Match string `regroup:"match"`
|
|
}
|
|
m := &matchResult{}
|
|
rets, err := matchLineRE.MatchAllToTarget(string(body), -1, m)
|
|
if err == nil {
|
|
|
|
for _, elem := range rets {
|
|
match := elem.(*matchResult)
|
|
|
|
uri, _ := url.Parse(baseURL)
|
|
uri.Path += "cddb"
|
|
|
|
query := uri.Query()
|
|
query.Add("cmd", fmt.Sprintf("cddb read %s %s", match.Group, match.CDDB1))
|
|
query.Add("hello", "anonymous localhost METANOIA "+utilities.Version)
|
|
query.Add("proto", "6")
|
|
|
|
uri.RawQuery = query.Encode()
|
|
response, err := s.client.Request(&http.Request{
|
|
Method: "GET",
|
|
URL: uri,
|
|
}, time.Hour*24*30)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
defer response.Body.Close()
|
|
body, err := ioutil.ReadAll(response.Body)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
t := &struct {
|
|
AlbumId int `regroup:"album"`
|
|
}{}
|
|
if matchURLRE.MatchToTarget(string(body), t) == nil {
|
|
results = mergeByAlbumUniqueIdentifier(results, s.GetAlbumInformation(t.AlbumId))
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func mergeByAlbumUniqueIdentifier(existing []*metadata.Album, albums ...*metadata.Album) []*metadata.Album {
|
|
for _, r := range albums {
|
|
if r == nil {
|
|
continue
|
|
}
|
|
exists := false
|
|
for _, m := range existing {
|
|
if m.SourceUniqueIdentifier == r.SourceUniqueIdentifier {
|
|
exists = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if !exists {
|
|
existing = append(existing, r)
|
|
}
|
|
}
|
|
|
|
return existing
|
|
}
|
|
|
|
func (s *Source) FindByAlbumNames(names []metadata.Name) (results []*metadata.Album) {
|
|
|
|
for _, v := range names {
|
|
results = mergeByAlbumUniqueIdentifier(results, s.FindQueryArguments(v.Name)...)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (s *Source) FindByCatalogNumber(catalog metadata.CatalogNumber) (results []*metadata.Album) {
|
|
results = s.FindQueryArguments(string(catalog))
|
|
if strings.Index(string(catalog), "-") != -1 {
|
|
results = mergeByAlbumUniqueIdentifier(results, s.FindQueryArguments(strings.Replace(string(catalog), "-", "", -1))...)
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
type namesEntry struct {
|
|
English string `json:"en"`
|
|
Japanese string `json:"ja"`
|
|
Romaji string `json:"ja-latn"`
|
|
}
|
|
type titlesEntry struct {
|
|
English string `json:"English"`
|
|
Japanese string `json:"Japanese"`
|
|
Romaji string `json:"Romaji"`
|
|
}
|
|
|
|
type entityEntry struct {
|
|
Link string `json:"link"`
|
|
Names namesEntry `json:"names"`
|
|
}
|
|
|
|
type imageEntry struct {
|
|
Name string `json:"name"`
|
|
Full string `json:"full"`
|
|
Medium string `json:"medium"`
|
|
Thumb string `json:"thumb"`
|
|
}
|
|
|
|
type trackEntry struct {
|
|
Names titlesEntry `json:"names"`
|
|
Duration string `json:"track_length"`
|
|
}
|
|
|
|
type discEntry struct {
|
|
Length string `json:"disc_length"`
|
|
Name string `json:"name"`
|
|
Tracks []trackEntry `json:"tracks"`
|
|
}
|
|
|
|
func (s *Source) FindQueryArguments(queryArgs string) (albums []*metadata.Album) {
|
|
|
|
uri, _ := url.Parse(baseAPIURL)
|
|
uri.Path += "search/albums"
|
|
|
|
query := uri.Query()
|
|
query.Add("q", queryArgs)
|
|
query.Add("format", "json")
|
|
|
|
uri.RawQuery = query.Encode()
|
|
response, err := s.client.Request(&http.Request{
|
|
Method: "GET",
|
|
URL: uri,
|
|
}, time.Hour*24*14)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer response.Body.Close()
|
|
body, err := ioutil.ReadAll(response.Body)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
type SearchResult struct {
|
|
Link string `json:"link"`
|
|
Meta map[string]interface{} `json:"meta"`
|
|
Query string `json:"query"`
|
|
Results struct {
|
|
Albums []struct {
|
|
Catalog string `json:"catalog"`
|
|
Category string `json:"category"`
|
|
Link string `json:"link"`
|
|
MediaFormat string `json:"media_format"`
|
|
ReleaseDate string `json:"release_date"`
|
|
Titles namesEntry `json:"titles"`
|
|
} `json:"albums"`
|
|
} `json:"results"`
|
|
}
|
|
|
|
result := &SearchResult{}
|
|
|
|
err = json.Unmarshal(body, result)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
for _, r := range result.Results.Albums {
|
|
|
|
parts := strings.Split(r.Link, "/")
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
albumId, _ := strconv.Atoi(parts[1])
|
|
if albumId == 0 {
|
|
continue
|
|
}
|
|
album := s.GetAlbumInformation(albumId)
|
|
if album != nil {
|
|
albums = append(albums, album)
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (s *Source) GetAlbumInformation(id int) *metadata.Album {
|
|
uri, _ := url.Parse(baseAPIURL)
|
|
uri.Path += fmt.Sprintf("album/%d", id)
|
|
|
|
query := uri.Query()
|
|
query.Add("format", "json")
|
|
uri.RawQuery = query.Encode()
|
|
response, err := s.client.Request(&http.Request{
|
|
Method: "GET",
|
|
URL: uri,
|
|
}, time.Hour*24*60)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer response.Body.Close()
|
|
body, err := ioutil.ReadAll(response.Body)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
result := &struct {
|
|
Arrangers []entityEntry `json:"arrangers"`
|
|
Catalog string `json:"catalog"`
|
|
Categories []string `json:"categories"`
|
|
Category string `json:"category"`
|
|
Classification string `json:"classification"`
|
|
Composers []entityEntry `json:"composers"`
|
|
Covers []imageEntry `json:"covers"`
|
|
Discs []discEntry `json:"discs"`
|
|
Link string `json:"link"`
|
|
Lyricists []entityEntry `json:"lyricists"`
|
|
MediaFormat string `json:"media_format"`
|
|
Meta struct {
|
|
AddedDate string `json:"added_date"`
|
|
EditedDate string `json:"edited_date"`
|
|
FetchedDate string `json:"fetched_date"`
|
|
TTL int `json:"ttl"`
|
|
Visitors int `json:"visitors"`
|
|
}
|
|
Name string `json:"name"`
|
|
Names namesEntry `json:"names"`
|
|
Notes string `json:"notes"`
|
|
Organizations []struct {
|
|
Link string `json:"link"`
|
|
Role string `json:"role"`
|
|
Names namesEntry `json:"names"`
|
|
} `json:"organizations"`
|
|
Performers []entityEntry `json:"performers"`
|
|
PictureFull string `json:"picture_full"`
|
|
PictureMedium string `json:"picture_medium"`
|
|
PictureThumb string `json:"picture_thumb"`
|
|
Platforms []string `json:"platforms"`
|
|
Products []entityEntry `json:"products"`
|
|
PublishFormat string `json:"publish_format"`
|
|
Publisher entityEntry `json:"publisher"`
|
|
Related []struct {
|
|
Catalog string `json:"catalog"`
|
|
Link string `json:"link"`
|
|
Names namesEntry `json:"names"`
|
|
Type string `json:"type"`
|
|
} `json:"related"`
|
|
ReleaseDate string `json:"release_date"`
|
|
ReleaseEvents []struct {
|
|
Link string `json:"link"`
|
|
Name string `json:"name"`
|
|
ShortName string `json:"shortname"`
|
|
} `json:"release_events"`
|
|
ReleasePrice struct {
|
|
Currency string `json:"currency"`
|
|
Price float64 `json:"price"`
|
|
} `json:"release_price"`
|
|
//reprints
|
|
Votes int `json:"votes"`
|
|
Websites map[string][]struct {
|
|
Link string `json:"link"`
|
|
Name string `json:"name"`
|
|
} `json:"websites"`
|
|
}{}
|
|
|
|
err = json.Unmarshal(body, result)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
album := &metadata.Album{
|
|
License: s.GetLicense(),
|
|
SourceUniqueIdentifier: baseURL + fmt.Sprintf("album/%d", id),
|
|
}
|
|
|
|
getNamesFromNameEntry := func(entry namesEntry) (results map[string]string) {
|
|
results = make(map[string]string)
|
|
if len(entry.English) > 0 {
|
|
results["english"] = entry.English
|
|
if len(entry.Japanese) > 0 && entry.Japanese != entry.English {
|
|
results["japanese"] = entry.Japanese
|
|
}
|
|
if len(entry.Romaji) > 0 && entry.Romaji != entry.English && entry.Japanese != entry.Romaji {
|
|
results["romaji"] = entry.Romaji
|
|
}
|
|
} else if len(entry.Japanese) > 0 {
|
|
if entry.Romaji == entry.Japanese {
|
|
results["romaji"] = entry.Romaji
|
|
} else {
|
|
results["japanese"] = entry.Japanese
|
|
if len(entry.Romaji) > 0 && entry.Romaji != entry.Japanese {
|
|
results["romaji"] = entry.Romaji
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
addEntityRoles := func(roleName string, entities []entityEntry) {
|
|
for _, r := range entities {
|
|
role := metadata.Role{
|
|
Kind: roleName,
|
|
}
|
|
for k, n := range getNamesFromNameEntry(r.Names) {
|
|
role.Name = append(role.Name, metadata.Name{
|
|
Kind: k,
|
|
Name: n,
|
|
})
|
|
}
|
|
|
|
if len(r.Link) > 0 {
|
|
role.Name = append(role.Name, metadata.Name{
|
|
Kind: "url",
|
|
Name: baseURL + r.Link,
|
|
})
|
|
}
|
|
album.Roles = append(album.Roles, role)
|
|
}
|
|
}
|
|
|
|
album.Name = append(album.Name, metadata.Name{
|
|
Kind: "original",
|
|
Name: result.Name,
|
|
})
|
|
|
|
for k, n := range getNamesFromNameEntry(result.Names) {
|
|
if result.Name != n {
|
|
album.Name = append(album.Name, metadata.Name{
|
|
Kind: k,
|
|
Name: n,
|
|
})
|
|
}
|
|
}
|
|
|
|
addEntityRoles("arranger", result.Arrangers)
|
|
addEntityRoles("composer", result.Composers)
|
|
addEntityRoles("lyrics", result.Lyricists)
|
|
addEntityRoles("performer", result.Performers)
|
|
|
|
for _, o := range result.Organizations {
|
|
org := metadata.Role{
|
|
Kind: o.Role,
|
|
}
|
|
for k, n := range getNamesFromNameEntry(o.Names) {
|
|
org.Name = append(org.Name, metadata.Name{
|
|
Kind: k,
|
|
Name: n,
|
|
})
|
|
}
|
|
if len(o.Link) > 0 {
|
|
org.Name = append(org.Name, metadata.Name{
|
|
Kind: "url",
|
|
Name: baseURL + o.Link,
|
|
})
|
|
}
|
|
album.Roles = append(album.Roles, org)
|
|
}
|
|
|
|
publisher := metadata.Role{
|
|
Kind: "publisher",
|
|
}
|
|
for k, n := range getNamesFromNameEntry(result.Publisher.Names) {
|
|
publisher.Name = append(publisher.Name, metadata.Name{
|
|
Kind: k,
|
|
Name: n,
|
|
})
|
|
}
|
|
if len(result.Publisher.Link) > 0 {
|
|
publisher.Name = append(publisher.Name, metadata.Name{
|
|
Kind: "url",
|
|
Name: baseURL + result.Publisher.Link,
|
|
})
|
|
}
|
|
if len(publisher.Name) > 0 {
|
|
album.Roles = append(album.Roles, publisher)
|
|
}
|
|
|
|
if len(result.Catalog) > 0 {
|
|
album.Identifiers = append(album.Identifiers, metadata.Name{
|
|
Kind: "catalog",
|
|
Name: result.Catalog,
|
|
})
|
|
}
|
|
|
|
if len(result.Link) > 0 {
|
|
album.Identifiers = append(album.Identifiers, metadata.Name{
|
|
Kind: "url",
|
|
Name: baseURL + result.Link,
|
|
})
|
|
}
|
|
|
|
for _, e := range result.ReleaseEvents {
|
|
album.Links = append(album.Links, metadata.Link{
|
|
Kind: "release",
|
|
Name: []metadata.Name{
|
|
{Kind: "url", Name: baseURL + e.Link},
|
|
{Kind: "name", Name: e.Name},
|
|
},
|
|
})
|
|
}
|
|
|
|
for _, c := range result.Covers {
|
|
if len(c.Full) > 0 {
|
|
album.Art = append(album.Art, metadata.Name{
|
|
Kind: strings.ToLower(c.Name),
|
|
Name: c.Full,
|
|
})
|
|
}
|
|
}
|
|
|
|
for _, g := range strings.Split(result.Classification, ",") {
|
|
n := strings.ToLower(strings.TrimSpace(g))
|
|
if len(n) > 0 {
|
|
album.Tags = append(album.Tags, metadata.Name{
|
|
Kind: "genre",
|
|
Name: n,
|
|
})
|
|
}
|
|
}
|
|
|
|
for _, p := range result.Products {
|
|
product := metadata.Link{
|
|
Kind: "work",
|
|
}
|
|
for k, n := range getNamesFromNameEntry(p.Names) {
|
|
product.Name = append(product.Name, metadata.Name{
|
|
Kind: k,
|
|
Name: n,
|
|
})
|
|
}
|
|
if len(p.Link) > 0 {
|
|
product.Name = append(product.Name, metadata.Name{
|
|
Kind: "url",
|
|
Name: baseURL + p.Link,
|
|
})
|
|
}
|
|
album.Links = append(album.Links, product)
|
|
}
|
|
|
|
for k, l := range result.Websites {
|
|
for _, link := range l {
|
|
album.Links = append(album.Links, metadata.Link{
|
|
Kind: strings.ToLower(k),
|
|
Name: []metadata.Name{{Kind: "url", Name: link.Link}},
|
|
})
|
|
}
|
|
}
|
|
|
|
for _, r := range result.Related {
|
|
related := metadata.Link{
|
|
Kind: "related",
|
|
}
|
|
|
|
if len(r.Catalog) > 0 {
|
|
related.Name = append(related.Name, metadata.Name{
|
|
Kind: "catalog",
|
|
Name: r.Catalog,
|
|
})
|
|
}
|
|
|
|
for k, n := range getNamesFromNameEntry(r.Names) {
|
|
related.Name = append(related.Name, metadata.Name{
|
|
Kind: k,
|
|
Name: n,
|
|
})
|
|
}
|
|
|
|
if len(r.Link) > 0 {
|
|
related.Name = append(related.Name, metadata.Name{
|
|
Kind: "url",
|
|
Name: baseURL + r.Link,
|
|
})
|
|
}
|
|
album.Links = append(album.Links, related)
|
|
}
|
|
|
|
album.ReleaseDate, _ = time.ParseInLocation("2006-01-02", result.ReleaseDate, time.UTC)
|
|
|
|
for _, d := range result.Discs {
|
|
disc := metadata.Disc{}
|
|
for _, t := range d.Tracks {
|
|
track := metadata.Track{}
|
|
for k, n := range getNamesFromNameEntry(namesEntry(t.Names)) {
|
|
track.Name = append(track.Name, metadata.Name{
|
|
Kind: k,
|
|
Name: n,
|
|
})
|
|
}
|
|
sub := strings.Split(t.Duration, ":")
|
|
s1, _ := strconv.ParseInt(strings.TrimLeft(sub[0], "0"), 10, 0)
|
|
if len(sub) > 2 {
|
|
//TODO
|
|
} else if len(sub) > 1 {
|
|
s2, _ := strconv.ParseInt(strings.TrimLeft(sub[1], "0"), 10, 0)
|
|
track.Duration = time.Minute*time.Duration(s1) + time.Second*time.Duration(s2)
|
|
} else {
|
|
track.Duration = time.Second * time.Duration(s1)
|
|
}
|
|
|
|
disc.Tracks = append(disc.Tracks, track)
|
|
}
|
|
|
|
album.Discs = append(album.Discs, disc)
|
|
}
|
|
|
|
return album
|
|
}
|
|
|
|
func (s *Source) Test() {
|
|
album := s.FindByTOC(metadata.NewTOCFromString("267453 150 24647 71194 95579 139576 174573 199089 244305"))
|
|
//album := s.GetAlbumInformation(42266)
|
|
|
|
log.Print(album)
|
|
}
|