S.O.N.G/METANOIA
S.O.N.G/METANOIA
3
0
Fork 0
Code Issues Pull requests Releases Activity

update schema, use new Hibiki

Browse source
This commit is contained in:
DataHoarder 2022-01-30 19:14:50 +01:00
parent f1fde99d40
commit fd326df5e5
7 changed files with 187 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
METANOIA.go
View file

@ -282,7 +282,21 @@ func main() {
log.Fatal(err)
}
dbGcTicker := time.NewTicker(15 * time.Minute)
go func() {
for range dbGcTicker.C {
for printDb.RunGarbageCollection() == nil {
}
}
}()
defer printDb.Close()
defer func() {
for printDb.RunGarbageCollection() == nil {
}
}()
defer dbGcTicker.Stop()
//TODO: check if mutex is correct!
printStrategy = panako.NewStrategy(printDb, true)

126
database/artist.go Normal file
View file

@ -0,0 +1,126 @@
package database
import (
"database/sql"
"encoding/json"
)
type Artist struct {
DatabaseType
id int64
metadata []byte
}
func InsertArtistToDatabase(db *Database, names map[string]string) *Artist {
//TODO: do it in single tx
rows, err := db.Query("INSERT INTO artist DEFAULT VALUES RETURNING id;")
if err != nil {
return nil
}
defer rows.Close()
rows.Next()
var id int64
err = rows.Scan(&id)
if err != nil {
return nil
}
artist := GetArtistFromDatabase(db, id)
if artist == nil {
return nil
}
for k, n := range names {
artist.AddName(k, n)
}
return artist
}
func GetArtistsFromDatabaseByName(db *Database, name string) (artists []*Artist) {
rows, err := db.Query("SELECT id, metadata FROM artists WHERE id IN(SELECT artist FROM artists_names WHERE name ILIKE $1);", name)
if err != nil {
return nil
}
defer rows.Close()
for rows.Next() {
artist, err := GetArtistFromRow(db, rows)
if err != nil {
break
}
artists = append(artists, artist)
}
return
}
func GetArtistFromDatabase(db *Database, id int64) *Artist {
rows, err := db.Query("SELECT id, metadata FROM artists WHERE id = $1;", id)
if err != nil {
return nil
}
defer rows.Close()
rows.Next()
r, err := GetArtistFromRow(db, rows)
if err != nil {
return nil
}
return r
}
func GetArtistFromRow(db *Database, row *sql.Rows) (*Artist, error) {
r := &Artist{
DatabaseType: DatabaseType{
db: db,
},
}
err := row.Scan(&r.id, &r.metadata)
if err != nil {
return nil, err
}
return r, nil
}
func (r *Artist) GetId() int64 {
return r.id
}
func (r *Artist) GetNames() map[string]string {
result := make(map[string]string)
rows, err := r.db.Query("SELECT kind, name FROM artists_names WHERE id $1;", r.id)
if err != nil {
return nil
}
defer rows.Close()
for rows.Next() {
var kind string
var name string
err = rows.Scan(&kind, &name)
if err != nil {
break
}
result[kind] = name
}
return result
}
func (r *Artist) RemoveName(kind string) {
r.db.Exec("DELETE FROM artists_names WHERE id = $1 AND kind = $2;", r.id, kind)
}
func (r *Artist) AddName(kind, name string) {
r.db.Exec("INSERT INTO artists_names (artist, kind, name) VALUES ($1, $2, $3) ON CONFLICT (artist, kind) DO UPDATE SET name = $3;", r.id, kind, name)
}
func (r *Artist) GetMetadata() (result map[string]interface{}) {
json.Unmarshal(r.metadata, &result)
return result
}
func (r *Artist) SetMetadata(value map[string]interface{}) {
r.metadata, _ = json.Marshal(value)
r.db.Exec("UPDATE artists SET metadata = $2::jsonb WHERE id = $1;", r.id, r.metadata)
}

12
database/release.go
View file

@ -41,7 +41,7 @@ func GetReleasesFromDatabaseByIdentifier(db *Database, identifier string) (relea
}
releases = append(releases, release)
}
return releases
return
}
func GetReleaseFromDatabase(db *Database, id int64) *Release {
@ -71,26 +71,26 @@ func GetReleaseFromRow(db *Database, row *sql.Rows) (*Release, error) {
return r, nil
}
func (r Release) GetId() int64 {
func (r *Release) GetId() int64 {
return r.id
}
func (r Release) GetIdentifiers() []string {
func (r *Release) GetIdentifiers() []string {
return r.identifiers
}
func (r Release) SetIdentifiers(identifiers []string) {
func (r *Release) SetIdentifiers(identifiers []string) {
r.identifiers = identifiers
r.db.Exec("UPDATE releases SET identifiers = $2 WHERE id = $1;", r.id, pq.Array(r.identifiers))
}
func (r Release) GetMetadata() (result map[string]interface{}) {
func (r *Release) GetMetadata() (result map[string]interface{}) {
json.Unmarshal(r.metadata, &result)
return result
}
func (r Release) SetMetadata(value map[string]interface{}) {
func (r *Release) SetMetadata(value map[string]interface{}) {
r.metadata, _ = json.Marshal(value)
r.db.Exec("UPDATE releases SET metadata = $2::jsonb WHERE id = $1;", r.id, r.metadata)
}

2
database/resource.go
View file

@ -140,7 +140,7 @@ func (r *Resource) GetReleases() (releases []*Release) {
}
releases = append(releases, release)
}
return releases
return
}
func (r *Resource) AddRelease(release *Release) {

4
go.mod
View file

@ -3,7 +3,7 @@ module git.gammaspectra.live/S.O.N.G/METANOIA
go 1.17
require (
git.gammaspectra.live/S.O.N.G/Hibiki v0.0.0-20220130030711-030193fa3726
git.gammaspectra.live/S.O.N.G/Hibiki v0.0.0-20220130170252-8689f79eb0cd
github.com/dhowden/tag v0.0.0-20201120070457-d52dcb253c63
github.com/ipfs/go-cid v0.1.0
github.com/lib/pq v1.10.4
@ -13,7 +13,7 @@ require (
require (
git.gammaspectra.live/S.O.N.G/go-pus v0.0.0-20220130003320-c9b07c6bec7a // indirect
git.gammaspectra.live/S.O.N.G/goborator v0.0.0-20220130014715-837c1db0113a // indirect
git.gammaspectra.live/S.O.N.G/goborator v0.0.0-20220130143007-b0d46ec375ac // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cocoonlife/goflac v0.0.0-20170210142907-50ea06ed5a9d // indirect

8
go.sum
View file

@ -1,10 +1,10 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
git.gammaspectra.live/S.O.N.G/Hibiki v0.0.0-20220130030711-030193fa3726 h1:Sobaj0pXvyKKS+CLHG0HLoNF6ye1lmcqt4U5tCOdqfM=
git.gammaspectra.live/S.O.N.G/Hibiki v0.0.0-20220130030711-030193fa3726/go.mod h1:N46fIbI6SZ/VHdSf/aAWBl02UkdOdTJ3xtXvy7BdzyQ=
git.gammaspectra.live/S.O.N.G/Hibiki v0.0.0-20220130170252-8689f79eb0cd h1:3xlFX6pwteLvMMJE7XW5fqPG+mu2slPvkYwcxeZLpos=
git.gammaspectra.live/S.O.N.G/Hibiki v0.0.0-20220130170252-8689f79eb0cd/go.mod h1:o1NqDI9/1EwqL0FPHKOrZmQSeRXnzVBLJPxdpwrNqHM=
git.gammaspectra.live/S.O.N.G/go-pus v0.0.0-20220130003320-c9b07c6bec7a h1:LxrTp9gf4w5KnFHRPFLXYfoxC58GCSEmZrHI6Ogtrm0=
git.gammaspectra.live/S.O.N.G/go-pus v0.0.0-20220130003320-c9b07c6bec7a/go.mod h1:vkoHSHVM9p6vAUmXAik0gvaLcIfiQYrD6bQqVpOulUk=
git.gammaspectra.live/S.O.N.G/goborator v0.0.0-20220130014715-837c1db0113a h1:xVEXo4qTGZK/6O0KVvoPH9XTdGbUQfb7jP/itO8kykk=
git.gammaspectra.live/S.O.N.G/goborator v0.0.0-20220130014715-837c1db0113a/go.mod h1:ySjuueqe5HUqvf7lWS51Cy5UP2tgJWsezOv8UIm2arA=
git.gammaspectra.live/S.O.N.G/goborator v0.0.0-20220130143007-b0d46ec375ac h1:UcWeLjW7963Cpd4cB7TOn5OZPl73goxQ5WhIFO0DIdU=
git.gammaspectra.live/S.O.N.G/goborator v0.0.0-20220130143007-b0d46ec375ac/go.mod h1:ySjuueqe5HUqvf7lWS51Cy5UP2tgJWsezOv8UIm2arA=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=

50
schema.sql
View file

@ -2,7 +2,8 @@ CREATE EXTENSION IF NOT EXISTS hstore;
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE EXTENSION IF NOT EXISTS btree_gin;
CREATE TYPE artist_kind AS ENUM ('main', 'performer', 'composer', 'arranger');
CREATE TYPE name_kind AS ENUM ('main', 'original', 'romaji', 'kanji', 'english');
CREATE TYPE artist_kind AS ENUM ('main', 'original', 'performer', 'composer', 'arranger');
-- groups resources in specific releases / groupings
@ -12,7 +13,7 @@ CREATE TABLE releases (
metadata jsonb NOT NULL DEFAULT '{}'::jsonb
);
CREATE INDEX idx_releases_identifiers_gin ON releases USING GIN (identifiers);
CREATE INDEX idx_releases_identifiers_gin ON releases USING GIN (identifiers) INCLUDE (id);
CREATE TABLE resources (
id BIGSERIAL PRIMARY KEY,
@ -24,7 +25,7 @@ CREATE TABLE resources (
CREATE INDEX idx_resources_hash ON resources USING BTREE (hash); -- TODO: check if you can do partial prefix queries
CREATE INDEX idx_resources_path ON resources USING BTREE (path);
CREATE INDEX idx_resources_path_gin ON resources USING GIN (path); -- Allows for partial path queries
CREATE INDEX idx_resources_path_gin ON resources USING GIN (path gin_trgm_ops); -- Allows for partial path queries
CREATE TABLE resource_alternate_identifiers (
resource BIGINT NOT NULL,
@ -51,36 +52,46 @@ CREATE INDEX idx_resource_releases_release ON resource_releases USING BTREE (rel
CREATE TABLE albums (
id SERIAL PRIMARY KEY,
main_name TEXT NOT NULL,
cover BIGINT, -- can be null if not available
names hstore NOT NULL, -- key -> value where keys are language / kinds
identifiers TEXT[] NOT NULL DEFAULT '{}'::TEXT[], -- array of "semi-unique" identifiers, like catalog number
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
CONSTRAINT fk_cover FOREIGN KEY (cover) REFERENCES resources(id)
);
CREATE INDEX idx_albums_main_name ON albums USING BTREE (main_name);
CREATE INDEX idx_albums_names_gin ON albums USING GIN (names);
CREATE INDEX idx_albums_identifiers_gin ON albums USING GIN (identifiers);
CREATE TABLE albums_names (
album INTEGER NOT NULL,
kind name_kind NOT NULL,
name TEXT NOT NULL,
CONSTRAINT idx_albums_names UNIQUE (album, kind),
CONSTRAINT fk_album FOREIGN KEY (album) REFERENCES albums(id)
);
CREATE INDEX idx_albums_names_album ON albums_names USING BTREE (album);
CREATE INDEX idx_albums_names_name_gin ON albums_names USING GIN (name gin_trgm_ops) INCLUDE (album);
CREATE TABLE artists (
id SERIAL PRIMARY KEY,
main_name TEXT NOT NULL,
names hstore NOT NULL, -- key -> value where keys are language / kinds
metadata jsonb NOT NULL DEFAULT '{}'::jsonb
);
CREATE INDEX idx_artists_main_name ON artists USING BTREE (main_name);
CREATE INDEX idx_artists_names_gin ON artists USING GIN (names);
CREATE TABLE artists_names (
artist INTEGER NOT NULL,
kind name_kind NOT NULL,
name TEXT NOT NULL,
CONSTRAINT idx_artists_names UNIQUE (artist, kind),
CONSTRAINT fk_artist FOREIGN KEY (artist) REFERENCES artists(id)
);
CREATE INDEX idx_artists_names_artist ON artists_names USING BTREE (artist);
CREATE INDEX idx_artists_names_name_gin ON artists_names USING GIN (name gin_trgm_ops) INCLUDE (artist);
CREATE TABLE songs (
id BIGSERIAL PRIMARY KEY,
resource BIGINT NOT NULL,
cover BIGINT, -- set to override album cover. can be null if not available
album INTEGER, -- can be null
main_name TEXT NOT NULL,
names hstore NOT NULL, -- key -> value where keys are language / kinds
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
CONSTRAINT fk_resource FOREIGN KEY (resource) REFERENCES resources(id),
@ -89,9 +100,16 @@ CREATE TABLE songs (
);
CREATE INDEX idx_songs_resource ON songs USING BTREE (resource);
CREATE INDEX idx_songs_album ON songs USING BTREE (album);
CREATE INDEX idx_songs_main_name ON songs USING BTREE (main_name);
CREATE INDEX idx_songs_names_gin ON albums USING GIN (names);
CREATE TABLE songs_names (
song INTEGER NOT NULL,
kind name_kind NOT NULL,
name TEXT NOT NULL,
CONSTRAINT idx_songs_names UNIQUE (song, kind),
CONSTRAINT fk_song FOREIGN KEY (song) REFERENCES songs(id)
);
CREATE INDEX idx_songs_names_song ON songs_names USING BTREE (song);
CREATE INDEX idx_songs_names_name_gin ON songs_names USING GIN (name gin_trgm_ops) INCLUDE (song);
-- extra mapping table of artists <-> song
CREATE TABLE album_artists (