402 lines
14 KiB
PHP
402 lines
14 KiB
PHP
<?php
|
|
|
|
include "AutoLoader.php";
|
|
|
|
use Animarr\Database;
|
|
use Animarr\Filter;
|
|
use Animarr\Release\Release;
|
|
use Animarr\Release\MultiRelease;
|
|
use Animarr\Downloader;
|
|
use Animarr\Torrent\Torrent;
|
|
use Animarr\Source\SourceList;
|
|
|
|
ini_set('memory_limit', '2048M');
|
|
setlocale(LC_CTYPE, "en_US.UTF-8");
|
|
|
|
chdir(__DIR__ . "/" . "..");
|
|
|
|
Database::parseConfig(isset($argv[1]) ? $argv[1] : "config.ini");
|
|
Downloader::log("Starting config...");
|
|
|
|
|
|
$aniDBPath = Database::getConfigKey("database.anidb", "/tmp/anidb.cache.dat");
|
|
//$aniDBExtraPath = Database::getConfigKey("database.anidb.extra", "/tmp/anidb.extra.dat");
|
|
|
|
Downloader::log("Loading AniDB...");
|
|
@mkdir(dirname($aniDBPath), 0755, true);
|
|
$aniDB = new \Animarr\AniDB($aniDBPath, false);
|
|
|
|
Downloader::log("Loading Database...");
|
|
$database = new Database($aniDB);
|
|
|
|
foreach($database->getAlternateTitles() as $t){
|
|
$aniDB->addEntryName($t["aid"], $t["language"], $t["title"], $t["type"]);
|
|
}
|
|
|
|
$extractor = new \Animarr\Extractor\SceneExtractor($aniDB);
|
|
|
|
$sources = new SourceList();
|
|
|
|
if(Database::getConfigKey("tracker.animebytes.enable", false)){
|
|
$abCategory = constant(\Animarr\Source\AnimeBytes::class ."::" . Database::getConfigKey("tracker.animebytes.category", "CATEGORY_SUBS"));
|
|
|
|
Downloader::log("Starting AnimeBytes...");
|
|
$ab = new \Animarr\Source\AnimeBytes(new \Animarr\Extractor\AnimeBytesExtractor($aniDB), $abCategory);
|
|
$sources->addSource($ab);
|
|
}
|
|
|
|
if(Database::getConfigKey("tracker.nyaa.enable", false)){
|
|
$nyaaFilter = constant(\Animarr\Source\Nyaa::class ."::" . Database::getConfigKey("tracker.nyaa.filter", "FILTER_REMAKES"));
|
|
$nyaaCategory = constant(\Animarr\Source\Nyaa::class ."::" . Database::getConfigKey("tracker.nyaa.category", "CATEGORY_ANIME_ENGLISH"));
|
|
|
|
Downloader::log("Starting Nyaa...");
|
|
$nyaa = new \Animarr\Source\Nyaa($extractor, $nyaaCategory, $nyaaFilter);
|
|
$sources->addSource($nyaa);
|
|
}
|
|
|
|
if(Database::getConfigKey("tracker.nyaasi.enable", false)){
|
|
$nyaaSiFilter = constant(\Animarr\Source\NyaaSi::class ."::" . Database::getConfigKey("tracker.nyaasi.filter", "FILTER_REMAKES"));
|
|
$nyaaSiCategory = constant(\Animarr\Source\NyaaSi::class ."::" . Database::getConfigKey("tracker.nyaasi.category", "CATEGORY_ANIME_ENGLISH"));
|
|
|
|
Downloader::log("Starting NyaaSi...");
|
|
$nyaaSi = new \Animarr\Source\NyaaSi($extractor, $nyaaSiCategory, $nyaaSiFilter);
|
|
$sources->addSource($nyaaSi);
|
|
}
|
|
|
|
if(Database::getConfigKey("tracker.anidex.enable", true)){
|
|
$anidexFilterMode = constant(\Animarr\Source\AniDex::class ."::" . Database::getConfigKey("tracker.anidex.filterMode", "FILTER_MODE_INCLUDE"));
|
|
$anidexSelector = constant(\Animarr\Source\AniDex::class ."::" . Database::getConfigKey("tracker.anidex.selector", "SELECTOR_DEFAULT"));
|
|
$anidexLanguage = constant(\Animarr\Source\AniDex::class ."::" . Database::getConfigKey("tracker.anidex.language", "LANGUAGE_ENGLISH"));
|
|
|
|
Downloader::log("Starting AniDex...");
|
|
$anidex = new \Animarr\Source\AniDex($extractor, $anidexLanguage, $anidexSelector, $anidexFilterMode);
|
|
$sources->addSource($anidex);
|
|
}
|
|
|
|
if(Database::getConfigKey("tracker.hs.enable", false)){
|
|
Downloader::log("Starting HorribleSubs...");
|
|
$hs = new \Animarr\Source\HorribleSubs($extractor);
|
|
$sources->addSource($hs);
|
|
}
|
|
|
|
if(Database::getConfigKey("tracker.hsold.enable", false)){
|
|
Downloader::log("Starting HorribleSubsOld...");
|
|
$hs = new \Animarr\Source\HorribleSubsOld($extractor);
|
|
$sources->addSource($hs);
|
|
}
|
|
|
|
foreach(Database::getConfigKey("tracker.webfeed", []) as $name => $feed){
|
|
Downloader::log("Starting WebFeed $name...");
|
|
$feed = new \Animarr\Source\WebFeed($extractor, Database::getConfigKey("tracker.webfeed[$name].", $feed), $name);
|
|
$sources->addSource($feed);
|
|
}
|
|
|
|
$baseFilters = [
|
|
[Filter::PROPERTY_SEEDS, Filter::LESS_THAN, (int) Database::getConfigKey("filter.minseeds", 2), [
|
|
[Filter::PROPERTY_UPLOAD_DATE, Filter::LESS_THAN, time() - Database::getConfigKey("filter.newseed.time", 2400), Filter::ACTION_DROP]
|
|
]],
|
|
[Filter::PROPERTY_ORIGINAL_TITLE, Filter::MATCH, "/[ _]preview/i", Filter::ACTION_DROP],
|
|
//[Filter::PROPERTY_ORIGINAL_TITLE, Filter::MATCH, "/[ _]short/i", Filter::ACTION_DROP],
|
|
//[Filter::PROPERTY_ORIGINAL_TITLE, Filter::MATCH, "/[ _]movie/i", Filter::ACTION_DROP],
|
|
//[Filter::PROPERTY_ORIGINAL_TITLE, Filter::MATCH, "/[ _]special/i", Filter::ACTION_DROP],
|
|
];
|
|
|
|
foreach(Database::getConfigKey("filter.keyword.blacklist", []) as $kw){
|
|
$baseFilters[] = [Filter::PROPERTY_ORIGINAL_TITLE, Filter::MATCH, "/$kw/i", Filter::ACTION_DROP];
|
|
}
|
|
|
|
if(Database::getConfigKey("filter.dubs", true)){
|
|
$baseFilters[] = [Filter::PROPERTY_ORIGINAL_TITLE, Filter::MATCH, "/(funi|simul)\\-?(dub|dl)/i", Filter::ACTION_DROP];
|
|
$baseFilters[] = [Filter::PROPERTY_TAGS, Filter::MATCH, "/^(eng|english|)[ ]?dub(bed|)$/i", Filter::ACTION_DROP];
|
|
}
|
|
|
|
if(Database::getConfigKey("filter.invalidDownload", true)){
|
|
$baseFilters[] = [Filter::PROPERTY_DOWNLOAD_LINK, Filter::LAMBDA, function($val){
|
|
return !Torrent::isTorrentDownloadPath($val);
|
|
}, Filter::ACTION_DROP];
|
|
}
|
|
|
|
foreach (Database::getConfigKey("filter.groups.blacklist", []) as $group){
|
|
$baseFilters[] = [Filter::PROPERTY_GROUP, Filter::EQUAL_THAN, $group, Filter::ACTION_DROP];
|
|
}
|
|
|
|
$extraQuery = "";
|
|
|
|
if(Database::getConfigKey("filter.groups.onlyGreyGoldGroups", false)){
|
|
$extraQuery .= " && (".implode(" | ", Database::getConfigKey("filter.groups.goldlist", [])).")";
|
|
//foreach(Database::getConfigKey("filter.groups.goldlist", []) as $g){
|
|
// $extraQuery .= "[$g] |
|
|
//}
|
|
$baseFilters[] = [Filter::PROPERTY_GROUP, Filter::LAMBDA, function($group){
|
|
foreach(array_merge([$group], explode("-", $group)) as $g){
|
|
|
|
if(in_array($g, Database::getConfigKey("filter.groups.goldlist", []), true)){
|
|
return false;
|
|
}else if(in_array($g, Database::getConfigKey("filter.groups.greylist", []), true)){
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}, Filter::ACTION_DROP];
|
|
}
|
|
|
|
$filter = new Filter(Filter::ACTION_KEEP, $baseFilters);
|
|
|
|
if(Database::getConfigKey("filter.invalidDownload", true)){
|
|
$baseFilters[] = [Filter::PROPERTY_DOWNLOAD_LINK, Filter::LAMBDA, function($val){
|
|
return !Torrent::isTorrentDownloadPath($val);
|
|
}, Filter::ACTION_DROP];
|
|
}
|
|
|
|
$existingFilter = new Filter(Filter::ACTION_KEEP, $baseFilters);
|
|
|
|
|
|
$selector = new \Animarr\Selector();
|
|
|
|
$downloader = new Downloader($database);
|
|
|
|
|
|
$checkLimit = (int) Database::getConfigKey("tracker.startpoll.limit", 1200);
|
|
|
|
$counter = 0;
|
|
$fastCounter = 0;
|
|
|
|
/** @var Release[] $releases */
|
|
$releases = [];
|
|
|
|
/*
|
|
foreach(scandir("data/database/track") as $f){
|
|
if(preg_match("/anidb\\-([0-9]+)$/", $f, $matches) > 0){
|
|
$aid = intval($matches[1]);
|
|
if(!$database->isTracking($aid)){
|
|
echo "Want to track ".$aniDB->getAnime($aid)["title"]."\n";
|
|
if(trim(fgets(STDIN)) === "y"){
|
|
$database->addTrack($aid);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
foreach($aniDB->getAllMatches("baka to test ni") as $aid => $lev){
|
|
$info = $aniDB->getAnime($aid);
|
|
echo $info["title"] . ": $lev\n";
|
|
}
|
|
}*/
|
|
|
|
$episodes = [];
|
|
|
|
if(Database::getConfigKey("refresh.force", false)){
|
|
foreach($database->getTracked() as $data){
|
|
Downloader::log("Forcing refresh of \"".$aniDB->getAnime($data["aid"])["title"]."\"!");
|
|
$episodes[$data["aid"]] = $database->getAnimeEpisodes($data["aid"]);
|
|
}
|
|
}
|
|
|
|
|
|
while(true){
|
|
try{
|
|
|
|
if(Database::getConfigKey("interval.poll.anidb") !== false and ($counter % Database::getConfigKey("interval.poll.anidb", 172800)) === 0){
|
|
Downloader::log("Doing anidb update...");
|
|
foreach($database->getTracked() as $data){
|
|
$xml = $aniDB->getAnimeData($data["aid"]);
|
|
Downloader::log("Fetched data of \"".$aniDB->getAnime($data["aid"])["title"]."\": " . strval($xml->type) . " (".$data["episode_count"]."/".$xml->episodecount.")");
|
|
}
|
|
}
|
|
|
|
if(Database::getConfigKey("interval.poll.new") !== false and ($counter % Database::getConfigKey("interval.poll.new", 60)) === 0){
|
|
Downloader::log("Fetching new tracked...");
|
|
foreach($database->getTracked() as $aid => $data){
|
|
if(!$database->getTrackFlag($aid, Database::FLAG_LOCKED) and (($counter === 0 and Database::getConfigKey("search.force", false)) or $data["episode_count"] <= 0)){
|
|
Downloader::log("Searching \"".$aniDB->getAnime($aid)["title"]."\" [anidb-$aid]");
|
|
|
|
if(($counter === 0 and Database::getConfigKey("search.forceclear", false)) or $data["episode_count"] < 0){
|
|
$database->clearTrackedEpisodes($aid);
|
|
$database->updateTrackEpisodes($aid, 0);
|
|
$data = $database->getTrackInfo($aid);
|
|
}else if(($counter === 0 and Database::getConfigKey("search.force", false))){
|
|
$database->updateTrackEpisodes($aid);
|
|
}
|
|
|
|
$results = [];
|
|
|
|
foreach($database->getAnimeReleases($aid) as $r){
|
|
$results[] = $r;
|
|
}
|
|
|
|
foreach($database->getAniDB()->getSearchQueries($aid, Database::getConfigKey("anidb.query.languages", ["en", "x-jat"])) as $q){
|
|
if(strlen($q) < Database::getConfigKey("anidb.query.minlength", 4)){
|
|
continue;
|
|
}
|
|
Downloader::log("Querying \"$q\"");
|
|
$results = array_merge($results, $sources->find($q, (int) Database::getConfigKey("tracker.firstpoll.limit", -1), $extraQuery));
|
|
}
|
|
|
|
Downloader::log("Found ".count($results)." result(s) [anidb-$aid]");
|
|
|
|
foreach($filter->applyFilter($results) as $release){
|
|
$releases[$release->getId()] = [$release, true];
|
|
Downloader::log("Found \"".$release->getOriginalTitle()."\"");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if(Database::getConfigKey("interval.poll.tracker") !== false and ($counter % Database::getConfigKey("interval.poll.tracker", 10)) === 0){
|
|
Downloader::log("Doing normal poll...");
|
|
foreach($filter->applyFilter($sources->find(null, $checkLimit, $extraQuery)) as $release){
|
|
if(!isset($releases[$release->getId()])){
|
|
$releases[$release->getId()] = [$release, false];
|
|
}
|
|
}
|
|
}
|
|
|
|
if(count($releases) > 0){
|
|
Downloader::log("Found ".count($releases)." new release(s), checking!");
|
|
|
|
foreach ($releases as $id => $r){
|
|
/** @var Release $result */
|
|
$result = $r[0];
|
|
$result->setFilter($filter);
|
|
Downloader::debug("Checking \"".$result->getOriginalTitle()."\"...");
|
|
$m = $database->matchRelease($result, Database::getConfigKey("database.forceclean", false));
|
|
if($m === null){
|
|
Downloader::log("Could not match \"".$result->getOriginalTitle()."\" (".$result->getTitle().")");
|
|
continue;
|
|
}
|
|
|
|
if(!$r[1] and !$m["new"] and !Database::getConfigKey("search.forcenew", false)){
|
|
continue;
|
|
}
|
|
|
|
if(($result->getType() === Release::TYPE_SINGLE and $result->getNumber() >= 0) or $result instanceof MultiRelease){
|
|
foreach($result->getContents($extractor) as $rr){
|
|
|
|
$nm = $database->getAniDB()->matchRelease($rr);
|
|
|
|
if($nm === null){
|
|
Downloader::log("Invalid release \"".$rr->getOriginalTitle()."\" (base: \"".$result->getOriginalTitle()."\")");
|
|
continue;
|
|
}
|
|
|
|
if(!isset($episodes[$nm["aid"]])){
|
|
$episodes[$nm["aid"]] = [];
|
|
}
|
|
|
|
//if($m["new"]){
|
|
Downloader::log("Found \"".$rr->getOriginalTitle()."\" for \"".$nm["title"]."\" (base: \"".$result->getOriginalTitle()."\")");
|
|
//}
|
|
|
|
$episodes[$nm["aid"]][$rr->getNumber()] = isset($episodes[$nm["aid"]][$rr->getNumber()]) ? $episodes[$nm["aid"]][$rr->getNumber()] or $m["new"] : $m["new"];
|
|
}
|
|
}else{
|
|
Downloader::log("Invalid release \"".$result->getOriginalTitle()."\"");
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
if(count($episodes) > 0){
|
|
Downloader::log("Found ".count($episodes)." new shows(s) to match!");
|
|
|
|
foreach ($episodes as $aid => $list){
|
|
Downloader::debug("Processing anidb-".$aid."...");
|
|
if(!$database->isTracking($aid) or $database->getTrackFlag($aid, Database::FLAG_LOCKED)){
|
|
continue;
|
|
}
|
|
$eList = $database->getAnimeEpisodes($aid);
|
|
|
|
$info = $database->getAniDB()->getAnime($aid);
|
|
$groups = $database->getTrackGroups($aid);
|
|
|
|
foreach ($list as $episode => $check){
|
|
$trackEpisode = $database->getTrackEpisode($aid, $episode);
|
|
if($trackEpisode !== null){
|
|
$trackEpisode = $existingFilter->applyFilterSingle($trackEpisode);
|
|
}
|
|
|
|
if(isset($eList[$episode])){
|
|
$best = $selector->findBestRelease($filter->applyFilter($eList[$episode]), $groups);
|
|
if($best === null){
|
|
continue;
|
|
}
|
|
if($trackEpisode === null){
|
|
$best->setFilter($filter);
|
|
$database->saveTrackEpisode($aid, $episode, $best);
|
|
$downloader->addToDownload($aid, $best);
|
|
}else{
|
|
if($trackEpisode->getId() === $best->getId()){ //They are the same!
|
|
continue;
|
|
}
|
|
|
|
if($selector->selectBest($trackEpisode, $best, $groups) === $trackEpisode){ //Other is still best!
|
|
continue;
|
|
}
|
|
$best->setFilter($filter);
|
|
$database->saveTrackEpisode($aid, $episode, $best);
|
|
|
|
$downloader->addToDownload($aid, $best);
|
|
}
|
|
|
|
$tList = $database->getTrackEpisodes($aid);
|
|
Downloader::log("Matching new episode $episode for \"".$info["title"]."\": \"".$best->getOriginalTitle()."\"");
|
|
}
|
|
}
|
|
|
|
$database->updateTrackEpisodes($aid);
|
|
}
|
|
|
|
Downloader::log("Trying to download torrents...");
|
|
$downloader->handleNewDownloads();
|
|
}
|
|
|
|
|
|
if(Database::getConfigKey("interval.poll.finished") !== false and ($counter % Database::getConfigKey("interval.poll.finished", 5)) === 0){
|
|
Downloader::log("Polling finished downloads...");
|
|
$downloader->setUseLink(false);
|
|
$added = $downloader->handleCompletedDownloads(Database::getConfigKey("torrent.folder.finished", "./Processing"));
|
|
$downloader->setUseLink(true);
|
|
$added += $downloader->handleCompletedDownloads(Database::getConfigKey("torrent.folder.private.finished", "./ProcessingPrivate"));
|
|
|
|
if($added > 0){
|
|
foreach(Database::getConfigKey("webhook.episodes.added", []) as $url){
|
|
\Animarr\Request::getURL($url);
|
|
}
|
|
}
|
|
}
|
|
|
|
//if($save){
|
|
// Downloader::log("Saving database...", "SAVE");
|
|
// $database->save();
|
|
// Downloader::log("Database saved!", "SAVE");
|
|
//}
|
|
|
|
$releases = [];
|
|
|
|
$episodes = [];
|
|
|
|
sleep(60);
|
|
$checkLimit = (int) Database::getConfigKey("tracker.poll.limit", 0);
|
|
|
|
$counter++;
|
|
|
|
if(file_exists("dopoll.flag")){
|
|
unlink("dopoll.flag");
|
|
$counter = Database::getConfigKey("interval.poll.new", 60) * Database::getConfigKey("interval.poll.finished", 5) * Database::getConfigKey("interval.poll.tracker", 10);
|
|
$aniDB->resetCache($aniDBPath, false);
|
|
foreach($database->getAlternateTitles() as $t){
|
|
$aniDB->addEntryName($t["aid"], $t["language"], $t["title"], $t["type"]);
|
|
}
|
|
}
|
|
|
|
Torrent::$urls = [];
|
|
}catch(Exception $e){
|
|
Downloader::log($e);
|
|
}
|
|
|
|
}
|
|
|
|
//$downloader = new \Animarr\Downloader($aniDB)
|
|
|
|
//https://img7.anidb.net/pics/anime/177065.jpg
|