regular scedule com.sofascore.grapper

This commit is contained in:
salam committed 2026-07-02 07:04:59 +02:00
1 parent fb31825b22
commit 3fc0724137
12 files changed
+758 -590

No files matched your search

@@ -2,8 +2,11 @@
namespace App\Console\Commands\Grapper\Sofascore;
use App\Console\Commands\Grapper\Sofascore\Tools\LoadTournaments;
use App\Console\Commands\Grapper\Sofascore\Tools\LoadDays;
use App\Console\Commands\Grapper\Sofascore\Tools\Sleeper;
use App\Console\Commands\Nuh;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreTournament;
use App\Utilities\Notice;
use Illuminate\Support\Facades\File;
@@ -11,132 +14,106 @@ class EventsDatabase
{
public static $storage_path = '/Console/Commands/Grapper/Sofascore/grapped/';
public static function load($days)
public static function load()
{
$debug = Nuh::$options['d'] ?? 0;
$print = Nuh::$options['p'] ?? 0;
$langsam = Nuh::$options['l'] ?? 0;
$tornaments = LoadTournaments::load();
$days = LoadDays::load();
if ($print) {
Notice::note($days, 'p');
Notice::note($tornaments->toArray(), 'b', 1);
}
foreach ($days as $d) {
foreach ($days as $day) {
$langsam = $d['run_slow'];
$sport_id = $d['sport_id'];
$day = $d['day'];
if ($debug) {
Notice::note($day, 'b');
}
$active_tournaments = SofaScoreTournament::select('name', 'tournament_id')->where('sport_id', $sport_id)->where('enabled', 1)->get();
foreach ($tornaments as $tournament) {
foreach ($active_tournaments as $tournament) {
if ($debug) {
Notice::note($tournament->name, 'b');
}
$has = SofaScoreDayTournament::where('day', $day)->where('tournament_id', $tournament->tournament_id)->first();
$url = Nuh::$options['base_url']."api/v1/unique-tournament/$tournament->tournament_id/scheduled-events/$day";
if ($has) {
if ($debug) {
Notice::note($day.' - '.$tournament->name.' grapping', 'g');
}
$rand = self::rand($langsam);
if ($debug) {
Notice::note("Sleep $rand msec", 'y');
}
$url = Nuh::$options['base_url']."api/v1/unique-tournament/$tournament->tournament_id/scheduled-events/$day";
usleep($rand);
Sleeper::rand($langsam);
$data = CurlSofa::load($url);
$data = CurlSofa::load($url);
Nuh::$options['s'] = 0;
Nuh::$options['s'] = 0;
Nuh::$options['r'] = 0;
Nuh::$options['r'] = 0;
if (isset($data['events'])) {
if (isset($data['events'])) {
foreach ($data['events'] as $event) {
foreach ($data['events'] as $event) {
Notice::note($event['season']['name'].' '.$day, 'bb');
Notice::note($event['season']['name'].' '.$day, 'bb');
$folder = app_path().self::$storage_path.$day.'/'.$event['tournament']['slug'].'/'.$event['slug'];
$folder = app_path().self::$storage_path.$day.'/'.$event['tournament']['slug'].'/'.$event['slug'];
File::makeDirectory($folder, 0775, true, true);
File::makeDirectory($folder, 0775, true, true);
/*********************************
* EVENT event
*********************************/
/*********************************
* EVENT event
*********************************/
File::put($folder.'/event.json', json_encode($event));
File::put($folder.'/event.json', json_encode($event));
/*********************************
* EVENT statistics
*********************************/
/*********************************
* EVENT statistics
*********************************/
$event_id = $event['id'];
$event_id = $event['id'];
$url = Nuh::$options['base_url']."api/v1/event/$event_id/statistics";
$url = Nuh::$options['base_url']."api/v1/event/$event_id/statistics";
$rand = self::rand($langsam);
if ($debug) {
Notice::note("Sleep $rand msec", 'y');
Sleeper::rand($langsam);
$data = CurlSofa::load($url);
$data['id'] = $event_id;
File::put($folder.'/statistics.json', json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = Nuh::$options['base_url']."api/v1/event/$event_id/lineups";
Sleeper::rand($langsam);
$data = CurlSofa::load($url);
$data['id'] = $event_id;
File::put($folder.'/lineups.json', json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = Nuh::$options['base_url']."api/v1/event/$event_id/odds/1/all";
Sleeper::rand($langsam);
$data = CurlSofa::load($url);
$data['id'] = $event_id;
File::put($folder.'/odds.json', json_encode($data));
}
usleep($rand);
$data = CurlSofa::load($url);
$data['id'] = $event_id;
File::put($folder.'/statistics.json', json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = Nuh::$options['base_url']."api/v1/event/$event_id/lineups";
$rand = self::rand($langsam);
if ($debug) {
Notice::note("Sleep $rand msec", 'y');
}
usleep($rand);
$data = CurlSofa::load($url);
$data['id'] = $event_id;
File::put($folder.'/lineups.json', json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = Nuh::$options['base_url']."api/v1/event/$event_id/odds/1/all";
$rand = self::rand($langsam);
if ($debug) {
Notice::note("Sleep $rand msec", 'y');
}
usleep($rand);
$data = CurlSofa::load($url);
$data['id'] = $event_id;
File::put($folder.'/odds.json', json_encode($data));
}
} else {
if ($debug) {
Notice::note($day.' - '.$tournament->name.' not active', 'r');
}
}
}
}
}
public static function rand(bool $langsam = false)
{
if($langsam){
return rand(500000, 50000000);
}
else {
return rand(500000, 1000000);
}
}
}
@@ -2,18 +2,23 @@
namespace App\Console\Commands\Grapper\Sofascore\Specials;
use App\Console\Commands\Grapper\Sofascore\Tools\LoadDays;
use App\Utilities\Notice;
use App\Utilities\ZipFolder;
class CreateZip
{
public static function run($days)
public static function run()
{
foreach ($days as $day) {
$days = LoadDays::load();
foreach ($days as $d) {
$day = $d['day'];
ZipFolder::create($day);
}
Notice::note('Archive created', 'g', 1);
}
}
@@ -4,150 +4,167 @@ namespace App\Console\Commands\Grapper\Sofascore\Specials;
use App\Console\Commands\Grapper\Sofascore\CurlSofa;
use App\Console\Commands\Grapper\Sofascore\Tools\LoadDays;
use App\Console\Commands\Grapper\Sofascore\Tools\Sleeper;
use App\Console\Commands\Nuh;
use App\Models\Country;
use App\Models\SofaScoreCategorie;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreTournament;
use App\Utilities\Notice;
use Illuminate\Support\Facades\Log;
class DaysTournament
{
public static function run()
{
$debug = Nuh::$options['d'] ?? 0;
$days = LoadDays::load();
dd($days);
$tournaments_counter = 0;
foreach ($days as $day) {
foreach ($days as $d) {
$page = 1;
$langsam = $d['run_slow'];
$sport_id = $d['sport_id'];
$day = $d['day'];
$slug = $d['slug'];
while ($page) {
$has = SofaScoreDayTournament::where('day', $day)->first();
$url = Nuh::$options['base_url'] . "api/v1/sport/$sport->ss_slug/scheduled-tournaments/$day/page/$page";
$rand = rand(500000, 1000000);
if ($has) {
if ($debug) {
Notice::note("Sleep $rand msec", 'y');
Notice::note("$day\nis allready grapped", 'r');
}
usleep($rand);
$data = CurlSofa::load($url);
if ($data['hasNextPage']) {
$page++;
} else {
$page = false;
} else {
if ($debug) {
Notice::note("$day\ngrappig", 'g');
}
// $page = false;
$page = 1;
foreach ($data['scheduled'] as $row) {
while ($page) {
if (isset($row['tournament']['uniqueTournament'])) {
$tournament = $row['tournament']['uniqueTournament'];
$url = Nuh::$options['base_url']."api/v1/sport/$slug/scheduled-tournaments/$day/page/$page";
Sleeper::rand($langsam);
$data = CurlSofa::load($url);
if ($data['hasNextPage']) {
$page++;
} else {
$tournament = $row['tournament'];
$page = false;
}
$category = $tournament['category'];
foreach ($data['scheduled'] as $row) {
if ($sport_id == 5) {
$country_id = 470;
} else {
if (isset($row['tournament']['uniqueTournament'])) {
$tournament = $row['tournament']['uniqueTournament'];
} else {
$tournament = $row['tournament'];
}
$category = $tournament['category'];
if ($sport_id == 5) {
$country_id = 470;
} else {
/**
* CATEGORY
*/
$find = [
'Amateur',
'Bosnia & Herzegovina',
'Czechia',
'DR Congo',
'Netherlands',
'North & Central America',
'Bonaire',
'Sao Tome and Principe',
'Cape Verde',
'Macao',
'Saint Kitts and Nevis',
'Eswatini',
'Timor Leste',
'Laos',
'Congo Republic',
];
$replace = [
'',
'Bosnia Herzegovina',
'Czech Republic',
'Congo DR',
'Holland',
'North America',
'Bonaire, Sint Eustatius and Saba',
'Sao Tome e Principe',
'Cape Verde Islands',
'Macau',
'St. Kitts and Nevis',
'Swaziland',
'East Timor',
'Lao Peoples\'s Democratic Republic',
'Congo DR',
];
$country_id = Country::where('name', trim(str_replace($find, $replace, $category['name'])))->value('id');
}
// dd($category, $country_id);
if (! $country_id) {
// platzhalter fix later sofa_score_categories => txodds countries
// CATEGORY find <-> replace
Log::channel('sofascore')->info($tournament);
$sofa_score_categorie_id = 10000000;
} else {
$call = SofaScoreCategorie::updateOrCreate([
'id' => (int) $category['id'],
'country_id' => (int) $country_id,
'slug' => $category['slug'],
], [
'name' => $category['name'],
]);
$sofa_score_categorie_id = $call->id;
}
/**
* CATEGORY
* TOURNAMENT
*/
$find = [
'Amateur',
'Bosnia & Herzegovina',
'Czechia',
'DR Congo',
'Netherlands',
'North & Central America',
'Bonaire',
'Sao Tome and Principe',
'Cape Verde',
'Macao',
'Saint Kitts and Nevis',
'Eswatini',
'Timor Leste',
'Laos',
'Congo Republic',
];
$replace = [
'',
'Bosnia Herzegovina',
'Czech Republic',
'Congo DR',
'Holland',
'North America',
'Bonaire, Sint Eustatius and Saba',
'Sao Tome e Principe',
'Cape Verde Islands',
'Macau',
'St. Kitts and Nevis',
'Swaziland',
'East Timor',
'Lao Peoples\'s Democratic Republic',
'Congo DR',
];
$country_id = Country::where('name', trim(str_replace($find, $replace, $category['name'])))->value('id');
}
// dd($category, $country_id);
if (! $country_id) {
// platzhalter fix later sofa_score_categories => txodds countries
// CATEGORY find <-> replace
Log::channel('sofascore')->info($tournament);
$sofa_score_categorie_id = 10000000;
} else {
$call = SofaScoreCategorie::updateOrCreate([
'id' => (int) $category['id'],
'country_id' => (int) $country_id,
'slug' => $category['slug'],
SofaScoreTournament::updateOrCreate([
'sport_id' => (int) $sport_id,
'tournament_id' => (int) $tournament['id'],
'sofa_score_categorie_id' => (int) $sofa_score_categorie_id,
], [
'name' => $category['name'],
'name' => $tournament['name'],
'slug' => $tournament['slug'],
]);
$tournaments_counter++;
$sofa_score_categorie_id = $call->id;
if ($debug) {
Notice::note('tournament: '.$tournament['name'], 'p');
}
/**
* DAY TOURNAMENT
*/
SofaScoreDayTournament::updateOrCreate([
'tournament_id' => (int) $tournament['id'],
'day' => $day,
]);
}
/**
* TOURNAMENT
*/
SofaScoreTournament::updateOrCreate([
'sport_id' => (int) $sport_id,
'tournament_id' => (int) $tournament['id'],
'sofa_score_categorie_id' => (int) $sofa_score_categorie_id,
], [
'name' => $tournament['name'],
'slug' => $tournament['slug'],
]);
$tournaments_counter++;
/**
* DAY TOURNAMENT
*/
SofaScoreDayTournament::updateOrCreate([
'tournament_id' => (int) $tournament['id'],
'day' => $day,
]);
if ($debug) {
Notice::note("tournaments: $tournaments_counter", 'p');
}
}
if ($debug) {
Notice::note("tournaments: $tournaments_counter", 'p');
}
}
}
Notice::note('Archive created', 'g', 1);
Notice::note('days tournament done', 'g');
}
}
@@ -3,29 +3,34 @@
namespace App\Console\Commands\Grapper\Sofascore\Specials;
use App\Console\Commands\Grapper\Sofascore\Special;
use App\Console\Commands\Grapper\Sofascore\Tools\LoadDays;
use App\Utilities\Notice;
use Illuminate\Support\Facades\Mail;
class SendMail
{
public static function run($days)
public static function run()
{
foreach ($days as $day) {
$days = LoadDays::load();
foreach ($days as $d) {
$day = $d['day'];
$zip_file = Special::$storage_path.'/'.$day.'.zip';
Mail::send([], [], function ($message) use($day, $zip_file) {
$message->to("hermann.telser@gmail.com", "sofa archive ".$day)
->cc("ubaldo.perrucci@gmail.com", "sofa archive ".$day)
Mail::send([], [], function ($message) use ($day, $zip_file) {
$message->to('hermann.telser@gmail.com', 'sofa archive '.$day)
->cc('ubaldo.perrucci@gmail.com', 'sofa archive '.$day)
->subject("sofa archive $day")
->attach($zip_file, [
'mime' => 'application/x-zip-compressed',
'as' => 'sofa archive '.$day.'.zip'
'as' => 'sofa archive '.$day.'.zip',
]);
});
}
Notice::note('Mail send', 'g');
}
}
@@ -15,29 +15,33 @@ class LoadDays
$yesterday = Nuh::$options['y'] ?? 0;
if($today){
if ($today) {
return [Carbon::now()->format('Y-m-d')];
}
else if($yesterday){
} elseif ($yesterday) {
return [Carbon::now()->subDay()->format('Y-m-d')];
}
else {
} else {
$res = SofaScoreTask::with('sport')->where('is_active', 1)->get();
$dates = [];
foreach ($res as $row) {
switch($row->mode){
switch ($row->mode) {
case 'yesterday':
$dates[] = [
'day' => Carbon::now()->subDay()->format('Y-m-d'),
'slug' => $row->sport->ss_slug
'slug' => $row->sport->ss_slug,
'run_slow' => $row->run_slow,
'sport_id' => $row->sport_id,
'run_repeatedly' => $row->run_repeatedly,
];
break;
case 'date':
$dates[] = [
'day' => self::validateDate($row->day_from),
'slug' => $row->sport->ss_slug
'slug' => $row->sport->ss_slug,
'run_slow' => $row->run_slow,
'sport_id' => $row->sport_id,
'run_repeatedly' => $row->run_repeatedly,
];
break;
case 'range':
@@ -49,39 +53,40 @@ class LoadDays
$lengthdates = $from->diffInDays($to);
if($lengthdates < 1){
if ($lengthdates < 1) {
Notice::note('wrong date start must be before end', 'r', 1);
}
else if($lengthdates > 60){
} elseif ($lengthdates > 60) {
Notice::note('wrong date maximal 60 dates allowed', 'r', 1);
}
while ($from < $to){
while ($from < $to) {
$dates[] = [
'day' => $from->format('Y-m-d'),
'slug' => $row->sport->ss_slug
'slug' => $row->sport->ss_slug,
'run_slow' => $row->run_slow,
'sport_id' => $row->sport_id,
'run_repeatedly' => $row->run_repeatedly,
];
$from->addDay();
}
}
if (! $dates) {
Notice::note(__file__.PHP_EOL.'no dates', 'r', 1);
Notice::note(__FILE__.PHP_EOL.'no dates', 'r', 1);
}
}
}
// remove when element in array is double (day and slug)
$dates = array_unique($dates, SORT_REGULAR);
return $dates;
}
public static function validateDate(string $date)
{
if(Carbon::createFromDate($date) !== false){
if (Carbon::createFromDate($date) !== false) {
return $date;
}
else {
Notice::note('Wrong date format','r', 1);
} else {
Notice::note('Wrong date format', 'r', 1);
}
}
}
@@ -10,8 +10,7 @@ use App\Utilities\Notice;
* FIFA World Cup 16
* China Super League 649
* Veikkausliiga 41
*/
*/
class LoadTournaments
{
public static function load()
@@ -21,15 +20,14 @@ class LoadTournaments
$sport_id = Nuh::$options['sport_id'];
$grps = Nuh::$options['grps'] ?? 0;
if($grps){
$tornaments = SofaScoreTournament::select('name', 'tournament_id')->whereIn('tournament_id', explode(",", $grps))->where('enabled', 1)->get();
}
else{
if ($grps) {
$tornaments = SofaScoreTournament::select('name', 'tournament_id')->whereIn('tournament_id', explode(',', $grps))->where('enabled', 1)->get();
} else {
$tornaments = SofaScoreTournament::select('name', 'tournament_id')->where('sport_id', $sport_id)->where('enabled', 1)->get();
}
if (! $tornaments) {
Notice::note(__file__.PHP_EOL.'no tornaments', 'r', 1);
if (! $tornaments) {
Notice::note(__FILE__.PHP_EOL.'no tornaments', 'r', 1);
}
return $tornaments;
@@ -0,0 +1,30 @@
<?php
namespace App\Console\Commands\Grapper\Sofascore\Tools;
use App\Console\Commands\Nuh;
use App\Utilities\Notice;
class Sleeper
{
public static function rand(bool $langsam = false)
{
$debug = Nuh::$options['d'] ?? 0;
if ($debug) {
$langsam = false;
}
if ($langsam) {
$sleep = rand(500000, 50000000);
} else {
$sleep = rand(500000, 1000000);
}
if ($debug) {
Notice::note("sleeping $sleep msec", 'y');
}
usleep($sleep);
}
}