salam/nuhodds_fi4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

centralized grapper tournaments wi

Browse Source
This commit is contained in:
salam committed 2026-10-11 15:42:00 +02:00
1 parent 34701ecbff
commit db917bdc92
8 files changed
+379 -173
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Models/SofaScoreDayTournament.php
+4
View File
@@ -6,6 +6,10 @@ use Illuminate\Database\Eloquent\Model;
class SofaScoreDayTournament extends Model
{
protected $connection = 'mariadb';
protected $table = 'sofa_score_day_tournaments';
protected $fillable = [
'id',
'tournament_id',
app/Models/SofaScoreEventGrapped.php
+2
View File
@@ -11,6 +11,8 @@ class SofaScoreEventGrapped extends Model
'grapper_id',
'name',
'url',
'day',
'liga',
'starttime',
'got_statistics',
'got_lineups',
app/Traits/Sofascore/GrappingEvent.php
+170 -132
View File
@@ -2,9 +2,7 @@
namespace App\Traits\Sofascore;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreTaskDay;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
@@ -14,150 +12,190 @@ trait GrappingEvent
{
protected function get_event(string $day, array $tournament)
{
$res = SofaScoreDayTournament::whereDate('day', '=', $day)->where('tournament_id', $tournament['sofa_score_tournament_id'])->first();
foreach ($this->day_tournaments as $day_tournament) {
if ($res) {
if ($day_tournament->tournament_id == $tournament['sofa_score_tournament_id']) {
if ($this->debug) {
Notice::note("$day - " . $tournament['tournament']['name'] . ' - ' . $tournament['sofa_score_tournament_id'], 'bb');
}
if ($this->debug) {
Notice::note("$day - ".$tournament['tournament']['name'].' - '.$tournament['sofa_score_tournament_id'], 'bb');
}
$url = $this->base_url . 'api/v1/unique-tournament/' . $tournament['sofa_score_tournament_id'] . "/scheduled-events/$day";
$url = $this->base_url.'api/v1/unique-tournament/'.$tournament['sofa_score_tournament_id']."/scheduled-events/$day";
if ($this->debug) {
Notice::note($url, 'p');
}
if ($this->debug) {
Notice::note($url, 'p');
}
$this->sleep(1);
$this->sleep(1);
$cloudflareService = new CloudflareSolverService;
$cloudflareService = new CloudflareSolverService;
$data = $cloudflareService->getJsonFromUrl($url);
$data = $cloudflareService->getJsonFromUrl($url);
if (isset($data['events'])) {
if (isset($data['events'])) {
foreach ($data['events'] as $event) {
foreach ($data['events'] as $event) {
if (! $this->restart && SofaScoreEventGrapped::where('grapper_id', $this->task_id)->where('id', (int) $event['id'])->exists()) {
if ($this->debug) {
Notice::note('DB - Allready grapped at ' . $day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'b');
$base_check_string = $day.'-'.$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug'];
$res = SofaScoreEventGrapped::where('id', (int) $event['id'])->where('grapper_id', $this->task_id)->first();
if (! $res) {
$base_check_string = $day.'-'.$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug'];
// chinese-super-league
if (! $this->restart && SofaScoreEventGrapped::where('grapper_id', $this->task_id)->where('id', (int) $event['id'])->exists()) {
if ($this->debug) {
Notice::note('DB - Allready grapped at '.$day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'b');
}
} else {
if ($this->debug) {
Notice::note("$day - ".$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'g');
}
$folder = strtolower("$this->data_dir/$day/$day-".$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug']);
File::makeDirectory($folder, 0775, true, true);
/*********************************
* EVENT event
*********************************/
File::put("$folder/event.json", json_encode($event));
/*********************************
* EVENT statistics
*********************************/
$event_id = $event['id'];
if (! $this->grapped($base_check_string.'/statistics.json')) {
$url = $this->base_url."api/v1/event/$event_id/statistics";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/statistics.json", json_encode($data));
} else {
if ($this->debug) {
Notice::note('already grapped statistics.json', 'y');
}
}
/*********************************
* EVENT lineups
*********************************/
if (! $this->grapped($base_check_string.'/lineups.json')) {
$url = $this->base_url."api/v1/event/$event_id/lineups";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/lineups.json", json_encode($data));
} else {
if ($this->debug) {
Notice::note('already grapped lineups.json', 'y');
}
}
/*********************************
* EVENT odds
*********************************/
if (! $this->grapped($base_check_string.'/odds.json')) {
$url = $this->base_url."api/v1/event/$event_id/odds/1/all";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/odds.json", json_encode($data));
} else {
if ($this->debug) {
Notice::note('already grapped odds.json', 'y');
}
}
/*********************************
* EVENT incidents
*********************************/
if (! $this->grapped($base_check_string.'/incidents.json')) {
$url = $this->base_url."api/v1/event/$event_id/incidents";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/incidents.json", json_encode($data));
} else {
if ($this->debug) {
Notice::note('already grapped incidents.json', 'y');
}
}
$starttime = Carbon::createFromTimestamp($event['startTimestamp']);
SofaScoreEventGrapped::updateOrCreate([
'id' => (int) $event['id'],
], [
'grapper_id' => $this->task_id,
'day' => $day,
'liga' => $event['tournament']['name'],
'name' => $event['homeTeam']['name'].' vs '.$event['awayTeam']['name'],
'url' => $url,
'starttime' => $starttime->toDateTimeString(),
]);
}
} else {
if ($this->debug) {
Notice::note('grapped: '.$base_check_string, 'p');
}
}
} else {
if ($this->debug) {
Notice::note("$day - " . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'g');
}
$folder = strtolower("$this->data_dir/$day/$day-" . $event['tournament']['category']['slug'] . '-' . $event['tournament']['slug'] . '/' . $event['slug']);
File::makeDirectory($folder, 0775, true, true);
/*********************************
* EVENT event
*********************************/
File::put("$folder/event.json", json_encode($event));
/*********************************
* EVENT statistics
*********************************/
$event_id = $event['id'];
$url = $this->base_url . "api/v1/event/$event_id/statistics";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/statistics.json", json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = $this->base_url . "api/v1/event/$event_id/lineups";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/lineups.json", json_encode($data));
/*********************************
* EVENT odds
*********************************/
$url = $this->base_url . "api/v1/event/$event_id/odds/1/all";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/odds.json", json_encode($data));
/*********************************
* EVENT incidents
*********************************/
$url = $this->base_url . "api/v1/event/$event_id/incidents";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/incidents.json", json_encode($data));
$starttime = Carbon::createFromTimestamp($event['startTimestamp']);
SofaScoreEventGrapped::updateOrCreate([
'id' => (int) $event['id'],
], [
'grapper_id' => $this->task_id,
'day' => $day,
'liga' => $event['tournament']['name'],
'name' => $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'],
'url' => $url,
'starttime' => $starttime->toDateTimeString(),
]);
}
}
}
} else {
if ($this->debug) {
Notice::note($day . ' - ' . $tournament['tournament']['name'] . " no events", 'y');
return;
}
}
SofaScoreTaskDay::updateOrCreate([
'grapper_id' => (int) $this->task_id,
'day' => $day,
], [
'done' => 1,
]);
if ($this->debug) {
Notice::note($day.' - '.$tournament['tournament']['name'].' no events', 'y');
}
}
}
app/Traits/Sofascore/GrappingHelper.php
+35 -21
View File
@@ -2,17 +2,12 @@
namespace App\Traits\Sofascore;
use App\Models\Country;
use App\Models\SofaScoreCategory;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreRosa;
use App\Models\SofaScoreTask;
use App\Models\SofaScoreTaskDay;
use App\Models\SofaScoreTaskTournament;
use App\Models\SofaScoreTournament;
use App\Models\StatistikoImport;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
@@ -97,16 +92,16 @@ trait GrappingHelper
{
switch ($grapperSelected) {
case 'daily':
$this->data_dir = storage_path($this->base_dir) . '/daily';
$this->data_dir = storage_path($this->base_dir).'/daily';
break;
case 'range':
$this->data_dir = storage_path($this->base_dir) . '/range';
$this->data_dir = storage_path($this->base_dir).'/range';
break;
case 'rose':
$this->data_dir = storage_path($this->base_dir) . '/rose';
$this->data_dir = storage_path($this->base_dir).'/rose';
break;
case 'player':
$this->data_dir = storage_path($this->base_dir) . '/player';
$this->data_dir = storage_path($this->base_dir).'/player';
break;
case 'excel':
case 'mail':
@@ -125,6 +120,12 @@ trait GrappingHelper
*/
protected function load_tournaments_requested(string $grapperSelected)
{
/**
* SofaScoreDayTournament
*/
if ($this->debug) {
Notice::note('get tournaments for grapper', 'g');
}
switch ($grapperSelected) {
case 'daily':
$this->tournaments_requested = SofaScoreTaskTournament::with('tournament')->where('sofa_score_task_id', 1)->where('disabled', false)->get()->toArray();
@@ -217,20 +218,33 @@ trait GrappingHelper
protected function get_tree(string $dir, &$results = [])
{
$files = scandir($dir);
foreach ($files as $value) {
$path = realpath($dir . DIRECTORY_SEPARATOR . $value);
if (! is_dir($path)) {
$results[] = $path;
} elseif ($value != '.' && $value != '..') {
self::get_tree($path, $results);
$results[] = $path;
if (File::isDirectory($dir)) {
$files = scandir($dir);
foreach ($files as $value) {
$path = realpath($dir.DIRECTORY_SEPARATOR.$value);
if (! is_dir($path)) {
$results[] = $path;
} elseif ($value != '.' && $value != '..') {
self::get_tree($path, $results);
$results[] = $path;
}
}
}
return $results;
}
protected function grapped(string $string)
{
foreach ($this->grapped_files as $file) {
if (strpos($file, $string) !== false) {
return true;
}
}
return false;
}
protected function clean_string(string $string)
{
$string = str_replace(' ', '-', $string); // Replaces all spaces with hyphens.
@@ -242,7 +256,7 @@ trait GrappingHelper
protected function clear_folder(string $folder, $what = '')
{
$mainFolder = storage_path($this->base_dir . '/' . $folder);
$mainFolder = storage_path($this->base_dir.'/'.$folder);
if (! $what) {
$what = $this->select_what_delete();
@@ -273,8 +287,8 @@ trait GrappingHelper
$tasks = StatistikoImport::pluck('file_name')->toArray();
collect(Storage::disk('public')->allFiles('statistiko'))
->reject(fn(string $file) => $file === '.gitignore')
->reject(fn(string $file) => in_array($file, $tasks))
->each(fn($file) => Storage::disk('public')->delete($file));
->reject(fn (string $file) => $file === '.gitignore')
->reject(fn (string $file) => in_array($file, $tasks))
->each(fn ($file) => Storage::disk('public')->delete($file));
}
}
app/Traits/Sofascore/GrappingSofascore.php
+40 -15
View File
@@ -3,8 +3,11 @@
namespace App\Traits\Sofascore;
use App\Mail\SimpleMail;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreTaskDay;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Mail;
@@ -37,6 +40,10 @@ trait GrappingSofascore
protected array $tournaments_requested = [];
protected array $grapped_files = [];
protected Collection $day_tournaments;
protected array $days = [];
public function handle(): void
@@ -116,8 +123,6 @@ trait GrappingSofascore
$this->task_id = 1;
$this->restart = 1;
$this->set_base_dir($folder);
$this->load_tournaments_requested($folder);
@@ -126,9 +131,14 @@ trait GrappingSofascore
switch ($restart) {
case 'restart':
$this->restart = 1;
SofaScoreEventGrapped::truncate();
break;
case 'clear':
$this->restart = 1;
SofaScoreEventGrapped::truncate();
$this->clear_folder($folder, 'all');
break;
}
@@ -136,37 +146,47 @@ trait GrappingSofascore
/**
* day 1
*/
$day = Carbon::now()->subDay()->format('Y-m-d');
$this->grap_tournaments_for_day($day);
$res = SofaScoreTaskDay::where('grapper_id', (int) $this->task_id)->where('day', $day)->first();
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
if (! $res) {
$this->grapped_files = $this->get_tree($this->data_dir.'/'.$day);
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
SofaScoreTaskDay::updateOrCreate(['grapper_id' => (int) $this->task_id, 'day' => $day], ['done' => 1]);
}
/**
* day 5
*/
$day = Carbon::now()->subDays(5)->format('Y-m-d');
$this->grap_tournaments_for_day($day);
$res = SofaScoreTaskDay::where('grapper_id', (int) $this->task_id)->where('day', $day)->first();
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
if (! $res) {
$this->grapped_files = $this->get_tree($this->data_dir.'/'.$day);
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
SofaScoreTaskDay::updateOrCreate(['grapper_id' => (int) $this->task_id, 'day' => $day], ['done' => 1]);
}
/**
* to excel day by day
*/
$this->import_to_excel($folder);
/**
* mail it
*/
$this->send($folder);
}
@@ -188,6 +208,8 @@ trait GrappingSofascore
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
SofaScoreTaskDay::updateOrCreate(['grapper_id' => (int) $this->task_id, 'day' => $day], ['done' => 1]);
}
if ($this->debug) {
@@ -243,6 +265,8 @@ trait GrappingSofascore
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
SofaScoreTaskDay::updateOrCreate(['grapper_id' => (int) $this->task_id, 'day' => $day], ['done' => 1]);
}
}
@@ -258,8 +282,9 @@ trait GrappingSofascore
$directories = File::directories($this->data_dir);
$arr = [];
foreach ($directories as $directory) {
$arr[] = substr(strrchr($directory, "/"), 1);
$arr[] = substr(strrchr($directory, '/'), 1);
}
return $arr;
}
app/Traits/Sofascore/GrappingTournaments.php
+43 -4
View File
@@ -20,25 +20,52 @@ trait GrappingTournaments
{
$sport_id = 1;
$has = SofaScoreDayTournament::where('day', $day)->first();
if ($this->debug) {
Notice::note('⚽ tournaments for day '.$day);
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
$slug = 'football';
if ($this->debug) {
Notice::note('⚽ open vpn');
}
exec('ssh -f -N -L 13306:127.0.0.1:3306 -p 58371 root@162.55.242.190');
/**
* SofaScoreDayTournament
*/
if ($this->debug) {
Notice::note('get tournaments', 'g');
}
if (! $this->force && $has) {
$this->day_tournaments = SofaScoreDayTournament::where('day', $day)->get();
if (! $this->restart && $this->day_tournaments->count() > 0) {
if ($this->debug) {
Notice::note("$day Tournaments allready grapped", 'bb');
}
if ($this->debug) {
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
return;
} else {
$page = 1;
$tournaments_counter = 0;
$slug = 'football';
while ($page) {
// $data = $this->curl($url);
if ($this->debug) {
Notice::note('⚽ close all vpn and load tournaments');
}
$this->sleep();
$url = $this->base_url . "api/v1/sport/$slug/scheduled-tournaments/$day/page/$page";
$url = $this->base_url."api/v1/sport/$slug/scheduled-tournaments/$day/page/$page";
$cloudflareService = new CloudflareSolverService;
@@ -158,6 +185,18 @@ trait GrappingTournaments
if ($this->debug) {
Notice::note("$day Tournaments loaded", 'bb');
}
if ($this->debug) {
Notice::note('⚽ close all vpn');
}
$this->day_tournaments = SofaScoreDayTournament::where('day', $day)->get();
if ($this->debug) {
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
}
}
}