From 46eb3767f992950b317810729632d59e82d85b36 Mon Sep 17 00:00:00 2001 From: salam Date: Mon, 20 Jul 2026 08:27:33 +0200 Subject: [PATCH] wip --- .../Commands/Grapper/Sofascore/Grapper.php | 187 ++++++++++-------- .../Grapper/Sofascore/Helper/LoadDays.php | 55 ++++-- app/Console/Commands/Nuh.php | 26 +-- app/Models/SofaScoreTask.php | 1 - app/Models/SofaScoreTaskDay.php | 14 ++ ...1_062922_create_sofa_score_tasks_table.php | 1 - ...2820_create_sofa_score_task_days_table.php | 30 +++ grapper.sh | 16 +- 8 files changed, 208 insertions(+), 122 deletions(-) create mode 100644 app/Models/SofaScoreTaskDay.php create mode 100644 database/migrations/2026_07_19_082820_create_sofa_score_task_days_table.php diff --git a/app/Console/Commands/Grapper/Sofascore/Grapper.php b/app/Console/Commands/Grapper/Sofascore/Grapper.php index 5cba83f2..3d9cc527 100644 --- a/app/Console/Commands/Grapper/Sofascore/Grapper.php +++ b/app/Console/Commands/Grapper/Sofascore/Grapper.php @@ -9,6 +9,7 @@ use App\Console\Commands\Grapper\Sofascore\Helper\Sleeper; use App\Console\Commands\Nuh; use App\Models\SofaScoreDayTournament; use App\Models\SofaScoreEventGrapped; +use App\Models\SofaScoreTaskDay; use App\Models\SofaScoreTaskTournament; use App\Utilities\Notice; use Illuminate\Support\Carbon; @@ -22,6 +23,8 @@ class Grapper $task_id = Nuh::$options['task'] ?? 0; + $restart = Nuh::$options['restart'] ?? 0; + $days = LoadDays::load(); $active_tournaments = SofaScoreTaskTournament::with('tournament')->where('sofa_score_task_id', $task_id)->where('disabled', false)->get(); @@ -30,15 +33,22 @@ class Grapper * empty grapping folder * empty mailing text */ - if ($debug) - Notice::note('File clean directory', 'b'); + if ($restart) { - File::cleanDirectory(Sofascore::$storage); + File::cleanDirectory(Sofascore::$storage); - if ($debug) - Notice::note('Truncate SofaScoreEventGrapped', 'b'); + SofaScoreEventGrapped::where('grapper_id', $task_id)->delete(); - SofaScoreEventGrapped::where('grapper_id', $task_id)->delete(); + if ($debug) { + Notice::note('Restart grapper', 'br'); + Notice::note('Cleand task directory', 'b'); + Notice::note('Flush table SofaScoreEventGrapped for task', 'b'); + } + } else { + if ($debug) { + Notice::note('Continuing grapper', 'bg'); + } + } foreach ($days as $d) { @@ -46,16 +56,16 @@ class Grapper foreach ($active_tournaments as $tournament) { - if ($debug) { - Notice::note($tournament->tournament->name, 'bg'); - } - - $day = $d['day']; + if ($debug) { + Notice::note($day . ' - ' . $tournament->tournament->name, 'b'); + } + $res = SofaScoreDayTournament::whereDate('day', '=', $day)->where('tournament_id', $tournament->sofa_score_tournament_id)->first(); if ($res) { + if ($debug) { Notice::note($day, 'g'); } @@ -74,125 +84,140 @@ class Grapper foreach ($data['events'] as $event) { - $now = Carbon::now()->subHours(3); - - $carbon_day = new Carbon($day); - - $starttime = Carbon::createFromTimestamp($event['startTimestamp']); - - if ($now->greaterThan($starttime) && $starttime->isSameDay($carbon_day)) { - - Sleeper::rand(1); - + if (SofaScoreEventGrapped::where('id', (int) $event['id'])->exists()) { if ($debug) { - Notice::note($day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'bg'); + Notice::note('Grapped ' . $day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'g'); } + } else { - $grapped = SofaScoreEventGrapped::firstOrNew(['id' => $event['id']]); - $grapped->grapper_id = $task_id; - $grapped->day = $day; - $grapped->liga = $event['tournament']['name']; - $grapped->name = $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name']; - $grapped->url = $url; - $grapped->starttime = $starttime->toDateTimeString(); + $now = Carbon::now()->subHours(3); - $folder = Sofascore::$storage . '/' . $day . '/' . $event['tournament']['slug'] . '/' . $event['slug']; + $carbon_day = new Carbon($day); - File::makeDirectory($folder, 0775, true, true); + $starttime = Carbon::createFromTimestamp($event['startTimestamp']); - /********************************* - * EVENT event - *********************************/ + if ($now->greaterThan($starttime) && $starttime->isSameDay($carbon_day)) { - File::put($folder . '/event.json', json_encode($event)); + Sleeper::rand(1); - /********************************* - * EVENT statistics - *********************************/ + if ($debug) { + Notice::note('New ' . $day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'bg'); + } - $event_id = $event['id']; + $grapped = SofaScoreEventGrapped::firstOrNew(['id' => $event['id']]); + $grapped->grapper_id = $task_id; + $grapped->day = $day; + $grapped->liga = $event['tournament']['name']; + $grapped->name = $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name']; + $grapped->url = $url; + $grapped->starttime = $starttime->toDateTimeString(); - // if (!$grapped['got_statistics'] && isset($event['hasEventPlayerStatistics']) && $event['hasEventPlayerStatistics'] == true) { + $folder = Sofascore::$storage . '/' . $day . '/' . $event['tournament']['slug'] . '/' . $event['slug']; - $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/statistics"; + File::makeDirectory($folder, 0775, true, true); - Sleeper::rand(); + /********************************* + * EVENT event + *********************************/ - $data = CurlSofa::load($url); + File::put($folder . '/event.json', json_encode($event)); - $data['id'] = $event_id; + /********************************* + * EVENT statistics + *********************************/ - File::put($folder . '/statistics.json', json_encode($data)); + $event_id = $event['id']; - $grapped->got_statistics = 1; + // if (!$grapped['got_statistics'] && isset($event['hasEventPlayerStatistics']) && $event['hasEventPlayerStatistics'] == true) { + + $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/statistics"; + + Sleeper::rand(); + + $data = CurlSofa::load($url); + + $data['id'] = $event_id; + + File::put($folder . '/statistics.json', json_encode($data)); + + $grapped->got_statistics = 1; // } - /********************************* - * EVENT lineups - *********************************/ + /********************************* + * EVENT lineups + *********************************/ - // if (!$grapped['got_lineups']) { + // if (!$grapped['got_lineups']) { - $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/lineups"; + $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/lineups"; - Sleeper::rand(); + Sleeper::rand(); - $data = CurlSofa::load($url); + $data = CurlSofa::load($url); - $data['id'] = $event_id; + $data['id'] = $event_id; - File::put($folder . '/lineups.json', json_encode($data)); + File::put($folder . '/lineups.json', json_encode($data)); - $grapped->got_lineups = 1; + $grapped->got_lineups = 1; // } - /********************************* - * EVENT odds - *********************************/ + /********************************* + * EVENT odds + *********************************/ - // if (!$grapped['got_odds']) { + // if (!$grapped['got_odds']) { - $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/odds/1/all"; + $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/odds/1/all"; - Sleeper::rand(); + Sleeper::rand(); - $data = CurlSofa::load($url); + $data = CurlSofa::load($url); - $data['id'] = $event_id; + $data['id'] = $event_id; - File::put($folder . '/odds.json', json_encode($data)); + File::put($folder . '/odds.json', json_encode($data)); - $grapped->got_odds = 1; + $grapped->got_odds = 1; // } - /********************************* - * EVENT incidents - *********************************/ + /********************************* + * EVENT incidents + *********************************/ - // if (!$grapped['got_incidents']) { + // if (!$grapped['got_incidents']) { - $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/incidents"; + $url = Nuh::$options['base_url'] . "api/v1/event/$event_id/incidents"; - Sleeper::rand(); + Sleeper::rand(); - $data = CurlSofa::load($url); + $data = CurlSofa::load($url); - $data['id'] = $event_id; + $data['id'] = $event_id; - File::put($folder . '/incidents.json', json_encode($data)); + File::put($folder . '/incidents.json', json_encode($data)); - $grapped->got_incidents = 1; - // } + $grapped->got_incidents = 1; + // } - $grapped->save(); + $grapped->save(); + } } } } } + if ($debug) { + Notice::note('END ' . $tournament->tournament->name, 'p'); + } } - if ($debug) { - Notice::note('tournament done', 'bb'); - } + SofaScoreTaskDay::updateOrCreate([ + 'grapper_id' => (int) $task_id, + 'day' => $day, + ], [ + 'done' => 1, + ]); + + Notice::note('DAY DONE', 'br', 1); } } } diff --git a/app/Console/Commands/Grapper/Sofascore/Helper/LoadDays.php b/app/Console/Commands/Grapper/Sofascore/Helper/LoadDays.php index 52fb3463..63298487 100644 --- a/app/Console/Commands/Grapper/Sofascore/Helper/LoadDays.php +++ b/app/Console/Commands/Grapper/Sofascore/Helper/LoadDays.php @@ -4,6 +4,7 @@ namespace App\Console\Commands\Grapper\Sofascore\Helper; use App\Console\Commands\Nuh; use App\Models\SofaScoreTask; +use App\Models\SofaScoreTaskDay; use App\Utilities\Notice; use Illuminate\Support\Carbon; @@ -13,6 +14,8 @@ class LoadDays { $debug = Nuh::$options['d'] ?? 0; + $restart = Nuh::$options['restart'] ?? 0; + $task_id = Nuh::$options['task'] ?? 0; $dates = []; @@ -25,30 +28,28 @@ class LoadDays $task = SofaScoreTask::with('sport')->where('id', $task_id)->first(); if ($debug) { - Notice::note('LoadDays ' . $task->mode . ' sport: ' . $task->sport_id); + Notice::note('Task id:' . $task->id . ', mode:' . $task->mode . ', sport:' . $task->sport->name); } - switch ($task->id) { + switch ($task->mode) { - case 1: + case 'auto_updater': $dates[] = [ 'day' => Carbon::now()->subDay()->format('Y-m-d'), 'slug' => $task->sport->ss_slug, 'sport_id' => $task->sport_id, - 'run_repeatedly' => $task->run_repeatedly, ]; break; - case 2: + case 'date': $dates[] = [ 'day' => self::validateDate($task->day_from), 'slug' => $task->sport->ss_slug, 'sport_id' => $task->sport_id, - 'run_repeatedly' => $task->run_repeatedly, ]; break; - case 3: + case 'range': $from = self::validateDate($task->day_from); $to = self::validateDate($task->day_to); @@ -58,18 +59,36 @@ class LoadDays $lengthdates = $from->diffInDays($to); if ($lengthdates < 1) { - Notice::note('wrong date start must be before end', 'r', 1); - } elseif ($lengthdates > 60) { - Notice::note('wrong date maximal 60 dates allowed', 'r', 1); + Notice::note('start must be before end', 'r', 1); } - while ($from <= $to) { - $dates[] = [ - 'day' => $from->format('Y-m-d'), - 'slug' => $task->sport->ss_slug, - 'sport_id' => $task->sport_id, - 'run_repeatedly' => $task->run_repeatedly, - ]; - $from->addDay(); + + if ($restart) { + // get all days + while ($from <= $to) { + $dates[] = [ + 'day' => $from->format('Y-m-d'), + 'slug' => $task->sport->ss_slug, + 'sport_id' => $task->sport_id, + ]; + $from->addDay(); + } + } else { + // remove days done + while ($from <= $to) { + + $res = SofaScoreTaskDay::firstOrNew(['grapper_id' => (int) $task_id, 'day' => $from->format('Y-m-d')]); + + if (!$res->done) { + $dates[] = [ + 'day' => $from->format('Y-m-d'), + 'slug' => $task->sport->ss_slug, + 'sport_id' => $task->sport_id, + ]; + } + $from->addDay(); + } + + shuffle($dates); } break; } diff --git a/app/Console/Commands/Nuh.php b/app/Console/Commands/Nuh.php index d8d96c64..0a5f928c 100644 --- a/app/Console/Commands/Nuh.php +++ b/app/Console/Commands/Nuh.php @@ -23,7 +23,7 @@ class Nuh extends Command public static $id = 0; - protected $signature = 'nuh {process=help} {--truncate} {--id=0} {--fun=} {--task=0} {--gid=} {--p} {--d} {--x} {--t} {--y} {--m} {--l} {--w} {--a} {--r} {--s} {--g} {--t} {--q} {--f} {--b} {--e} {--z}'; + protected $signature = 'nuh {process=help} {--truncate} {--id=0} {--fun=} {--task=0} {--restart=0} {--gid=} {--p} {--d} {--x} {--t} {--y} {--m} {--l} {--w} {--a} {--r} {--s} {--g} {--t} {--q} {--f} {--b} {--e} {--z}'; public function handle() { @@ -31,12 +31,12 @@ class Nuh extends Command self::$options['supervisor_id'] = $this->argument('process'); $queries = self::$options['q'] ?? 0; - + $show_supervisors = self::$options['z'] ?? 0; if ($this->argument('process') == 'help') { $txt = PHP_EOL; - if(!$show_supervisors){ + if (!$show_supervisors) { $txt .= 'Parameters' . PHP_EOL; $txt .= '-----------------------------------------------------------' . PHP_EOL; $txt .= 'process [id] (required)' . PHP_EOL; @@ -47,15 +47,15 @@ class Nuh extends Command $txt .= '-----------------------------------------------------------' . PHP_EOL . PHP_EOL; $txt .= 'Sofascore (67)' . PHP_EOL; $txt .= '-----------------------------------------------------------' . PHP_EOL; - $txt .= '--fun=[1,2,3,4,5,6,7]' - .PHP_EOL.chr(9).'1) player/season' - .PHP_EOL.chr(9).'2) setting to local' - .PHP_EOL.chr(9).'3) create zip' - .PHP_EOL.chr(9).'4) mail folder' - .PHP_EOL.chr(9).'5) grap tornaments/day' - .PHP_EOL.chr(9).'6) run grapper' - .PHP_EOL.chr(9).'7) update task' - .PHP_EOL; + $txt .= '--fun=[1,2,3,4,5]' + . PHP_EOL . chr(9) . '1) player/season' + . PHP_EOL . chr(9) . '2) setting to local' + . PHP_EOL . chr(9) . '3) grapper' + . PHP_EOL . chr(9) . '4) create zip' + . PHP_EOL . chr(9) . '5) mail folder' + . PHP_EOL; + $txt .= '--task=[id from sofa_score_tasks table]'; + $txt .= '--restart=[0|1] delete archive and flusch DB for taks, start new'; $txt .= '-----------------------------------------------------------' . PHP_EOL . PHP_EOL; $txt .= 'Options' . PHP_EOL; $txt .= '-----------------------------------------------------------' . PHP_EOL; @@ -80,7 +80,7 @@ class Nuh extends Command $txt .= '-----------------------------------------------------------' . PHP_EOL . PHP_EOL; $txt .= 'Supervisors (--z)' . PHP_EOL; - if($show_supervisors){ + if ($show_supervisors) { $txt .= '-----------------------------------------------------------' . PHP_EOL; $tmp = Supervisor::orderBy('provider_id')->get(); foreach ($tmp as $value) { diff --git a/app/Models/SofaScoreTask.php b/app/Models/SofaScoreTask.php index d35bf82b..42d680fe 100644 --- a/app/Models/SofaScoreTask.php +++ b/app/Models/SofaScoreTask.php @@ -17,7 +17,6 @@ class SofaScoreTask extends Model 'day_to', 'is_active', 'compleated', - 'run_repeatedly', 'last_start', 'last_end', ]; diff --git a/app/Models/SofaScoreTaskDay.php b/app/Models/SofaScoreTaskDay.php new file mode 100644 index 00000000..cfb53051 --- /dev/null +++ b/app/Models/SofaScoreTaskDay.php @@ -0,0 +1,14 @@ +date('day_to')->nullable(); $table->boolean('is_active')->default(0); $table->boolean('compleated')->default(0); - $table->boolean('run_repeatedly')->default(0); $table->dateTime('last_start')->nullable(); $table->dateTime('last_end')->nullable(); $table->timestamps(); diff --git a/database/migrations/2026_07_19_082820_create_sofa_score_task_days_table.php b/database/migrations/2026_07_19_082820_create_sofa_score_task_days_table.php new file mode 100644 index 00000000..7a2592c3 --- /dev/null +++ b/database/migrations/2026_07_19_082820_create_sofa_score_task_days_table.php @@ -0,0 +1,30 @@ +id(); + $table->integer('grapper_id'); + $table->date('day')->nullable(); + $table->boolean('done')->default(false); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('sofa_score_task_days'); + } +}; diff --git a/grapper.sh b/grapper.sh index 5a8d3449..28f956d6 100755 --- a/grapper.sh +++ b/grapper.sh @@ -8,7 +8,7 @@ case "$1" in echo "yesterday 1" echo "dates 2" echo "--------------------------" - echo "SELECT liga, name, starttime FROM sofa_score_event_grappeds WHERE in_current_grapper = 1;" + echo "SELECT liga, name, starttime FROM sofa_score_event_grappeds WHERE grapper_id = 1;" echo "" exit 1 ;; @@ -17,22 +17,22 @@ case "$1" in echo "\n⚽ sofa_score_task $1" - # echo "\n⚽ close all vpn" + echo "\n⚽ close all vpn" # killall ssh - # echo "\n⚽ open vpn" + echo "\n⚽ open vpn" # ssh -f -N -L 13306:127.0.0.1:3306 -p 58371 root@162.55.242.190 - # echo "\n⚽ Settings To Local" + echo "\n⚽ Settings To Local" # php artisan nuh --d --f 67 --fun=2 --task=$1 - # echo "\n⚽ close vpn" + echo "\n⚽ close vpn" # killall ssh - # echo "\n⚽ Grapper" - php artisan nuh --d --f 67 --fun=3 --task=$1 + echo "\n⚽ Grapper" + php artisan nuh --d --f 67 --fun=3 --task=$1 --restart=$2 - # echo "\n⚽ CreateZip" + echo "\n⚽ CreateZip" # php artisan nuh --d --f 67 --fun=4 --task=$1 # echo "\n⚽ SendMail"