From 911b5150213fc7a68a68ed5a1b8bd68247d3b349 Mon Sep 17 00:00:00 2001 From: salam Date: Sat, 26 Sep 2026 17:49:06 +0200 Subject: [PATCH] wip --- app/Console/Commands/Grap.php | 82 ++-- app/Console/Commands/Grapper.php | 20 + .../Commands/Grapper/Sofascore/Grapper.php | 52 +-- .../Grapper/Sofascore/Helper/CurlSofa.php | 5 +- app/Console/Sofascore | 1 - app/Models/SofaScoreEventGrapped.php | 1 + app/Services/CloudflareSolverService.php | 63 +++ app/Services/docker-compose.yml | 11 + app/Traits/CurlSofa.php | 87 ++++ app/Traits/GrapSofascore.php | 146 +++++++ app/Traits/GrapperEvent.php | 159 ++++++++ app/Traits/GrappingHelper.php | 386 ++++++++++++++++++ app/Traits/GrappingToExcel.php | 85 ++++ config/services.php | 4 + ...create_sofa_score_event_grappeds_table.php | 2 +- 15 files changed, 1048 insertions(+), 56 deletions(-) create mode 100644 app/Console/Commands/Grapper.php delete mode 120000 app/Console/Sofascore create mode 100644 app/Services/CloudflareSolverService.php create mode 100644 app/Services/docker-compose.yml create mode 100644 app/Traits/CurlSofa.php create mode 100644 app/Traits/GrapSofascore.php create mode 100644 app/Traits/GrapperEvent.php create mode 100644 app/Traits/GrappingHelper.php create mode 100644 app/Traits/GrappingToExcel.php diff --git a/app/Console/Commands/Grap.php b/app/Console/Commands/Grap.php index af297178..5cfc2e77 100644 --- a/app/Console/Commands/Grap.php +++ b/app/Console/Commands/Grap.php @@ -5,7 +5,6 @@ namespace App\Console\Commands; use App\Console\Commands\Grapper\Sofascore\Grapper; use App\Console\Commands\Grapper\Sofascore\GrapperTest; use App\Console\Commands\Grapper\Sofascore\Helper\ClearDir; -use App\Console\Commands\Grapper\Sofascore\Helper\RemoveDir; use App\Console\Commands\Grapper\Sofascore\Helper\SetStorage; use App\Console\Commands\Grapper\Sofascore\MakeExcel; use App\Console\Commands\Grapper\Sofascore\Rose; @@ -29,7 +28,7 @@ class Grap extends Command public static string $useragent = '4'; - protected $signature = 'grp {id=0} {day=0} {--date=} {--folder=} {--d} {--g} {--r} {--c} {--m} {--f} {--F} {--R} {--S}'; + protected $signature = 'grp {id=0} {mode} {day=0} {--date=} {--folder=} {--d} {--g} {--r} {--c} {--m} {--f} {--F} {--R} {--S}'; public function handle() { @@ -39,38 +38,71 @@ class Grap extends Command self::$options['base_url'] = 'https://www.sofascore.com/'; if (! self::$options['id']) { - $txt = 'PARAMETERS' . PHP_EOL; - $txt .= '--d debug' . PHP_EOL; - $txt .= '--r restart range' . PHP_EOL; - $txt .= '--f force Events' . PHP_EOL; - $txt .= '--F force DayTournament links' . PHP_EOL; - $txt .= '--c clear grap dir' . PHP_EOL; - $txt .= '--m mail only' . PHP_EOL; - $txt .= '--C clear folder' . PHP_EOL; - $txt .= '--G grap' . PHP_EOL; - $txt .= '--E make excel' . PHP_EOL; - $txt .= '--M mail' . PHP_EOL; - $txt .= '--folder' . PHP_EOL; - $txt .= '-----------------------------------------------------------' . PHP_EOL; - $txt .= 'FUNCTION' . PHP_EOL; - $txt .= ' 1 grapper DAILY & subDays ( default 3)' . PHP_EOL; - $txt .= ' 2 grapper MORE DAYS' . PHP_EOL; - $txt .= ' 3 grapper RANGE' . PHP_EOL; - $txt .= ' 4 grapper ROSE' . PHP_EOL; - $txt .= ' 5 import to excel from folder' . PHP_EOL; - $txt .= ' 7 settings to local' . PHP_EOL; - $txt .= ' 11 move SQL' . PHP_EOL; - $txt .= ' 12 testing url for grapping' . PHP_EOL; + $txt = 'PARAMETERS'.PHP_EOL; + $txt .= '--d debug'.PHP_EOL; + $txt .= '--r restart range'.PHP_EOL; + $txt .= '--f force Events'.PHP_EOL; + $txt .= '--F force DayTournament links'.PHP_EOL; + $txt .= '--c clear grap dir'.PHP_EOL; + $txt .= '--m mail only'.PHP_EOL; + $txt .= '--C clear folder'.PHP_EOL; + $txt .= '--G grap'.PHP_EOL; + $txt .= '--E make excel'.PHP_EOL; + $txt .= '--M mail'.PHP_EOL; + $txt .= '--folder'.PHP_EOL; + $txt .= '-----------------------------------------------------------'.PHP_EOL; + $txt .= 'FUNCTION'.PHP_EOL; + $txt .= ' 1 grapper DAILY & subDays ( default 3)'.PHP_EOL; + $txt .= ' 2 grapper MORE DAYS'.PHP_EOL; + $txt .= ' 3 grapper RANGE'.PHP_EOL; + $txt .= ' 4 grapper ROSE'.PHP_EOL; + $txt .= ' 5 import to excel from folder'.PHP_EOL; + $txt .= ' 7 settings to local'.PHP_EOL; + $txt .= ' 11 move SQL'.PHP_EOL; + $txt .= ' 12 testing url for grapping'.PHP_EOL; Notice::note($txt, 'b', 1); } - Notice::note('Task Id ' . self::$options['id'], 'gb'); + Notice::note('Task Id '.self::$options['id'], 'gb'); + + if (self::$options['cear']) { + SetStorage::set(1); + ClearDir::clear(); + } + + if (self::$options['grap']) { + SetStorage::set(1); + Grapper::load(1); + } + + if (self::$options['excel']) { + SetStorage::set(1); + MakeExcel::create(); + } + + if (self::$options['mail']) { + SetStorage::set(1); + SendMail::run(1); + } + + case 4: + // ROSE + SetStorage::set(4); + Rose::run(); + MakeExcel::create(); + + break; switch (self::$options['id']) { case 1: // DAILY + if (self::$options['c']) { + SetStorage::set(1); + ClearDir::clear(); + } + if (self::$options['C']) { Grapper::load(1); } diff --git a/app/Console/Commands/Grapper.php b/app/Console/Commands/Grapper.php new file mode 100644 index 00000000..16b4b33e --- /dev/null +++ b/app/Console/Commands/Grapper.php @@ -0,0 +1,20 @@ +runGrapper($grapperSelected); + } +} diff --git a/app/Console/Commands/Grapper/Sofascore/Grapper.php b/app/Console/Commands/Grapper/Sofascore/Grapper.php index 3de95cb6..58d7f65a 100644 --- a/app/Console/Commands/Grapper/Sofascore/Grapper.php +++ b/app/Console/Commands/Grapper/Sofascore/Grapper.php @@ -47,9 +47,8 @@ class Grapper break; } - /** - * Restart + * Restart */ if ($restart) { @@ -83,21 +82,21 @@ class Grapper Notice::note($tournament->sofa_score_tournament_id, 'bb'); } - if (! $force) + if (! $force) { Sleeper::rand(1); - else + } else { Sleeper::rand(); + } if ($debug) { Notice::note($day, 'br'); } - $url = Grap::$options['base_url'] . "api/v1/unique-tournament/$tournament->sofa_score_tournament_id/scheduled-events/$day"; + $url = Grap::$options['base_url']."api/v1/unique-tournament/$tournament->sofa_score_tournament_id/scheduled-events/$day"; $data = CurlSofa::load($url); Notice::note($data, 'bb'); - if (isset($data['events'])) { if ($debug) { @@ -110,7 +109,7 @@ class Grapper if (! $force && SofaScoreEventGrapped::where('grapper_id', $task_id)->where('id', (int) $event['id'])->exists()) { if ($debug) { - Notice::note('DB - Allready grapped at ' . $day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'g'); + Notice::note('DB - Allready grapped at '.$day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'g'); } } else { @@ -124,17 +123,17 @@ class Grapper // if ($now->greaterThan($starttime) && $starttime->isSameDay($carbon_day)) { - if (! $force) + if (! $force) { Sleeper::rand(1); - else + } else { Sleeper::rand(); - - - if ($debug) { - Notice::note($day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'bg'); } - $folder = strtolower(Grap::$storage . '/' . $day . '/' . $day . '-' . $event['tournament']['category']['slug'] . '-' . $event['tournament']['slug'] . '/' . $event['slug']); + if ($debug) { + Notice::note($day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'bg'); + } + + $folder = strtolower(Grap::$storage.'/'.$day.'/'.$day.'-'.$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug']); File::makeDirectory($folder, 0775, true, true); @@ -142,7 +141,7 @@ class Grapper * EVENT event *********************************/ - File::put($folder . '/event.json', json_encode($event)); + File::put($folder.'/event.json', json_encode($event)); /********************************* * EVENT statistics @@ -150,7 +149,7 @@ class Grapper $event_id = $event['id']; - $url = Grap::$options['base_url'] . "api/v1/event/$event_id/statistics"; + $url = Grap::$options['base_url']."api/v1/event/$event_id/statistics"; Sleeper::rand(); @@ -158,13 +157,13 @@ class Grapper $data['id'] = $event_id; - File::put($folder . '/statistics.json', json_encode($data)); + File::put($folder.'/statistics.json', json_encode($data)); /********************************* * EVENT lineups *********************************/ - $url = Grap::$options['base_url'] . "api/v1/event/$event_id/lineups"; + $url = Grap::$options['base_url']."api/v1/event/$event_id/lineups"; Sleeper::rand(); @@ -172,13 +171,13 @@ class Grapper $data['id'] = $event_id; - File::put($folder . '/lineups.json', json_encode($data)); + File::put($folder.'/lineups.json', json_encode($data)); /********************************* * EVENT odds *********************************/ - $url = Grap::$options['base_url'] . "api/v1/event/$event_id/odds/1/all"; + $url = Grap::$options['base_url']."api/v1/event/$event_id/odds/1/all"; Sleeper::rand(); @@ -186,13 +185,13 @@ class Grapper $data['id'] = $event_id; - File::put($folder . '/odds.json', json_encode($data)); + File::put($folder.'/odds.json', json_encode($data)); /********************************* * EVENT incidents *********************************/ - $url = Grap::$options['base_url'] . "api/v1/event/$event_id/incidents"; + $url = Grap::$options['base_url']."api/v1/event/$event_id/incidents"; Sleeper::rand(); @@ -200,13 +199,13 @@ class Grapper $data['id'] = $event_id; - File::put($folder . '/incidents.json', json_encode($data)); + File::put($folder.'/incidents.json', json_encode($data)); $grapped = SofaScoreEventGrapped::firstOrNew(['id' => $event['id']]); $grapped->grapper_id = $task_id; $grapped->day = $day; $grapped->liga = $event['tournament']['name']; - $grapped->name = $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name']; + $grapped->name = $event['homeTeam']['name'].' vs '.$event['awayTeam']['name']; $grapped->url = $url; $grapped->starttime = $starttime->toDateTimeString(); $grapped->save(); @@ -230,10 +229,11 @@ class Grapper Notice::note("$count days done, sleep 10-20 sec", 'p'); } - if (! $force) + if (! $force) { Sleeper::rand(2); - else + } else { Sleeper::rand(); + } } } } diff --git a/app/Console/Commands/Grapper/Sofascore/Helper/CurlSofa.php b/app/Console/Commands/Grapper/Sofascore/Helper/CurlSofa.php index accd2fe2..d9fbf223 100644 --- a/app/Console/Commands/Grapper/Sofascore/Helper/CurlSofa.php +++ b/app/Console/Commands/Grapper/Sofascore/Helper/CurlSofa.php @@ -5,7 +5,6 @@ namespace App\Console\Commands\Grapper\Sofascore\Helper; use App\Console\Commands\Grap; use App\Utilities\Notice; - class CurlSofa { public static function load(string $url) @@ -49,7 +48,7 @@ class CurlSofa // Cache-Control: no-cache // TE: trailers $headers = [ - 'User-Agent: ' . $useragents[Grap::$useragent], + 'User-Agent: '.$useragents[Grap::$useragent], 'Accept: */*', 'Accept-Language: de,en-US;q=0.7,en;q=0.3', 'Accept-Encoding: gzip, deflate, br, zstd', @@ -64,7 +63,7 @@ class CurlSofa 'Priority: u=4', 'Pragma: no-cache', 'Cache-Control: no-cache', - 'TE: trailers' + 'TE: trailers', ]; $ch = curl_init(); diff --git a/app/Console/Sofascore b/app/Console/Sofascore deleted file mode 120000 index 95f0b4bb..00000000 --- a/app/Console/Sofascore +++ /dev/null @@ -1 +0,0 @@ -Commands/Grapper/Sofascore \ No newline at end of file diff --git a/app/Models/SofaScoreEventGrapped.php b/app/Models/SofaScoreEventGrapped.php index cdbfaa5e..adece5ec 100644 --- a/app/Models/SofaScoreEventGrapped.php +++ b/app/Models/SofaScoreEventGrapped.php @@ -8,6 +8,7 @@ class SofaScoreEventGrapped extends Model { protected $fillable = [ 'id', + 'grapper_id', 'name', 'url', 'starttime', diff --git a/app/Services/CloudflareSolverService.php b/app/Services/CloudflareSolverService.php new file mode 100644 index 00000000..358d3f1e --- /dev/null +++ b/app/Services/CloudflareSolverService.php @@ -0,0 +1,63 @@ + 'application/json', + ])->timeout(120)->post($this->solverUrl, [ + 'cmd' => 'request.get', + 'url' => $targetUrl, + 'maxTimeout' => 90000, + ]); + + if ($response->failed()) { + Log::error('FlareSolverr Verbindungsfehler. Status: '.$response->status()); + + return null; + } + + $data = $response->json(); + + if (isset($data['status']) && $data['status'] === 'ok') { + $htmlContent = $data['solution']['response']; + + // JSON aus dem HTML isolieren + if (preg_match('/\{.*\}/s', $htmlContent, $matches)) { + $rawJson = $matches[0]; + $decoded = json_decode($rawJson, true); + + if (json_last_error() === JSON_ERROR_NONE) { + // HIER: Erfolgreiche Rückgabe des Arrays! + return $decoded; + } + + Log::error('Extrahiertes Textfragment ist kein gültiges JSON.'); + } else { + Log::error('Kein JSON-Muster { ... } im HTML gefunden.'); + } + } else { + Log::error('FlareSolverr konnte die Seite nicht umgehen: '.json_encode($data)); + } + + } catch (\Exception $e) { + Log::error('PHP Exception im CloudflareSolverService: '.$e->getMessage()); + } + + // Falls irgendwo ein Fehler auftrat, geben wir null zurück + return null; + } +} diff --git a/app/Services/docker-compose.yml b/app/Services/docker-compose.yml new file mode 100644 index 00000000..8c3b6fcb --- /dev/null +++ b/app/Services/docker-compose.yml @@ -0,0 +1,11 @@ +services: + flaresolverr-server: + image: flaresolverr/flaresolverr:latest # Aktualisiert auf die neueste Version + container_name: flaresolverr_standalone + ports: + - "8191:8191" + environment: + - LOG_LEVEL=info + - TZ=Europe/Vienna + - HOST=0.0.0.0 + restart: always diff --git a/app/Traits/CurlSofa.php b/app/Traits/CurlSofa.php new file mode 100644 index 00000000..2639b439 --- /dev/null +++ b/app/Traits/CurlSofa.php @@ -0,0 +1,87 @@ +useragent(), + 'Accept: */*', + 'Accept-Language: de,en-US;q=0.7,en;q=0.8', + 'Accept-Encoding: gzip, deflate, br, zstd', + 'Referer: https://www.sofascore.com/', + 'DNT: 1', + 'Sec-GPC: 1', + 'Connection: keep-alive', + 'Sec-Fetch-Dest: empty', + 'Sec-Fetch-Mode: cors', + 'Sec-Fetch-Site: same-origin', + 'Priority: u=4', + 'Pragma: no-cache', + 'Cache-Control: no-cache', + 'TE: trailers', + ]; + + // 10.208.40.145 + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + // This line magically handles all decompression out-of-the-box: + curl_setopt($ch, CURLOPT_ENCODING, ''); + $data = curl_exec($ch); + + $data = json_decode($data, true); + + return $data; + + } + + protected function useragent() + { + $array = []; + $array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'; + $array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'; + $array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15'; + $array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0'; + $array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0'; + $array[] = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36'; + $array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15'; + $array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15'; + $array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; + $array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.1 Safari/605.1.15'; + $array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0'; + $array[] = 'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36'; + + return $array[array_rand($array)]; + } + + protected function sleep($speed = false) + { + switch ($speed) { + case 1: + // 5 -10 sec + $sleep = rand(5000000, 10000000); + break; + case 2: + // 10 - 20 sec + $sleep = rand(10000000, 20000000); + break; + case 3: + // 30 - 50 sec + $sleep = rand(30000000, 50000000); + break; + + default: + // 1 -1.5 sec + $sleep = rand(100000, 150000); + break; + } + + usleep($sleep); + } +} diff --git a/app/Traits/GrapSofascore.php b/app/Traits/GrapSofascore.php new file mode 100644 index 00000000..b7d35236 --- /dev/null +++ b/app/Traits/GrapSofascore.php @@ -0,0 +1,146 @@ +debug = $this->options()['d'] ?? 0; + + $this->restart = $this->options()['r'] ?? 0; + + $this->force = $this->options()['f'] ?? 0; + + $grapperSelected = $this->select_grapper(); + + $this->set_base_dir($grapperSelected); + + switch ($grapperSelected) { + case 'daily': + $this->task_id = 1; + $this->run_daily($grapperSelected); + break; + + case 'range': + $this->task_id = 3; + $this->run_range($grapperSelected); + break; + + case 'settings': + $this->settings_to_local(); + break; + case 'excel': + $folder = $this->select_folder(); + + $this->import_to_excel($folder); + break; + } + + } + + /** + * Summary of run_daily + */ + protected function run_daily(string $grapperSelected) + { + $this->load_tournaments_requested($grapperSelected); + + $this->load_days($grapperSelected); + + foreach ($this->days as $day) { + + $this->grap_tournaments_for_day($day); + + foreach ($this->tournaments_requested as $tournament) { + $this->get_event($day, $tournament); + } + } + + if ($this->debug) { + Notice::note('Day grapping compleated', 'g'); + } + } + + /** + * Summary of run_range + * + * @return never + */ + protected function run_range(string $grapperSelected) + { + $this->load_tournaments_requested($grapperSelected); + + $this->load_days($grapperSelected); + + dd($this->days); + } + + /** + * Summary of select_grapper + */ + protected function select_grapper(): string + { + return select( + label: 'What task should run?', + scroll: 10, + options: [ + 'daily' => 'Daily', + 'range' => 'Range', + 'rose' => 'Rose', + 'player' => 'Player', + 'settings' => 'Settings', + 'excel' => 'Excel', + ] + ); + } + + /** + * Summary of select_folder + */ + protected function select_folder(): string + { + return select( + label: 'What folder should be imported?', + scroll: 10, + options: [ + 'daily' => 'Daily', + 'range' => 'Range', + 'rose' => 'Rose', + 'player' => 'Player', + ] + ); + } +} diff --git a/app/Traits/GrapperEvent.php b/app/Traits/GrapperEvent.php new file mode 100644 index 00000000..fb0f48a0 --- /dev/null +++ b/app/Traits/GrapperEvent.php @@ -0,0 +1,159 @@ +where('tournament_id', $tournament['sofa_score_tournament_id'])->first(); + + if ($res) { + + if ($this->debug) { + Notice::note("$day - ".$tournament['tournament']['name'].' - '.$tournament['sofa_score_tournament_id'], 'b'); + } + + $this->sleep(1); + + $url = $this->base_url.'api/v1/unique-tournament/'.$tournament['sofa_score_tournament_id']."/scheduled-events/$day"; + + if ($this->debug) { + Notice::note($url, 'bb'); + } + + $cloudflareService = new CloudflareSolverService; + + $data = $cloudflareService->getJsonFromUrl($url); + + if (isset($data['events'])) { + + foreach ($data['events'] as $event) { + + if (! $this->force && SofaScoreEventGrapped::where('grapper_id', $this->task_id)->where('id', (int) $event['id'])->exists()) { + if ($this->debug) { + Notice::note('DB - Allready grapped at '.$day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'b'); + } + } else { + + if ($this->debug) { + Notice::note("$day - ".$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'g'); + } + + $folder = strtolower("$this->data_dir/$day/$day-".$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug']); + + File::makeDirectory($folder, 0775, true, true); + + /********************************* + * EVENT event + *********************************/ + + File::put("$folder/event.json", json_encode($event)); + + /********************************* + * EVENT statistics + *********************************/ + + $event_id = $event['id']; + + $url = $this->base_url."api/v1/event/$event_id/statistics"; + + if ($this->debug) { + Notice::note($url, 'b'); + } + + $this->sleep(); + + $data = $cloudflareService->getJsonFromUrl($url); + + $data['id'] = $event_id; + + File::put("$folder/statistics.json", json_encode($data)); + + /********************************* + * EVENT lineups + *********************************/ + + $url = $this->base_url."api/v1/event/$event_id/lineups"; + + if ($this->debug) { + Notice::note($url, 'b'); + } + + $this->sleep(); + + $data = $cloudflareService->getJsonFromUrl($url); + + $data['id'] = $event_id; + + File::put("$folder/lineups.json", json_encode($data)); + + /********************************* + * EVENT odds + *********************************/ + + $url = $this->base_url."api/v1/event/$event_id/odds/1/all"; + + if ($this->debug) { + Notice::note($url, 'b'); + } + + $this->sleep(); + + $data = $cloudflareService->getJsonFromUrl($url); + + $data['id'] = $event_id; + + File::put("$folder/odds.json", json_encode($data)); + + /********************************* + * EVENT incidents + *********************************/ + + $url = $this->base_url."api/v1/event/$event_id/incidents"; + + if ($this->debug) { + Notice::note($url, 'b'); + } + + $this->sleep(); + + $data = $cloudflareService->getJsonFromUrl($url); + + $data['id'] = $event_id; + + File::put("$folder/incidents.json", json_encode($data)); + + $starttime = Carbon::createFromTimestamp($event['startTimestamp']); + + SofaScoreEventGrapped::updateOrCreate([ + 'id' => (int) $event['id'], + ], [ + 'grapper_id' => $this->task_id, + 'day' => $day, + 'liga' => $event['tournament']['name'], + 'name' => $event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], + 'url' => $url, + 'starttime' => $starttime->toDateTimeString(), + ]); + } + } + } + } + + SofaScoreTaskDay::updateOrCreate([ + 'grapper_id' => (int) $this->task_id, + 'day' => $day, + ], [ + 'done' => 1, + ]); + } +} diff --git a/app/Traits/GrappingHelper.php b/app/Traits/GrappingHelper.php new file mode 100644 index 00000000..a4d2b5b7 --- /dev/null +++ b/app/Traits/GrappingHelper.php @@ -0,0 +1,386 @@ +days = []; + + $task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first(); + + switch ($grapperSelected) { + case 'daily': + + foreach ($this->daily_days as $dayBack) { + $this->days[] = Carbon::now()->subDays($dayBack)->format('Y-m-d'); + } + + break; + case 'range': + $task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first(); + + $from = $this->validate_date($task->day_from); + $from = Carbon::createFromDate($from); + + $to = $this->validate_date($task->day_to); + $to = Carbon::createFromDate($to); + + $lengthdates = $from->diffInDays($to); + if ($lengthdates < 1) { + Notice::note('start must be before end', 'r'); + } + + if ($this->restart) { + while ($from <= $to) { + $this->days[] = $from->format('Y-m-d'); + $from->addDay(); + } + } else { + while ($from <= $to) { + $res = SofaScoreTaskDay::firstOrNew(['grapper_id' => (int) 3, 'day' => $from->format('Y-m-d')]); + if (! $res->done) { + $this->days[] = $from->format('Y-m-d'); + } + $from->addDay(); + } + } + shuffle($this->days); + + break; + } + + if ($this->restart) { + SofaScoreEventGrapped::where('grapper_id', $this->task_id)->delete(); + + foreach ($this->days as $day) { + // if restarting find position + SofaScoreTaskDay::where('grapper_id', 1) + ->where('day', $day) + ->update(['done' => 0]); + } + if ($this->debug) { + Notice::note('Grapper restarted', 'r', 1); + } + } + + $this->days = array_unique($this->days, SORT_REGULAR); + } + + /** + * @return void + */ + protected function set_base_dir(string $grapperSelected) + { + switch ($grapperSelected) { + case 'daily': + $this->data_dir = storage_path($this->base_dir).'/daily'; + break; + case 'range': + $this->data_dir = storage_path($this->base_dir).'/range'; + break; + case 'rose': + $this->data_dir = storage_path($this->base_dir).'/rose'; + break; + case 'player': + $this->data_dir = storage_path($this->base_dir).'/player'; + break; + case 'excel': + $this->data_dir = storage_path($this->base_dir); + break; + } + + File::makeDirectory($this->data_dir, 0775, true, true); + } + + /** + * Summary of load_tournaments_requested + * + * @return void + */ + protected function load_tournaments_requested(string $grapperSelected) + { + switch ($grapperSelected) { + case 'daily': + case 'range': + $this->tournaments_requested = SofaScoreTaskTournament::with('tournament')->where('sofa_score_task_id', 1)->where('disabled', false)->get()->toArray(); + break; + } + } + + /** + * Summary of selectGrapper + */ + public function settings_to_local(): void + { + if ($this->debug) { + Notice::note('⚽ settings to local'); + Notice::note('⚽ close all vpn'); + } + exec('killall ssh'); + + if ($this->debug) { + Notice::note('⚽ open vpn'); + } + exec('ssh -f -N -L 13306:127.0.0.1:3306 -p 58371 root@162.55.242.190'); + /** + * SofaScoreRosa + */ + if ($this->debug) { + Notice::note('SofaScoreRosa', 'g'); + } + SofaScoreRosa::truncate(); + $rows = DB::connection('remote_mariadb')->table('sofa_score_rosas')->get(); + foreach ($rows as $row) { + SofaScoreRosa::insert((array) $row); + } + /** + * SofaScoreTask + */ + if ($this->debug) { + Notice::note('SofaScoreTask', 'g'); + } + SofaScoreTask::truncate(); + $rows = DB::connection('remote_mariadb')->table('sofa_score_tasks')->get(); + foreach ($rows as $row) { + SofaScoreTask::insert((array) $row); + } + /** + * SofaScoreTaskTournament + */ + if ($this->debug) { + Notice::note('SofaScoreTaskTournament', 'g'); + } + SofaScoreTaskTournament::truncate(); + $rows = DB::connection('remote_mariadb')->table('sofa_score_task_tournaments')->get(); + foreach ($rows as $row) { + SofaScoreTaskTournament::insert((array) $row); + } + /** + * SofaScoreTournament + */ + if ($this->debug) { + Notice::note('SofaScoreTournament', 'g'); + } + SofaScoreTournament::truncate(); + $rows = DB::connection('remote_mariadb')->table('sofa_score_tournaments')->get(); + foreach ($rows as $row) { + SofaScoreTournament::insert((array) $row); + } + + if ($this->debug) { + Notice::note('⚽ close all vpn'); + } + exec('killall ssh'); + } + + /** + * Summary of validate_date + * + * @return string + */ + protected function validate_date(string $date) + { + if (Carbon::createFromDate($date) !== false) { + return $date; + } else { + Notice::note('Wrong date format', 'r', 1); + } + } + + /** + * a Grap --d --f 67 --fun=4 --task=1 + * + * Ita Serie-A-Saison 23-08-2025 - 24-05-2026 + * Ger Bundesliga 22-08-2025 – 16-05-2026 + * Eng Premier league 15-08-2025 - 24-05-2026 + * Spa La Liga 15-08-2025 - 24-05-2026 + * Fra Liggue 15-08-2025 - 17-05-2026 + */ + protected function grap_tournaments_for_day(string $day) + { + $tournaments_counter = 0; + $sport_id = 1; + + $has = SofaScoreDayTournament::where('day', $day)->first(); + + $slug = 'football'; + + if (! $this->force && $has) { + if ($this->debug) { + Notice::note("tournaments $day for ".strtoupper($slug).' allready grapped', 'bb'); + } + } else { + + if ($this->debug) { + Notice::note("grappig tournaments for $day", 'bb'); + } + + $page = 1; + + while ($page) { + // $data = $this->curl($url); + + $this->sleep(1); + + $url = $this->base_url."api/v1/sport/$slug/scheduled-tournaments/$day/page/$page"; + + $cloudflareService = new CloudflareSolverService; + + $data = $cloudflareService->getJsonFromUrl($url); + + if ($data['hasNextPage']) { + $page++; + } else { + $page = false; + } + + foreach ($data['scheduled'] as $row) { + + if (isset($row['tournament']['uniqueTournament'])) { + $tournament = $row['tournament']['uniqueTournament']; + } else { + $tournament = $row['tournament']; + } + + $category = $tournament['category']; + + if ($sport_id == 5) { + $country_id = 470; + } else { + + /** + * CATEGORY + */ + $find = [ + 'Amateur', + 'Bosnia & Herzegovina', + 'Czechia', + 'DR Congo', + 'Netherlands', + 'North & Central America', + 'Bonaire', + 'Sao Tome and Principe', + 'Cape Verde', + 'Macao', + 'Saint Kitts and Nevis', + 'Eswatini', + 'Timor Leste', + 'Laos', + 'Congo Republic', + + ]; + $replace = [ + '', + 'Bosnia Herzegovina', + 'Czech Republic', + 'Congo DR', + 'Holland', + 'North America', + 'Bonaire, Sint Eustatius and Saba', + 'Sao Tome e Principe', + 'Cape Verde Islands', + 'Macau', + 'St. Kitts and Nevis', + 'Swaziland', + 'East Timor', + 'Lao Peoples\'s Democratic Republic', + 'Congo DR', + ]; + $country_id = Country::where('name', trim(str_replace($find, $replace, $category['name'])))->value('id'); + } + + if (! $country_id) { + // platzhalter fix later sofa_score_categories => txodds countries + // CATEGORY find <-> replace + // Log::channel('sofascore')->info($tournament); + + $sofa_score_category_id = 10000000; + } else { + $call = SofaScoreCategory::updateOrCreate([ + 'id' => (int) $category['id'], + 'country_id' => (int) $country_id, + 'slug' => $category['slug'], + ], [ + 'name' => $category['name'], + ]); + + $sofa_score_category_id = $call->id; + } + + /** + * TOURNAMENT + */ + SofaScoreTournament::updateOrCreate([ + 'sport_id' => (int) $sport_id, + 'tournament_id' => (int) $tournament['id'], + 'sofa_score_category_id' => (int) $sofa_score_category_id, + ], [ + 'name' => $tournament['name'], + 'slug' => $tournament['slug'], + ]); + $tournaments_counter++; + + // if ($debug) { + // Notice::note('tournament: ' . $tournament['name'], 'p'); + // } + + /** + * DAY TOURNAMENT + */ + SofaScoreDayTournament::updateOrCreate([ + 'tournament_id' => (int) $tournament['id'], + 'sport_id' => (int) $sport_id, + 'day' => $day, + ]); + } + + if ($this->debug) { + Notice::note("tournaments: $tournaments_counter", 'p'); + } + } + } + if ($this->debug) { + Notice::note('days tournament done', 'g'); + } + } + + protected function get_tree(string $dir, &$results = []) + { + $files = scandir($dir); + foreach ($files as $value) { + $path = realpath($dir.DIRECTORY_SEPARATOR.$value); + if (! is_dir($path)) { + $results[] = $path; + } elseif ($value != '.' && $value != '..') { + self::get_tree($path, $results); + $results[] = $path; + } + } + + return $results; + } +} diff --git a/app/Traits/GrappingToExcel.php b/app/Traits/GrappingToExcel.php new file mode 100644 index 00000000..331384d2 --- /dev/null +++ b/app/Traits/GrappingToExcel.php @@ -0,0 +1,85 @@ +debug) { + Notice::note("import $folder to excel", 'g'); + } + + $titel = ucfirst($folder).'_'.Carbon::now()->format('Y-m-d'); + $files = $this->get_tree("$this->data_dir/$folder"); + + $files = array_chunk($this->filter($files), self::$max); + + foreach ($files as $key => $value) { + + $has = count($value); + if (! $key && $has < self::$max) { + $from = 1; + $to = $has; + } elseif (! $key) { + $from = 1; + $key++; + $to = $key * self::$max; + } elseif ($has == self::$max) { + $from = $key * self::$max + 1; + $key++; + $to = $key * self::$max; + } else { + $from = $key * self::$max + 1; + $to = $key * self::$max + $has; + } + + $spreadsheet = new Spreadsheet; + $spreadsheet->getProperties()->setTitle("$titel-$from-$to"); + $activeWorksheet = $spreadsheet->getActiveSheet(); + + $column = 1; + foreach ($value as $file) { + $data = file_get_contents($file); + $data = json_decode($data, true); + $json_string = json_encode($data, JSON_PRETTY_PRINT); + + $arr = explode(PHP_EOL, $json_string); + $row = 0; + foreach ($arr as $string) { + $row++; + $replacedString = preg_replace('/\\\\u([0-9abcdef]{4})/', '&#x$1;', $string); + $utf8string = mb_convert_encoding($replacedString, 'UTF-8', 'HTML-ENTITIES'); + $activeWorksheet->setCellValue([$column, $row], $utf8string); + } + $column++; + } + + $spreadsheet->setActiveSheetIndex(0); + $writer = IOFactory::createWriter($spreadsheet, 'Xlsx'); + $writer->save("$titel-$from-$to.xlsx"); + } + } + + protected function filter(array $files, array $types = ['json']) + { + $filtered = []; + foreach ($files as $file) { + $path_parts = pathinfo($file); + if (isset($path_parts['extension']) && in_array($path_parts['extension'], $types)) { + $filtered[] = $file; + } + } + + return $filtered; + } +} diff --git a/config/services.php b/config/services.php index 6182e4b9..11a68f46 100644 --- a/config/services.php +++ b/config/services.php @@ -35,4 +35,8 @@ return [ ], ], + 'flaresolverr' => [ + 'url' => env('FLARESOLVERR_URL', '192.168.178'), + ], + ]; diff --git a/database/migrations/2026_07_11_145940_create_sofa_score_event_grappeds_table.php b/database/migrations/2026_07_11_145940_create_sofa_score_event_grappeds_table.php index aba855ac..32f92009 100644 --- a/database/migrations/2026_07_11_145940_create_sofa_score_event_grappeds_table.php +++ b/database/migrations/2026_07_11_145940_create_sofa_score_event_grappeds_table.php @@ -13,7 +13,7 @@ return new class extends Migration { Schema::create('sofa_score_event_grappeds', function (Blueprint $table) { $table->id(); - $table->integer('grapper_id'); + $table->integer('grapper_id')->default(0)->nullable(); $table->date('day')->nullable(); $table->string('liga')->nullable(); $table->string('name')->nullable();