salam/nuhodds_fi4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
salam committed 2026-09-26 17:49:06 +02:00
1 parent 267b5b9583
commit 911b515021
15 files changed
+1048 -56
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Console/Commands/Grap.php
+57 -25
View File
@@ -5,7 +5,6 @@ namespace App\Console\Commands;
use App\Console\Commands\Grapper\Sofascore\Grapper; use App\Console\Commands\Grapper\Sofascore\Grapper;
use App\Console\Commands\Grapper\Sofascore\GrapperTest; use App\Console\Commands\Grapper\Sofascore\GrapperTest;
use App\Console\Commands\Grapper\Sofascore\Helper\ClearDir; use App\Console\Commands\Grapper\Sofascore\Helper\ClearDir;
use App\Console\Commands\Grapper\Sofascore\Helper\RemoveDir;
use App\Console\Commands\Grapper\Sofascore\Helper\SetStorage; use App\Console\Commands\Grapper\Sofascore\Helper\SetStorage;
use App\Console\Commands\Grapper\Sofascore\MakeExcel; use App\Console\Commands\Grapper\Sofascore\MakeExcel;
use App\Console\Commands\Grapper\Sofascore\Rose; use App\Console\Commands\Grapper\Sofascore\Rose;
@@ -29,7 +28,7 @@ class Grap extends Command
public static string $useragent = '4'; public static string $useragent = '4';
protected $signature = 'grp {id=0} {day=0} {--date=} {--folder=} {--d} {--g} {--r} {--c} {--m} {--f} {--F} {--R} {--S}'; protected $signature = 'grp {id=0} {mode} {day=0} {--date=} {--folder=} {--d} {--g} {--r} {--c} {--m} {--f} {--F} {--R} {--S}';
public function handle() public function handle()
{ {
@@ -39,38 +38,71 @@ class Grap extends Command
self::$options['base_url'] = 'https://www.sofascore.com/'; self::$options['base_url'] = 'https://www.sofascore.com/';
if (! self::$options['id']) { if (! self::$options['id']) {
$txt = 'PARAMETERS' . PHP_EOL; $txt = 'PARAMETERS'.PHP_EOL;
$txt .= '--d debug' . PHP_EOL; $txt .= '--d debug'.PHP_EOL;
$txt .= '--r restart range' . PHP_EOL; $txt .= '--r restart range'.PHP_EOL;
$txt .= '--f force Events' . PHP_EOL; $txt .= '--f force Events'.PHP_EOL;
$txt .= '--F force DayTournament links' . PHP_EOL; $txt .= '--F force DayTournament links'.PHP_EOL;
$txt .= '--c clear grap dir' . PHP_EOL; $txt .= '--c clear grap dir'.PHP_EOL;
$txt .= '--m mail only' . PHP_EOL; $txt .= '--m mail only'.PHP_EOL;
$txt .= '--C clear folder' . PHP_EOL; $txt .= '--C clear folder'.PHP_EOL;
$txt .= '--G grap' . PHP_EOL; $txt .= '--G grap'.PHP_EOL;
$txt .= '--E make excel' . PHP_EOL; $txt .= '--E make excel'.PHP_EOL;
$txt .= '--M mail' . PHP_EOL; $txt .= '--M mail'.PHP_EOL;
$txt .= '--folder' . PHP_EOL; $txt .= '--folder'.PHP_EOL;
$txt .= '-----------------------------------------------------------' . PHP_EOL; $txt .= '-----------------------------------------------------------'.PHP_EOL;
$txt .= 'FUNCTION' . PHP_EOL; $txt .= 'FUNCTION'.PHP_EOL;
$txt .= ' 1 grapper DAILY & subDays ( default 3)' . PHP_EOL; $txt .= ' 1 grapper DAILY & subDays ( default 3)'.PHP_EOL;
$txt .= ' 2 grapper MORE DAYS' . PHP_EOL; $txt .= ' 2 grapper MORE DAYS'.PHP_EOL;
$txt .= ' 3 grapper RANGE' . PHP_EOL; $txt .= ' 3 grapper RANGE'.PHP_EOL;
$txt .= ' 4 grapper ROSE' . PHP_EOL; $txt .= ' 4 grapper ROSE'.PHP_EOL;
$txt .= ' 5 import to excel from folder' . PHP_EOL; $txt .= ' 5 import to excel from folder'.PHP_EOL;
$txt .= ' 7 settings to local' . PHP_EOL; $txt .= ' 7 settings to local'.PHP_EOL;
$txt .= ' 11 move SQL' . PHP_EOL; $txt .= ' 11 move SQL'.PHP_EOL;
$txt .= ' 12 testing url for grapping' . PHP_EOL; $txt .= ' 12 testing url for grapping'.PHP_EOL;
Notice::note($txt, 'b', 1); Notice::note($txt, 'b', 1);
} }
Notice::note('Task Id ' . self::$options['id'], 'gb'); Notice::note('Task Id '.self::$options['id'], 'gb');
if (self::$options['cear']) {
SetStorage::set(1);
ClearDir::clear();
}
if (self::$options['grap']) {
SetStorage::set(1);
Grapper::load(1);
}
if (self::$options['excel']) {
SetStorage::set(1);
MakeExcel::create();
}
if (self::$options['mail']) {
SetStorage::set(1);
SendMail::run(1);
}
case 4:
// ROSE
SetStorage::set(4);
Rose::run();
MakeExcel::create();
break;
switch (self::$options['id']) { switch (self::$options['id']) {
case 1: case 1:
// DAILY // DAILY
if (self::$options['c']) {
SetStorage::set(1);
ClearDir::clear();
}
if (self::$options['C']) { if (self::$options['C']) {
Grapper::load(1); Grapper::load(1);
} }
app/Console/Commands/Grapper.php
+20
View File
@@ -0,0 +1,20 @@
<?php
namespace App\Console\Commands;
use App\Traits\GrapSofascore;
use Illuminate\Console\Command;
class Grapper extends Command
{
use GrapSofascore;
protected $signature = 'grp:sofa {--d} {--r} {--f}';
protected $description = 'grap sofascore';
public function handleUpdate(string $grapperSelected): void
{
$this->runGrapper($grapperSelected);
}
}
app/Console/Commands/Grapper/Sofascore/Grapper.php
+26 -26
View File
@@ -47,9 +47,8 @@ class Grapper
break; break;
} }
/** /**
* Restart * Restart
*/ */
if ($restart) { if ($restart) {
@@ -83,21 +82,21 @@ class Grapper
Notice::note($tournament->sofa_score_tournament_id, 'bb'); Notice::note($tournament->sofa_score_tournament_id, 'bb');
} }
if (! $force) if (! $force) {
Sleeper::rand(1); Sleeper::rand(1);
else } else {
Sleeper::rand(); Sleeper::rand();
}
if ($debug) { if ($debug) {
Notice::note($day, 'br'); Notice::note($day, 'br');
} }
$url = Grap::$options['base_url'] . "api/v1/unique-tournament/$tournament->sofa_score_tournament_id/scheduled-events/$day"; $url = Grap::$options['base_url']."api/v1/unique-tournament/$tournament->sofa_score_tournament_id/scheduled-events/$day";
$data = CurlSofa::load($url); $data = CurlSofa::load($url);
Notice::note($data, 'bb'); Notice::note($data, 'bb');
if (isset($data['events'])) { if (isset($data['events'])) {
if ($debug) { if ($debug) {
@@ -110,7 +109,7 @@ class Grapper
if (! $force && SofaScoreEventGrapped::where('grapper_id', $task_id)->where('id', (int) $event['id'])->exists()) { if (! $force && SofaScoreEventGrapped::where('grapper_id', $task_id)->where('id', (int) $event['id'])->exists()) {
if ($debug) { if ($debug) {
Notice::note('DB - Allready grapped at ' . $day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'g'); Notice::note('DB - Allready grapped at '.$day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'g');
} }
} else { } else {
@@ -124,17 +123,17 @@ class Grapper
// if ($now->greaterThan($starttime) && $starttime->isSameDay($carbon_day)) { // if ($now->greaterThan($starttime) && $starttime->isSameDay($carbon_day)) {
if (! $force) if (! $force) {
Sleeper::rand(1); Sleeper::rand(1);
else } else {
Sleeper::rand(); Sleeper::rand();
if ($debug) {
Notice::note($day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'bg');
} }
$folder = strtolower(Grap::$storage . '/' . $day . '/' . $day . '-' . $event['tournament']['category']['slug'] . '-' . $event['tournament']['slug'] . '/' . $event['slug']); if ($debug) {
Notice::note($day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'bg');
}
$folder = strtolower(Grap::$storage.'/'.$day.'/'.$day.'-'.$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug']);
File::makeDirectory($folder, 0775, true, true); File::makeDirectory($folder, 0775, true, true);
@@ -142,7 +141,7 @@ class Grapper
* EVENT event * EVENT event
*********************************/ *********************************/
File::put($folder . '/event.json', json_encode($event)); File::put($folder.'/event.json', json_encode($event));
/********************************* /*********************************
* EVENT statistics * EVENT statistics
@@ -150,7 +149,7 @@ class Grapper
$event_id = $event['id']; $event_id = $event['id'];
$url = Grap::$options['base_url'] . "api/v1/event/$event_id/statistics"; $url = Grap::$options['base_url']."api/v1/event/$event_id/statistics";
Sleeper::rand(); Sleeper::rand();
@@ -158,13 +157,13 @@ class Grapper
$data['id'] = $event_id; $data['id'] = $event_id;
File::put($folder . '/statistics.json', json_encode($data)); File::put($folder.'/statistics.json', json_encode($data));
/********************************* /*********************************
* EVENT lineups * EVENT lineups
*********************************/ *********************************/
$url = Grap::$options['base_url'] . "api/v1/event/$event_id/lineups"; $url = Grap::$options['base_url']."api/v1/event/$event_id/lineups";
Sleeper::rand(); Sleeper::rand();
@@ -172,13 +171,13 @@ class Grapper
$data['id'] = $event_id; $data['id'] = $event_id;
File::put($folder . '/lineups.json', json_encode($data)); File::put($folder.'/lineups.json', json_encode($data));
/********************************* /*********************************
* EVENT odds * EVENT odds
*********************************/ *********************************/
$url = Grap::$options['base_url'] . "api/v1/event/$event_id/odds/1/all"; $url = Grap::$options['base_url']."api/v1/event/$event_id/odds/1/all";
Sleeper::rand(); Sleeper::rand();
@@ -186,13 +185,13 @@ class Grapper
$data['id'] = $event_id; $data['id'] = $event_id;
File::put($folder . '/odds.json', json_encode($data)); File::put($folder.'/odds.json', json_encode($data));
/********************************* /*********************************
* EVENT incidents * EVENT incidents
*********************************/ *********************************/
$url = Grap::$options['base_url'] . "api/v1/event/$event_id/incidents"; $url = Grap::$options['base_url']."api/v1/event/$event_id/incidents";
Sleeper::rand(); Sleeper::rand();
@@ -200,13 +199,13 @@ class Grapper
$data['id'] = $event_id; $data['id'] = $event_id;
File::put($folder . '/incidents.json', json_encode($data)); File::put($folder.'/incidents.json', json_encode($data));
$grapped = SofaScoreEventGrapped::firstOrNew(['id' => $event['id']]); $grapped = SofaScoreEventGrapped::firstOrNew(['id' => $event['id']]);
$grapped->grapper_id = $task_id; $grapped->grapper_id = $task_id;
$grapped->day = $day; $grapped->day = $day;
$grapped->liga = $event['tournament']['name']; $grapped->liga = $event['tournament']['name'];
$grapped->name = $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name']; $grapped->name = $event['homeTeam']['name'].' vs '.$event['awayTeam']['name'];
$grapped->url = $url; $grapped->url = $url;
$grapped->starttime = $starttime->toDateTimeString(); $grapped->starttime = $starttime->toDateTimeString();
$grapped->save(); $grapped->save();
@@ -230,10 +229,11 @@ class Grapper
Notice::note("$count days done, sleep 10-20 sec", 'p'); Notice::note("$count days done, sleep 10-20 sec", 'p');
} }
if (! $force) if (! $force) {
Sleeper::rand(2); Sleeper::rand(2);
else } else {
Sleeper::rand(); Sleeper::rand();
}
} }
} }
} }
app/Console/Commands/Grapper/Sofascore/Helper/CurlSofa.php
+2 -3
View File
@@ -5,7 +5,6 @@ namespace App\Console\Commands\Grapper\Sofascore\Helper;
use App\Console\Commands\Grap; use App\Console\Commands\Grap;
use App\Utilities\Notice; use App\Utilities\Notice;
class CurlSofa class CurlSofa
{ {
public static function load(string $url) public static function load(string $url)
@@ -49,7 +48,7 @@ class CurlSofa
// Cache-Control: no-cache // Cache-Control: no-cache
// TE: trailers // TE: trailers
$headers = [ $headers = [
'User-Agent: ' . $useragents[Grap::$useragent], 'User-Agent: '.$useragents[Grap::$useragent],
'Accept: */*', 'Accept: */*',
'Accept-Language: de,en-US;q=0.7,en;q=0.3', 'Accept-Language: de,en-US;q=0.7,en;q=0.3',
'Accept-Encoding: gzip, deflate, br, zstd', 'Accept-Encoding: gzip, deflate, br, zstd',
@@ -64,7 +63,7 @@ class CurlSofa
'Priority: u=4', 'Priority: u=4',
'Pragma: no-cache', 'Pragma: no-cache',
'Cache-Control: no-cache', 'Cache-Control: no-cache',
'TE: trailers' 'TE: trailers',
]; ];
$ch = curl_init(); $ch = curl_init();
app/Console/Sofascore
-1
View File
@@ -1 +0,0 @@
Commands/Grapper/Sofascore
app/Models/SofaScoreEventGrapped.php
+1
View File
@@ -8,6 +8,7 @@ class SofaScoreEventGrapped extends Model
{ {
protected $fillable = [ protected $fillable = [
'id', 'id',
'grapper_id',
'name', 'name',
'url', 'url',
'starttime', 'starttime',
app/Services/CloudflareSolverService.php
+63
View File
@@ -0,0 +1,63 @@
<?php
namespace App\Services;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
class CloudflareSolverService
{
protected string $solverUrl = 'http://192.168.178.145:8191/v1';
/**
* Holt die Daten und gibt das extrahierte JSON als PHP-Array zurück.
* Wenn etwas schiefgeht, wird null zurückgegeben.
*/
public function getJsonFromUrl(string $targetUrl): ?array
{
try {
$response = Http::withHeaders([
'Content-Type' => 'application/json',
])->timeout(120)->post($this->solverUrl, [
'cmd' => 'request.get',
'url' => $targetUrl,
'maxTimeout' => 90000,
]);
if ($response->failed()) {
Log::error('FlareSolverr Verbindungsfehler. Status: '.$response->status());
return null;
}
$data = $response->json();
if (isset($data['status']) && $data['status'] === 'ok') {
$htmlContent = $data['solution']['response'];
// JSON aus dem HTML isolieren
if (preg_match('/\{.*\}/s', $htmlContent, $matches)) {
$rawJson = $matches[0];
$decoded = json_decode($rawJson, true);
if (json_last_error() === JSON_ERROR_NONE) {
// HIER: Erfolgreiche Rückgabe des Arrays!
return $decoded;
}
Log::error('Extrahiertes Textfragment ist kein gültiges JSON.');
} else {
Log::error('Kein JSON-Muster { ... } im HTML gefunden.');
}
} else {
Log::error('FlareSolverr konnte die Seite nicht umgehen: '.json_encode($data));
}
} catch (\Exception $e) {
Log::error('PHP Exception im CloudflareSolverService: '.$e->getMessage());
}
// Falls irgendwo ein Fehler auftrat, geben wir null zurück
return null;
}
}
app/Services/docker-compose.yml
+11
View File
@@ -0,0 +1,11 @@
services:
flaresolverr-server:
image: flaresolverr/flaresolverr:latest # Aktualisiert auf die neueste Version
container_name: flaresolverr_standalone
ports:
- "8191:8191"
environment:
- LOG_LEVEL=info
- TZ=Europe/Vienna
- HOST=0.0.0.0
restart: always
app/Traits/CurlSofa.php
+87
View File
@@ -0,0 +1,87 @@
<?php
namespace App\Traits;
trait CurlSofa
{
protected function curl(string $url)
{
$headers = [
'User-Agent: '.$this->useragent(),
'Accept: */*',
'Accept-Language: de,en-US;q=0.7,en;q=0.8',
'Accept-Encoding: gzip, deflate, br, zstd',
'Referer: https://www.sofascore.com/',
'DNT: 1',
'Sec-GPC: 1',
'Connection: keep-alive',
'Sec-Fetch-Dest: empty',
'Sec-Fetch-Mode: cors',
'Sec-Fetch-Site: same-origin',
'Priority: u=4',
'Pragma: no-cache',
'Cache-Control: no-cache',
'TE: trailers',
];
// 10.208.40.145
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
// This line magically handles all decompression out-of-the-box:
curl_setopt($ch, CURLOPT_ENCODING, '');
$data = curl_exec($ch);
$data = json_decode($data, true);
return $data;
}
protected function useragent()
{
$array = [];
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0';
$array[] = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0';
$array[] = 'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36';
return $array[array_rand($array)];
}
protected function sleep($speed = false)
{
switch ($speed) {
case 1:
// 5 -10 sec
$sleep = rand(5000000, 10000000);
break;
case 2:
// 10 - 20 sec
$sleep = rand(10000000, 20000000);
break;
case 3:
// 30 - 50 sec
$sleep = rand(30000000, 50000000);
break;
default:
// 1 -1.5 sec
$sleep = rand(100000, 150000);
break;
}
usleep($sleep);
}
}
app/Traits/GrapSofascore.php
+146
View File
@@ -0,0 +1,146 @@
<?php
namespace App\Traits;
use App\Utilities\Notice;
use function Laravel\Prompts\select;
trait GrapSofascore
{
use GrapperEvent;
use GrappingHelper;
use GrappingToExcel;
protected int $debug;
protected int $restart;
protected int $force;
protected int $task_id;
protected array $daily_days = [1, 5];
protected string $base_dir = 'sofascore_data';
protected string $data_dir = '';
protected string $base_url = 'https://www.sofascore.com/';
protected array $tournaments_requested = [];
protected array $days = [];
public function handle(): void
{
error_reporting(E_ALL);
ini_set('display_errors', 1);
$this->debug = $this->options()['d'] ?? 0;
$this->restart = $this->options()['r'] ?? 0;
$this->force = $this->options()['f'] ?? 0;
$grapperSelected = $this->select_grapper();
$this->set_base_dir($grapperSelected);
switch ($grapperSelected) {
case 'daily':
$this->task_id = 1;
$this->run_daily($grapperSelected);
break;
case 'range':
$this->task_id = 3;
$this->run_range($grapperSelected);
break;
case 'settings':
$this->settings_to_local();
break;
case 'excel':
$folder = $this->select_folder();
$this->import_to_excel($folder);
break;
}
}
/**
* Summary of run_daily
*/
protected function run_daily(string $grapperSelected)
{
$this->load_tournaments_requested($grapperSelected);
$this->load_days($grapperSelected);
foreach ($this->days as $day) {
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
}
if ($this->debug) {
Notice::note('Day grapping compleated', 'g');
}
}
/**
* Summary of run_range
*
* @return never
*/
protected function run_range(string $grapperSelected)
{
$this->load_tournaments_requested($grapperSelected);
$this->load_days($grapperSelected);
dd($this->days);
}
/**
* Summary of select_grapper
*/
protected function select_grapper(): string
{
return select(
label: 'What task should run?',
scroll: 10,
options: [
'daily' => 'Daily',
'range' => 'Range',
'rose' => 'Rose',
'player' => 'Player',
'settings' => 'Settings',
'excel' => 'Excel',
]
);
}
/**
* Summary of select_folder
*/
protected function select_folder(): string
{
return select(
label: 'What folder should be imported?',
scroll: 10,
options: [
'daily' => 'Daily',
'range' => 'Range',
'rose' => 'Rose',
'player' => 'Player',
]
);
}
}
app/Traits/GrapperEvent.php
+159
View File
@@ -0,0 +1,159 @@
<?php
namespace App\Traits;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreTaskDay;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\File;
trait GrapperEvent
{
protected function get_event(string $day, array $tournament)
{
$res = SofaScoreDayTournament::whereDate('day', '=', $day)->where('tournament_id', $tournament['sofa_score_tournament_id'])->first();
if ($res) {
if ($this->debug) {
Notice::note("$day - ".$tournament['tournament']['name'].' - '.$tournament['sofa_score_tournament_id'], 'b');
}
$this->sleep(1);
$url = $this->base_url.'api/v1/unique-tournament/'.$tournament['sofa_score_tournament_id']."/scheduled-events/$day";
if ($this->debug) {
Notice::note($url, 'bb');
}
$cloudflareService = new CloudflareSolverService;
$data = $cloudflareService->getJsonFromUrl($url);
if (isset($data['events'])) {
foreach ($data['events'] as $event) {
if (! $this->force && SofaScoreEventGrapped::where('grapper_id', $this->task_id)->where('id', (int) $event['id'])->exists()) {
if ($this->debug) {
Notice::note('DB - Allready grapped at '.$day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'b');
}
} else {
if ($this->debug) {
Notice::note("$day - ".$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'g');
}
$folder = strtolower("$this->data_dir/$day/$day-".$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug']);
File::makeDirectory($folder, 0775, true, true);
/*********************************
* EVENT event
*********************************/
File::put("$folder/event.json", json_encode($event));
/*********************************
* EVENT statistics
*********************************/
$event_id = $event['id'];
$url = $this->base_url."api/v1/event/$event_id/statistics";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/statistics.json", json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = $this->base_url."api/v1/event/$event_id/lineups";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/lineups.json", json_encode($data));
/*********************************
* EVENT odds
*********************************/
$url = $this->base_url."api/v1/event/$event_id/odds/1/all";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/odds.json", json_encode($data));
/*********************************
* EVENT incidents
*********************************/
$url = $this->base_url."api/v1/event/$event_id/incidents";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/incidents.json", json_encode($data));
$starttime = Carbon::createFromTimestamp($event['startTimestamp']);
SofaScoreEventGrapped::updateOrCreate([
'id' => (int) $event['id'],
], [
'grapper_id' => $this->task_id,
'day' => $day,
'liga' => $event['tournament']['name'],
'name' => $event['homeTeam']['name'].' vs '.$event['awayTeam']['name'],
'url' => $url,
'starttime' => $starttime->toDateTimeString(),
]);
}
}
}
}
SofaScoreTaskDay::updateOrCreate([
'grapper_id' => (int) $this->task_id,
'day' => $day,
], [
'done' => 1,
]);
}
}
app/Traits/GrappingHelper.php
+386
View File
@@ -0,0 +1,386 @@
<?php
namespace App\Traits;
use App\Models\Country;
use App\Models\SofaScoreCategory;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreRosa;
use App\Models\SofaScoreTask;
use App\Models\SofaScoreTaskDay;
use App\Models\SofaScoreTaskTournament;
use App\Models\SofaScoreTournament;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\File;
trait GrappingHelper
{
use CurlSofa;
/**
* Summary of load_days
*
* @return void
*/
protected function load_days(string $grapperSelected)
{
$this->days = [];
$task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first();
switch ($grapperSelected) {
case 'daily':
foreach ($this->daily_days as $dayBack) {
$this->days[] = Carbon::now()->subDays($dayBack)->format('Y-m-d');
}
break;
case 'range':
$task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first();
$from = $this->validate_date($task->day_from);
$from = Carbon::createFromDate($from);
$to = $this->validate_date($task->day_to);
$to = Carbon::createFromDate($to);
$lengthdates = $from->diffInDays($to);
if ($lengthdates < 1) {
Notice::note('start must be before end', 'r');
}
if ($this->restart) {
while ($from <= $to) {
$this->days[] = $from->format('Y-m-d');
$from->addDay();
}
} else {
while ($from <= $to) {
$res = SofaScoreTaskDay::firstOrNew(['grapper_id' => (int) 3, 'day' => $from->format('Y-m-d')]);
if (! $res->done) {
$this->days[] = $from->format('Y-m-d');
}
$from->addDay();
}
}
shuffle($this->days);
break;
}
if ($this->restart) {
SofaScoreEventGrapped::where('grapper_id', $this->task_id)->delete();
foreach ($this->days as $day) {
// if restarting find position
SofaScoreTaskDay::where('grapper_id', 1)
->where('day', $day)
->update(['done' => 0]);
}
if ($this->debug) {
Notice::note('Grapper restarted', 'r', 1);
}
}
$this->days = array_unique($this->days, SORT_REGULAR);
}
/**
* @return void
*/
protected function set_base_dir(string $grapperSelected)
{
switch ($grapperSelected) {
case 'daily':
$this->data_dir = storage_path($this->base_dir).'/daily';
break;
case 'range':
$this->data_dir = storage_path($this->base_dir).'/range';
break;
case 'rose':
$this->data_dir = storage_path($this->base_dir).'/rose';
break;
case 'player':
$this->data_dir = storage_path($this->base_dir).'/player';
break;
case 'excel':
$this->data_dir = storage_path($this->base_dir);
break;
}
File::makeDirectory($this->data_dir, 0775, true, true);
}
/**
* Summary of load_tournaments_requested
*
* @return void
*/
protected function load_tournaments_requested(string $grapperSelected)
{
switch ($grapperSelected) {
case 'daily':
case 'range':
$this->tournaments_requested = SofaScoreTaskTournament::with('tournament')->where('sofa_score_task_id', 1)->where('disabled', false)->get()->toArray();
break;
}
}
/**
* Summary of selectGrapper
*/
public function settings_to_local(): void
{
if ($this->debug) {
Notice::note('⚽ settings to local');
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
if ($this->debug) {
Notice::note('⚽ open vpn');
}
exec('ssh -f -N -L 13306:127.0.0.1:3306 -p 58371 root@162.55.242.190');
/**
* SofaScoreRosa
*/
if ($this->debug) {
Notice::note('SofaScoreRosa', 'g');
}
SofaScoreRosa::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_rosas')->get();
foreach ($rows as $row) {
SofaScoreRosa::insert((array) $row);
}
/**
* SofaScoreTask
*/
if ($this->debug) {
Notice::note('SofaScoreTask', 'g');
}
SofaScoreTask::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_tasks')->get();
foreach ($rows as $row) {
SofaScoreTask::insert((array) $row);
}
/**
* SofaScoreTaskTournament
*/
if ($this->debug) {
Notice::note('SofaScoreTaskTournament', 'g');
}
SofaScoreTaskTournament::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_task_tournaments')->get();
foreach ($rows as $row) {
SofaScoreTaskTournament::insert((array) $row);
}
/**
* SofaScoreTournament
*/
if ($this->debug) {
Notice::note('SofaScoreTournament', 'g');
}
SofaScoreTournament::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_tournaments')->get();
foreach ($rows as $row) {
SofaScoreTournament::insert((array) $row);
}
if ($this->debug) {
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
}
/**
* Summary of validate_date
*
* @return string
*/
protected function validate_date(string $date)
{
if (Carbon::createFromDate($date) !== false) {
return $date;
} else {
Notice::note('Wrong date format', 'r', 1);
}
}
/**
* a Grap --d --f 67 --fun=4 --task=1
*
* Ita Serie-A-Saison 23-08-2025 - 24-05-2026
* Ger Bundesliga 22-08-2025 – 16-05-2026
* Eng Premier league 15-08-2025 - 24-05-2026
* Spa La Liga 15-08-2025 - 24-05-2026
* Fra Liggue 15-08-2025 - 17-05-2026
*/
protected function grap_tournaments_for_day(string $day)
{
$tournaments_counter = 0;
$sport_id = 1;
$has = SofaScoreDayTournament::where('day', $day)->first();
$slug = 'football';
if (! $this->force && $has) {
if ($this->debug) {
Notice::note("tournaments $day for ".strtoupper($slug).' allready grapped', 'bb');
}
} else {
if ($this->debug) {
Notice::note("grappig tournaments for $day", 'bb');
}
$page = 1;
while ($page) {
// $data = $this->curl($url);
$this->sleep(1);
$url = $this->base_url."api/v1/sport/$slug/scheduled-tournaments/$day/page/$page";
$cloudflareService = new CloudflareSolverService;
$data = $cloudflareService->getJsonFromUrl($url);
if ($data['hasNextPage']) {
$page++;
} else {
$page = false;
}
foreach ($data['scheduled'] as $row) {
if (isset($row['tournament']['uniqueTournament'])) {
$tournament = $row['tournament']['uniqueTournament'];
} else {
$tournament = $row['tournament'];
}
$category = $tournament['category'];
if ($sport_id == 5) {
$country_id = 470;
} else {
/**
* CATEGORY
*/
$find = [
'Amateur',
'Bosnia & Herzegovina',
'Czechia',
'DR Congo',
'Netherlands',
'North & Central America',
'Bonaire',
'Sao Tome and Principe',
'Cape Verde',
'Macao',
'Saint Kitts and Nevis',
'Eswatini',
'Timor Leste',
'Laos',
'Congo Republic',
];
$replace = [
'',
'Bosnia Herzegovina',
'Czech Republic',
'Congo DR',
'Holland',
'North America',
'Bonaire, Sint Eustatius and Saba',
'Sao Tome e Principe',
'Cape Verde Islands',
'Macau',
'St. Kitts and Nevis',
'Swaziland',
'East Timor',
'Lao Peoples\'s Democratic Republic',
'Congo DR',
];
$country_id = Country::where('name', trim(str_replace($find, $replace, $category['name'])))->value('id');
}
if (! $country_id) {
// platzhalter fix later sofa_score_categories => txodds countries
// CATEGORY find <-> replace
// Log::channel('sofascore')->info($tournament);
$sofa_score_category_id = 10000000;
} else {
$call = SofaScoreCategory::updateOrCreate([
'id' => (int) $category['id'],
'country_id' => (int) $country_id,
'slug' => $category['slug'],
], [
'name' => $category['name'],
]);
$sofa_score_category_id = $call->id;
}
/**
* TOURNAMENT
*/
SofaScoreTournament::updateOrCreate([
'sport_id' => (int) $sport_id,
'tournament_id' => (int) $tournament['id'],
'sofa_score_category_id' => (int) $sofa_score_category_id,
], [
'name' => $tournament['name'],
'slug' => $tournament['slug'],
]);
$tournaments_counter++;
// if ($debug) {
// Notice::note('tournament: ' . $tournament['name'], 'p');
// }
/**
* DAY TOURNAMENT
*/
SofaScoreDayTournament::updateOrCreate([
'tournament_id' => (int) $tournament['id'],
'sport_id' => (int) $sport_id,
'day' => $day,
]);
}
if ($this->debug) {
Notice::note("tournaments: $tournaments_counter", 'p');
}
}
}
if ($this->debug) {
Notice::note('days tournament done', 'g');
}
}
protected function get_tree(string $dir, &$results = [])
{
$files = scandir($dir);
foreach ($files as $value) {
$path = realpath($dir.DIRECTORY_SEPARATOR.$value);
if (! is_dir($path)) {
$results[] = $path;
} elseif ($value != '.' && $value != '..') {
self::get_tree($path, $results);
$results[] = $path;
}
}
return $results;
}
}
app/Traits/GrappingToExcel.php
+85
View File
@@ -0,0 +1,85 @@
<?php
namespace App\Traits;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
trait GrappingToExcel
{
public static $max = 2000;
public function import_to_excel(string $folder)
{
ini_set('memory_limit', -1);
if ($this->debug) {
Notice::note("import $folder to excel", 'g');
}
$titel = ucfirst($folder).'_'.Carbon::now()->format('Y-m-d');
$files = $this->get_tree("$this->data_dir/$folder");
$files = array_chunk($this->filter($files), self::$max);
foreach ($files as $key => $value) {
$has = count($value);
if (! $key && $has < self::$max) {
$from = 1;
$to = $has;
} elseif (! $key) {
$from = 1;
$key++;
$to = $key * self::$max;
} elseif ($has == self::$max) {
$from = $key * self::$max + 1;
$key++;
$to = $key * self::$max;
} else {
$from = $key * self::$max + 1;
$to = $key * self::$max + $has;
}
$spreadsheet = new Spreadsheet;
$spreadsheet->getProperties()->setTitle("$titel-$from-$to");
$activeWorksheet = $spreadsheet->getActiveSheet();
$column = 1;
foreach ($value as $file) {
$data = file_get_contents($file);
$data = json_decode($data, true);
$json_string = json_encode($data, JSON_PRETTY_PRINT);
$arr = explode(PHP_EOL, $json_string);
$row = 0;
foreach ($arr as $string) {
$row++;
$replacedString = preg_replace('/\\\\u([0-9abcdef]{4})/', '&#x$1;', $string);
$utf8string = mb_convert_encoding($replacedString, 'UTF-8', 'HTML-ENTITIES');
$activeWorksheet->setCellValue([$column, $row], $utf8string);
}
$column++;
}
$spreadsheet->setActiveSheetIndex(0);
$writer = IOFactory::createWriter($spreadsheet, 'Xlsx');
$writer->save("$titel-$from-$to.xlsx");
}
}
protected function filter(array $files, array $types = ['json'])
{
$filtered = [];
foreach ($files as $file) {
$path_parts = pathinfo($file);
if (isset($path_parts['extension']) && in_array($path_parts['extension'], $types)) {
$filtered[] = $file;
}
}
return $filtered;
}
}
config/services.php
+4
View File
@@ -35,4 +35,8 @@ return [
], ],
], ],
'flaresolverr' => [
'url' => env('FLARESOLVERR_URL', '192.168.178'),
],
]; ];
database/migrations/2026_07_11_145940_create_sofa_score_event_grappeds_table.php
+1 -1
View File
@@ -13,7 +13,7 @@ return new class extends Migration
{ {
Schema::create('sofa_score_event_grappeds', function (Blueprint $table) { Schema::create('sofa_score_event_grappeds', function (Blueprint $table) {
$table->id(); $table->id();
$table->integer('grapper_id'); $table->integer('grapper_id')->default(0)->nullable();
$table->date('day')->nullable(); $table->date('day')->nullable();
$table->string('liga')->nullable(); $table->string('liga')->nullable();
$table->string('name')->nullable(); $table->string('name')->nullable();