salam/nuhodds_fi4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip del err

Browse Source
This commit is contained in:
salam committed 2026-10-04 15:22:39 +02:00
1 parent 812736ca9a
commit 9bcd4432c6
1210 files changed
-220941
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Traits/Sofascore/CurlSofa.php
-87
View File
@@ -1,87 +0,0 @@
<?php
namespace App\Traits\Sofascore;
trait CurlSofa
{
protected function curl(string $url)
{
$headers = [
'User-Agent: '.$this->useragent(),
'Accept: */*',
'Accept-Language: de,en-US;q=0.7,en;q=0.8',
'Accept-Encoding: gzip, deflate, br, zstd',
'Referer: https://www.sofascore.com/',
'DNT: 1',
'Sec-GPC: 1',
'Connection: keep-alive',
'Sec-Fetch-Dest: empty',
'Sec-Fetch-Mode: cors',
'Sec-Fetch-Site: same-origin',
'Priority: u=4',
'Pragma: no-cache',
'Cache-Control: no-cache',
'TE: trailers',
];
// 10.208.40.145
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
// This line magically handles all decompression out-of-the-box:
curl_setopt($ch, CURLOPT_ENCODING, '');
$data = curl_exec($ch);
$data = json_decode($data, true);
return $data;
}
protected function useragent()
{
$array = [];
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0';
$array[] = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0';
$array[] = 'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36';
return $array[array_rand($array)];
}
protected function sleep($speed = false)
{
switch ($speed) {
case 1:
// 5 -10 sec
$sleep = rand(5000000, 10000000);
break;
case 2:
// 10 - 20 sec
$sleep = rand(10000000, 20000000);
break;
case 3:
// 30 - 50 sec
$sleep = rand(30000000, 50000000);
break;
default:
// 1 -1.5 sec
$sleep = rand(100000, 150000);
break;
}
usleep($sleep);
}
}
app/Traits/Sofascore/GrappingEvent.php
-163
View File
@@ -1,163 +0,0 @@
<?php
namespace App\Traits\Sofascore;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreTaskDay;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\File;
trait GrappingEvent
{
protected function get_event(string $day, array $tournament)
{
$res = SofaScoreDayTournament::whereDate('day', '=', $day)->where('tournament_id', $tournament['sofa_score_tournament_id'])->first();
if ($res) {
if ($this->debug) {
Notice::note("$day - " . $tournament['tournament']['name'] . ' - ' . $tournament['sofa_score_tournament_id'], 'bb');
}
$url = $this->base_url . 'api/v1/unique-tournament/' . $tournament['sofa_score_tournament_id'] . "/scheduled-events/$day";
if ($this->debug) {
Notice::note($url, 'p');
}
$this->sleep(1);
$cloudflareService = new CloudflareSolverService;
$data = $cloudflareService->getJsonFromUrl($url);
if (isset($data['events'])) {
foreach ($data['events'] as $event) {
if (! $this->restart && SofaScoreEventGrapped::where('grapper_id', $this->task_id)->where('id', (int) $event['id'])->exists()) {
if ($this->debug) {
Notice::note('DB - Allready grapped at ' . $day . ' - ' . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'b');
}
} else {
if ($this->debug) {
Notice::note("$day - " . $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'], 'g');
}
$folder = strtolower("$this->data_dir/$day/$day-" . $event['tournament']['category']['slug'] . '-' . $event['tournament']['slug'] . '/' . $event['slug']);
File::makeDirectory($folder, 0775, true, true);
/*********************************
* EVENT event
*********************************/
File::put("$folder/event.json", json_encode($event));
/*********************************
* EVENT statistics
*********************************/
$event_id = $event['id'];
$url = $this->base_url . "api/v1/event/$event_id/statistics";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/statistics.json", json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = $this->base_url . "api/v1/event/$event_id/lineups";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/lineups.json", json_encode($data));
/*********************************
* EVENT odds
*********************************/
$url = $this->base_url . "api/v1/event/$event_id/odds/1/all";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/odds.json", json_encode($data));
/*********************************
* EVENT incidents
*********************************/
$url = $this->base_url . "api/v1/event/$event_id/incidents";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/incidents.json", json_encode($data));
$starttime = Carbon::createFromTimestamp($event['startTimestamp']);
SofaScoreEventGrapped::updateOrCreate([
'id' => (int) $event['id'],
], [
'grapper_id' => $this->task_id,
'day' => $day,
'liga' => $event['tournament']['name'],
'name' => $event['homeTeam']['name'] . ' vs ' . $event['awayTeam']['name'],
'url' => $url,
'starttime' => $starttime->toDateTimeString(),
]);
}
}
}
} else {
if ($this->debug) {
Notice::note($day . ' - ' . $tournament['tournament']['name'] . " no events", 'y');
}
}
SofaScoreTaskDay::updateOrCreate([
'grapper_id' => (int) $this->task_id,
'day' => $day,
], [
'done' => 1,
]);
}
}
app/Traits/Sofascore/GrappingHelper.php
-280
View File
@@ -1,280 +0,0 @@
<?php
namespace App\Traits\Sofascore;
use App\Models\Country;
use App\Models\SofaScoreCategory;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreRosa;
use App\Models\SofaScoreTask;
use App\Models\SofaScoreTaskDay;
use App\Models\SofaScoreTaskTournament;
use App\Models\SofaScoreTournament;
use App\Models\StatistikoImport;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Storage;
trait GrappingHelper
{
use CurlSofa;
/**
* Summary of load_days
*
* @return void
*/
protected function load_days(string $grapperSelected)
{
$this->days = [];
$task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first();
switch ($grapperSelected) {
case 'daily':
$this->days[] = Carbon::now()->subDays($this->days_back)->format('Y-m-d');
break;
case 'range':
$task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first();
$from = $this->validate_date($task->day_from);
$from = Carbon::createFromDate($from);
$to = $this->validate_date($task->day_to);
$to = Carbon::createFromDate($to);
$lengthdates = $from->diffInDays($to);
if ($lengthdates < 1) {
Notice::note('start must be before end', 'r');
}
if ($this->restart) {
while ($from <= $to) {
$this->days[] = $from->format('Y-m-d');
$from->addDay();
}
} else {
while ($from <= $to) {
$res = SofaScoreTaskDay::firstOrNew(['grapper_id' => (int) 3, 'day' => $from->format('Y-m-d')]);
if (! $res->done) {
$this->days[] = $from->format('Y-m-d');
}
$from->addDay();
}
}
shuffle($this->days);
break;
}
// if ($this->restart) {
// SofaScoreEventGrapped::where('grapper_id', $this->task_id)->delete();
// foreach ($this->days as $day) {
// // if restarting find position
// SofaScoreTaskDay::where('grapper_id', 1)
// ->where('day', $day)
// ->update(['done' => 0]);
// }
// if ($this->debug) {
// Notice::note('Grapper restarted', 'r', 1);
// }
// }
$this->days = array_unique($this->days, SORT_REGULAR);
}
/**
* @return void
*/
protected function set_base_dir(string $grapperSelected)
{
switch ($grapperSelected) {
case 'daily':
$this->data_dir = storage_path($this->base_dir) . '/daily';
break;
case 'range':
$this->data_dir = storage_path($this->base_dir) . '/range';
break;
case 'rose':
$this->data_dir = storage_path($this->base_dir) . '/rose';
break;
case 'player':
$this->data_dir = storage_path($this->base_dir) . '/player';
break;
case 'excel':
case 'mail':
case 'clear':
$this->data_dir = storage_path($this->base_dir);
break;
}
File::makeDirectory($this->data_dir, 0775, true, true);
}
/**
* Summary of load_tournaments_requested
*
* @return void
*/
protected function load_tournaments_requested(string $grapperSelected)
{
switch ($grapperSelected) {
case 'daily':
$this->tournaments_requested = SofaScoreTaskTournament::with('tournament')->where('sofa_score_task_id', 1)->where('disabled', false)->get()->toArray();
break;
case 'range':
$this->tournaments_requested = SofaScoreTaskTournament::with('tournament')->where('sofa_score_task_id', 3)->where('disabled', false)->get()->toArray();
break;
}
}
/**
* Summary of selectGrapper
*/
public function settings_to_local(): void
{
if ($this->debug) {
Notice::note('⚽ settings to local');
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
if ($this->debug) {
Notice::note('⚽ open vpn');
}
exec('ssh -f -N -L 13306:127.0.0.1:3306 -p 58371 root@162.55.242.190');
/**
* SofaScoreRosa
*/
if ($this->debug) {
Notice::note('SofaScoreRosa', 'g');
}
SofaScoreRosa::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_rosas')->get();
foreach ($rows as $row) {
SofaScoreRosa::insert((array) $row);
}
/**
* SofaScoreTask
*/
if ($this->debug) {
Notice::note('SofaScoreTask', 'g');
}
SofaScoreTask::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_tasks')->get();
foreach ($rows as $row) {
SofaScoreTask::insert((array) $row);
}
/**
* SofaScoreTaskTournament
*/
if ($this->debug) {
Notice::note('SofaScoreTaskTournament', 'g');
}
SofaScoreTaskTournament::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_task_tournaments')->get();
foreach ($rows as $row) {
SofaScoreTaskTournament::insert((array) $row);
}
/**
* SofaScoreTournament
*/
if ($this->debug) {
Notice::note('SofaScoreTournament', 'g');
}
SofaScoreTournament::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_tournaments')->get();
foreach ($rows as $row) {
SofaScoreTournament::insert((array) $row);
}
if ($this->debug) {
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
}
/**
* Summary of validate_date
*
* @return string
*/
protected function validate_date(string $date)
{
if (Carbon::createFromDate($date) !== false) {
return $date;
} else {
Notice::note('Wrong date format', 'r', 1);
}
}
protected function get_tree(string $dir, &$results = [])
{
$files = scandir($dir);
foreach ($files as $value) {
$path = realpath($dir . DIRECTORY_SEPARATOR . $value);
if (! is_dir($path)) {
$results[] = $path;
} elseif ($value != '.' && $value != '..') {
self::get_tree($path, $results);
$results[] = $path;
}
}
return $results;
}
protected function clean_string(string $string)
{
$string = str_replace(' ', '-', $string); // Replaces all spaces with hyphens.
$string = preg_replace('/[^A-Za-z0-9\-]/', '', $string); // Removes special chars.
$string = strtolower($string);
return preg_replace('/-+/', '-', $string); // Replaces multiple hyphens with single one.
}
protected function clear_folder(string $folder, $what = '')
{
$mainFolder = storage_path($this->base_dir . '/' . $folder);
if (! $what) {
$what = $this->select_what_delete();
}
switch ($what) {
case 'all':
if (File::exists($mainFolder)) {
File::deleteDirectory($mainFolder);
File::makeDirectory($mainFolder, 0755, true);
Notice::note("folder $folder empty", 'bb');
}
break;
case 'data':
if (File::exists($mainFolder)) {
$subFolders = File::directories($mainFolder);
foreach ($subFolders as $subfolder) {
File::deleteDirectory($subfolder);
}
Notice::note("data folders from $folder removed", 'bb');
}
break;
}
}
protected function delete_imported_files_orphants(): void
{
$tasks = StatistikoImport::pluck('file_name')->toArray();
collect(Storage::disk('public')->allFiles('statistiko'))
->reject(fn(string $file) => $file === '.gitignore')
->reject(fn(string $file) => in_array($file, $tasks))
->each(fn($file) => Storage::disk('public')->delete($file));
}
}
app/Traits/Sofascore/GrappingMail.php
-35
View File
@@ -1,35 +0,0 @@
<?php
namespace App\Traits\Sofascore;
use App\Mail\SofaScoreDataMail;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Mail;
trait GrappingMail
{
public function send(string $folder)
{
ini_set('memory_limit', -1);
if ($this->debug) {
Notice::note("Mail Excel: $folder", 'b');
}
$files = $this->get_tree($this->data_dir);
$sendfiles = $this->filter($files, ['xlsx']);
$header = ucfirst($folder).' '.Carbon::now()->format('Y-m-d');
Mail::to('hermann.telser@gmail.com')
->cc('ubaldo.perrucci@gmail.com')
->send(
new SofaScoreDataMail($header, $sendfiles)
);
if ($this->debug) {
Notice::note('✉ Mail Send', 'bg');
}
}
}
app/Traits/Sofascore/GrappingSofascore.php
-338
View File
@@ -1,338 +0,0 @@
<?php
namespace App\Traits\Sofascore;
use App\Mail\SimpleMail;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Mail;
use function Laravel\Prompts\select;
use function Laravel\Prompts\text;
trait GrappingSofascore
{
use GrappingEvent;
use GrappingHelper;
use GrappingMail;
use GrappingToExcel;
use GrappingTournaments;
protected int $debug;
protected int $restart;
protected int $force;
protected int $task_id;
protected int $days_back = 0;
protected string $base_dir = 'sofascore_data';
protected string $data_dir = '';
protected string $base_url = 'https://www.sofascore.com/';
protected array $tournaments_requested = [];
protected array $days = [];
public function handle(): void
{
error_reporting(E_ALL);
ini_set('display_errors', 1);
$this->debug = $this->options()['d'] ?? 0;
$this->restart = $this->options()['r'] ?? 0;
$this->force = $this->options()['f'] ?? 0;
switch ($this->getName()) {
case 'grp:daily':
$this->daily_grapper_nobreak();
break;
default:
// code...
$grapperSelected = $this->select_grapper();
$this->set_base_dir($grapperSelected);
switch ($grapperSelected) {
case 'daily':
$this->task_id = 1;
$this->days_back = (int) $this->select_day();
$this->run_daily($grapperSelected);
break;
case 'range':
$this->task_id = 3;
$this->range_grapper_nobreak();
break;
case 'settings':
$this->settings_to_local();
break;
case 'excel':
$folder = $this->select_folder();
$this->import_to_excel($folder);
break;
case 'mail':
$folder = $this->select_folder();
$this->send($folder);
break;
case 'clear':
$folder = $this->select_folder();
$this->clear_folder($folder);
break;
case 'delete_orphants':
$this->delete_imported_files_orphants();
break;
}
break;
}
}
/**
* Summary of daily_grapper_nobreak
*
* @return void
*/
protected function daily_grapper_nobreak()
{
$folder = 'daily';
$this->task_id = 1;
$this->restart = 1;
$this->set_base_dir($folder);
$this->load_tournaments_requested($folder);
$restart = $this->ask_restart();
switch ($restart) {
case 'restart':
$this->restart = 1;
break;
case 'clear':
$this->restart = 1;
$this->clear_folder($folder, 'all');
break;
}
/**
* day 1
*/
// $day = Carbon::now()->subDay()->format('Y-m-d');
// $this->grap_tournaments_for_day($day);
// foreach ($this->tournaments_requested as $tournament) {
// $this->get_event($day, $tournament);
// }
/**
* day 5
*/
// $day = Carbon::now()->subDays(5)->format('Y-m-d');
// $this->grap_tournaments_for_day($day);
// foreach ($this->tournaments_requested as $tournament) {
// $this->get_event($day, $tournament);
// }
/**
* to excel day by day
*/
$this->import_to_excel($folder);
/**
* mail it
*/
$this->send($folder);
}
/**
* Summary of run_daily
*/
protected function run_daily(string $grapperSelected)
{
$this->load_tournaments_requested($grapperSelected);
$this->load_days($grapperSelected);
if ($this->confirm($this->days[0], true)) {
foreach ($this->days as $day) {
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
}
if ($this->debug) {
Notice::note('Day grapping compleated', 'g');
}
} else {
Notice::note('day grapping cancelled', 'r');
}
}
/**
* Summary of run_range
*/
protected function range_grapper_nobreak()
{
$folder = 'range';
$this->task_id = 3;
$this->set_base_dir($folder);
$restart = $this->ask_restart();
switch ($restart) {
case 'restart':
$this->restart = 1;
break;
case 'clear':
$this->restart = 1;
$this->clear_folder($folder, 'all');
break;
}
$day_done = $this->days_done();
$this->load_tournaments_requested($folder);
$this->load_days($folder);
$count = \count($this->days);
foreach ($this->days as $key => $day) {
if ($this->debug) {
$key++;
Notice::note("$key/$count", 'br');
}
if (in_array($day, $day_done)) {
if ($this->debug) {
Notice::note("$day Events allready grapped", 'br');
}
} else {
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
}
}
$this->import_to_excel($folder);
Notice::note('Done', 'g');
Mail::to('hermann.telser@gmail.com')->send(new SimpleMail('STATISTIKO', 'Grapping compleated'));
}
protected function days_done(): array
{
$directories = File::directories($this->data_dir);
$arr = [];
foreach ($directories as $directory) {
$arr[] = substr(strrchr($directory, "/"), 1);
}
return $arr;
}
/**
* Summary of select_grapper
*/
protected function select_grapper(): string
{
return select(
label: 'What task should run?',
scroll: 10,
options: [
'daily' => 'Select day back',
'range' => 'Range',
'rose' => 'Rose',
'player' => 'Player',
'settings' => 'Settings',
'excel' => 'Excel',
'mail' => 'Send Mail',
'clear' => 'Empty Folder',
'delete_orphants' => 'Delete Imported Orphants',
]
);
}
/**
* Summary of select_folder
*/
protected function select_folder(): string
{
return select(
label: 'What folder should be used?',
scroll: 10,
options: [
'daily' => 'Daily',
'range' => 'Range',
'rose' => 'Rose',
'player' => 'Player',
]
);
}
protected function select_day(): string
{
return text(
label: 'How many days back?',
placeholder: 'z.B. 1',
default: '1',
);
}
protected function ask_restart(): string
{
return select(
label: 'Restart Grapper?',
default: 'no',
options: [
'no' => 'No',
'restart' => 'Restart',
'clear' => 'Clear folder and restart',
]
);
}
protected function select_what_delete(): string
{
return select(
label: 'What files?',
scroll: 10,
options: [
'all' => 'All',
'data' => 'Data',
]
);
}
}
app/Traits/Sofascore/GrappingToExcel.php
-108
View File
@@ -1,108 +0,0 @@
<?php
namespace App\Traits\Sofascore;
use App\Models\SofaScoreTask;
use App\Utilities\Notice;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
use Illuminate\Support\Facades\File;
trait GrappingToExcel
{
protected int $max = 2000;
protected string $titel = '';
public function import_to_excel(string $folder)
{
ini_set('memory_limit', -1);
if ($this->debug) {
Notice::note("Import $folder to Excel", 'bb');
}
switch ($folder) {
case 'daily':
$directories = File::directories($this->data_dir);
foreach ($directories as $directory) {
$day = substr(strrchr($directory, "/"), 1);
$this->titel = ucfirst($folder) . '-' . $day;
$this->make_file($this->data_dir . '/' . $day);
}
break;
case 'range':
$task = SofaScoreTask::with('sport')->where('id', 3)->first();
$this->titel = ucfirst($folder) . '-' . $this->clean_string($task->name);
$this->make_file($this->data_dir);
break;
}
}
protected function make_file(string $directory)
{
$files = $this->get_tree($directory);
// only json max 2000
$files = array_chunk($this->filter($files), $this->max);
foreach ($files as $key => $value) {
$has = count($value);
if (! $key && $has < $this->max) {
$count_from = 1;
$count_to = $has;
} elseif (! $key) {
$count_from = 1;
$key++;
$count_to = $key * $this->max;
} elseif ($has == $this->max) {
$count_from = $key * $this->max + 1;
$key++;
$count_to = $key * $this->max;
} else {
$count_from = $key * $this->max + 1;
$count_to = $key * $this->max + $has;
}
$spreadsheet = new Spreadsheet;
$spreadsheet->getProperties()->setTitle($this->titel . '-' . $count_from . '-' . $count_to);
$activeWorksheet = $spreadsheet->getActiveSheet();
$column = 1;
foreach ($value as $file) {
$data = file_get_contents($file);
$data = json_decode($data, true);
$json_string = json_encode($data, JSON_PRETTY_PRINT);
$arr = explode(PHP_EOL, $json_string);
$row = 0;
foreach ($arr as $string) {
$row++;
$replacedString = preg_replace('/\\\\u([0-9abcdef]{4})/', '&#x$1;', $string);
$utf8string = mb_convert_encoding($replacedString, 'UTF-8', 'HTML-ENTITIES');
$activeWorksheet->setCellValue([$column, $row], $utf8string);
}
$column++;
}
$spreadsheet->setActiveSheetIndex(0);
$writer = IOFactory::createWriter($spreadsheet, 'Xlsx');
$writer->save("$this->data_dir/$this->titel-$count_from-$count_to.xlsx");
}
}
protected function filter(array $files, array $types = ['json'])
{
$filtered = [];
foreach ($files as $file) {
$path_parts = pathinfo($file);
if (isset($path_parts['extension']) && in_array($path_parts['extension'], $types)) {
$filtered[] = $file;
}
}
return $filtered;
}
}
app/Traits/Sofascore/GrappingTournaments.php
-162
View File
@@ -1,162 +0,0 @@
<?php
namespace App\Traits\Sofascore;
use App\Models\SofaScoreCategory;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreTournament;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
trait GrappingTournaments
{
/**
* Summary of grap_tournaments_for_day
*
* @return void
*/
protected function grap_tournaments_for_day(string $day)
{
$sport_id = 1;
$has = SofaScoreDayTournament::where('day', $day)->first();
$slug = 'football';
if (! $this->force && $has) {
if ($this->debug) {
Notice::note("$day Tournaments allready grapped", 'bb');
}
} else {
$page = 1;
$tournaments_counter = 0;
while ($page) {
// $data = $this->curl($url);
$this->sleep();
$url = $this->base_url . "api/v1/sport/$slug/scheduled-tournaments/$day/page/$page";
$cloudflareService = new CloudflareSolverService;
$data = $cloudflareService->getJsonFromUrl($url);
if ($data['hasNextPage']) {
$page++;
} else {
$page = false;
}
foreach ($data['scheduled'] as $row) {
if (isset($row['tournament']['uniqueTournament'])) {
$tournament = $row['tournament']['uniqueTournament'];
} else {
$tournament = $row['tournament'];
}
$category = $tournament['category'];
if ($sport_id == 5) {
$country_id = 470;
} else {
/**
* CATEGORY
*/
$find = [
'Amateur',
'Bosnia & Herzegovina',
'Czechia',
'DR Congo',
'Netherlands',
'North & Central America',
'Bonaire',
'Sao Tome and Principe',
'Cape Verde',
'Macao',
'Saint Kitts and Nevis',
'Eswatini',
'Timor Leste',
'Laos',
'Congo Republic',
];
$replace = [
'',
'Bosnia Herzegovina',
'Czech Republic',
'Congo DR',
'Holland',
'North America',
'Bonaire, Sint Eustatius and Saba',
'Sao Tome e Principe',
'Cape Verde Islands',
'Macau',
'St. Kitts and Nevis',
'Swaziland',
'East Timor',
'Lao Peoples\'s Democratic Republic',
'Congo DR',
];
$country_id = Country::where('name', trim(str_replace($find, $replace, $category['name'])))->value('id');
}
if (! $country_id) {
// platzhalter fix later sofa_score_categories => txodds countries
// CATEGORY find <-> replace
// Log::channel('sofascore')->info($tournament);
$sofa_score_category_id = 10000000;
} else {
$call = SofaScoreCategory::updateOrCreate([
'id' => (int) $category['id'],
'country_id' => (int) $country_id,
'slug' => $category['slug'],
], [
'name' => $category['name'],
]);
$sofa_score_category_id = $call->id;
}
/**
* TOURNAMENT
*/
SofaScoreTournament::updateOrCreate([
'sport_id' => (int) $sport_id,
'tournament_id' => (int) $tournament['id'],
'sofa_score_category_id' => (int) $sofa_score_category_id,
], [
'name' => $tournament['name'],
'slug' => $tournament['slug'],
]);
$tournaments_counter++;
// if ($debug) {
// Notice::note('tournament: ' . $tournament['name'], 'p');
// }
/**
* DAY TOURNAMENT
*/
SofaScoreDayTournament::updateOrCreate([
'tournament_id' => (int) $tournament['id'],
'sport_id' => (int) $sport_id,
'day' => $day,
]);
}
if ($this->debug) {
Notice::note("Tournaments: $tournaments_counter", 'p');
}
}
if ($this->debug) {
Notice::note("$day Tournaments loaded", 'bb');
}
}
}
}
app/Traits/UpdateHelper.php
-497
View File
@@ -1,497 +0,0 @@
<?php
namespace App\Traits;
use App\Models\SofaScorePlayerPerSeason;
use App\Models\StatistikoDetail;
use App\Models\StatistikoPlayer;
use App\Utilities\Notice;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Str;
use function Laravel\Prompts\select;
trait UpdateHelper
{
protected float $benchmarkStartTime;
protected int $benchmarkStartMemory;
protected int $startRowCount;
protected int $startQueries;
protected int $debug;
public function handle(): void
{
error_reporting(E_ALL);
ini_set('display_errors', 1);
$tmp = $this->options();
$this->debug = $tmp['d'] ?? 0;
if ($this->debug) {
Notice::note('Update Player Season', 'g');
}
if ($this->debug) {
$playerCount = $this->selectPlayer();
} else {
$playerCount = 1000000;
}
if ($this->debug) {
$this->startBenchmark();
}
try {
$this->updatePlayers($playerCount);
} catch (\Exception $e) {
$this->error(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value());
}
if ($this->debug) {
$this->endBenchmark();
}
}
// Flight::upsert([
// ['departure' => 'Oakland', 'destination' => 'San Diego', 'price' => 99],
// ['departure' => 'Chicago', 'destination' => 'New York', 'price' => 150]
// ], uniqueBy: ['departure', 'destination'], update: ['price']);
protected function selectPlayer(): string
{
$file = select(
label: 'How many players to update?',
options: ['100 Player', '1k Player', '10k Player', '100k Player', '1m Player']
);
return match ($file) {
'100 Player' => 100,
'1k Player' => 1000,
'10k Player' => 10000,
'100k Player' => 100000,
'1m Player' => 1000000,
};
}
protected function startBenchmark(string $table = 'ss_player_seasons'): void
{
$this->startRowCount = DB::table($table)->count();
$this->benchmarkStartTime = microtime(true);
$this->benchmarkStartMemory = memory_get_usage();
DB::enableQueryLog();
$this->startQueries = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value;
}
protected function endBenchmark(string $table = 'ss_player_seasons'): void
{
$executionTime = microtime(true) - $this->benchmarkStartTime;
$memoryUsage = round((memory_get_usage() - $this->benchmarkStartMemory) / 1024 / 1024, 2);
$queriesCount = DB::select("SHOW SESSION STATUS LIKE 'Questions'")[0]->Value - $this->startQueries - 1; // Subtract the Questions query itself
// Get row count after we've stopped tracking queries
$rowDiff = DB::table($table)->count() - $this->startRowCount;
$formattedTime = match (true) {
$executionTime >= 60 => sprintf('%dm %ds', floor($executionTime / 60), $executionTime % 60),
$executionTime >= 1 => round($executionTime, 2).'s',
default => round($executionTime * 1000).'ms',
};
$this->newLine();
$this->line(sprintf(
'⚡ <bg=bright-cyan;fg=black> TIME: %s </> <bg=bright-green;fg=black> MEM: %sMB </> <bg=bright-yellow;fg=black> SQL: %s </> <bg=bright-magenta;fg=black> ROWS: %s </>',
$formattedTime,
$memoryUsage,
number_format($queriesCount),
number_format($rowDiff)
));
$this->newLine();
}
protected function updatePlayers(int $playerCount)
{
$res_player = StatistikoDetail::query()->where('is_active', 1)->limit($playerCount)->pluck('ss_player_id')->toArray();
$chunks = array_chunk($res_player, 500);
foreach ($chunks as $chunk) {
$res = StatistikoPlayer::with('event')->with('details')->whereIn('ss_player_id', $chunk)->orderBy('ss_player_id')->get();
$collection = [];
foreach ($res as $val) {
if (isset($val->event)) {
$ss_player_id = (int) $val->ss_player_id;
$activeteam_id = (int) $val->activeteam_id;
$minutes_played = (int) $val->minutes_played;
$ss_team_id = (int) $val->details->ss_team_id;
$season = $val->event->season;
$home_id = (int) $val->event->home_id;
$away_id = (int) $val->event->away_id;
if (! isset($collection[$season][$ss_player_id])) {
$this->array_template($collection, $ss_player_id, $season);
}
if (! isset($collection['total'][$ss_player_id])) {
$this->array_template($collection, $ss_player_id, 'total');
}
if ($minutes_played > 0) {
if ($activeteam_id == $home_id) {
$collection[$season][$ss_player_id]['events_played_home'] += 1;
$collection[$season][$ss_player_id]['minutes_played_home'] += $minutes_played;
$collection['total'][$ss_player_id]['events_played_home'] += 1;
$collection['total'][$ss_player_id]['minutes_played_home'] += $minutes_played;
} elseif ($activeteam_id == $away_id) {
$collection[$season][$ss_player_id]['events_played_away'] += 1;
$collection[$season][$ss_player_id]['minutes_played_away'] += $minutes_played;
$collection['total'][$ss_player_id]['events_played_away'] += 1;
$collection['total'][$ss_player_id]['minutes_played_away'] += $minutes_played;
}
// ubaldo 18. Sep 26 intanto vado avanti, appena trovo qwuaqlcosa ti scrivo, tu togli i red cards dal conteggio
$cards = $val->card_yellow + $val->card_yellowred;
$collection[$season][$ss_player_id]['ss_player_id'] = $ss_player_id;
$collection[$season][$ss_player_id]['ss_team_id'] = $ss_team_id;
$collection[$season][$ss_player_id]['season'] = $season;
$collection[$season][$ss_player_id]['total_events']++;
$collection[$season][$ss_player_id]['total_min'] += $minutes_played;
$collection[$season][$ss_player_id]['average_min'] = round($collection[$season][$ss_player_id]['total_min'] / $collection[$season][$ss_player_id]['total_events']);
$collection[$season][$ss_player_id]['goals'] += $val->goals;
$collection[$season][$ss_player_id]['fouls'] += $val->fouls;
$collection[$season][$ss_player_id]['cards'] += $cards;
$collection[$season][$ss_player_id]['offsides'] += $val->offsides;
$collection[$season][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
$collection[$season][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
$collection[$season][$ss_player_id]['shots_blocked'] += $val->shots_blocked;
$collection[$season][$ss_player_id]['total_shots'] += $val->shots_on_target + $val->shots_off_target + $val->shots_blocked;
$collection['total'][$ss_player_id]['ss_player_id'] = $ss_player_id;
$collection['total'][$ss_player_id]['ss_team_id'] = $ss_team_id;
$collection['total'][$ss_player_id]['season'] = 'total';
$collection['total'][$ss_player_id]['total_events']++;
$collection['total'][$ss_player_id]['total_min'] += $minutes_played;
$collection['total'][$ss_player_id]['average_min'] = round($collection['total'][$ss_player_id]['total_min'] / $collection['total'][$ss_player_id]['total_events']);
$collection['total'][$ss_player_id]['goals'] += $val->goals;
$collection['total'][$ss_player_id]['fouls'] += $val->fouls;
$collection['total'][$ss_player_id]['cards'] += $cards;
$collection['total'][$ss_player_id]['offsides'] += $val->offsides;
$collection['total'][$ss_player_id]['shots_on_target'] += $val->shots_on_target;
$collection['total'][$ss_player_id]['shots_off_target'] += $val->shots_off_target;
$collection['total'][$ss_player_id]['shots_blocked'] += $val->shots_blocked;
$collection['total'][$ss_player_id]['total_shots'] += $val->shots_on_target + $val->shots_off_target + $val->shots_blocked;
}
}
}
$collection_1d = [];
foreach ($collection as $arr) {
foreach ($arr as $row) {
$collection_1d[] = $row;
}
}
SofaScorePlayerPerSeason::upsert(
$collection_1d,
uniqueBy: [
'ss_player_id',
'ss_team_id',
'season',
],
update: [
'total_events',
'total_min',
'average_min',
'events_played_home',
'events_played_away',
'minutes_played_home',
'minutes_played_away',
'goals',
'fouls',
'offsides',
'shots_on_target',
'shots_off_target',
'shots_blocked',
'total_shots',
'cards',
]
);
}
}
private function array_template(array &$collection, int $ss_player_id, string $season)
{
$collection[$season][$ss_player_id]['ss_player_id'] = 0;
$collection[$season][$ss_player_id]['ss_team_id'] = 0;
$collection[$season][$ss_player_id]['season'] = '';
$collection[$season][$ss_player_id]['total_events'] = 0;
$collection[$season][$ss_player_id]['total_min'] = 0;
$collection[$season][$ss_player_id]['average_min'] = 0;
$collection[$season][$ss_player_id]['events_played_home'] = 0;
$collection[$season][$ss_player_id]['events_played_away'] = 0;
$collection[$season][$ss_player_id]['minutes_played_home'] = 0;
$collection[$season][$ss_player_id]['minutes_played_away'] = 0;
$collection[$season][$ss_player_id]['goals'] = 0;
$collection[$season][$ss_player_id]['fouls'] = 0;
$collection[$season][$ss_player_id]['offsides'] = 0;
$collection[$season][$ss_player_id]['shots_on_target'] = 0;
$collection[$season][$ss_player_id]['shots_off_target'] = 0;
$collection[$season][$ss_player_id]['shots_blocked'] = 0;
$collection[$season][$ss_player_id]['total_shots'] = 0;
$collection[$season][$ss_player_id]['cards'] = 0;
}
// private function import08ManualStreamingWithPdo(string $filePath): void
// {
// // 100 7ms / 0MB
// // 1K 78ms / 0MB
// // 10K 328ms / 0MB
// // 100K 2.9s / 0MB
// // 1M 28s / 0MB
// $data = [];
// $handle = fopen($filePath, 'rb');
// fgetcsv($handle); // skip header
// $now = now()->format('Y-m-d H:i:s');
// $pdo = DB::connection()->getPdo();
// while (($row = fgetcsv($handle)) !== false) {
// $data[] = [
// 'custom_id' => $row[0],
// 'name' => $row[1],
// 'email' => $row[2],
// 'company' => $row[3],
// 'city' => $row[4],
// 'country' => $row[5],
// 'birthday' => $row[6],
// 'created_at' => $now,
// 'updated_at' => $now,
// ];
// if (count($data) === 1000) {
// // Build the SQL query for the chunk
// $columns = array_keys($data[0]);
// $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
// $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
// // Flatten the data array for the query
// $values = [];
// foreach ($data as $row) {
// $values = array_merge($values, array_values($row));
// }
// $pdo->prepare($sql)->execute($values);
// $data = [];
// }
// }
// if (! empty($data)) {
// $columns = array_keys($data[0]);
// $placeholders = rtrim(str_repeat('(?,?,?,?,?,?,?,?,?),', count($data)), ',');
// $sql = 'INSERT INTO customers ('.implode(',', $columns).') VALUES '.$placeholders;
// $values = [];
// foreach ($data as $row) {
// $values = array_merge($values, array_values($row));
// }
// $pdo->prepare($sql)->execute($values);
// }
// fclose($handle);
// }
// private function import09PDOPrepared(string $filePath): void
// {
// // Direct database connection with prepared statements
// // 100 41ms / 0 MB
// // 1K 237ms /
// // 10K 2.21s
// // 100K 25.27s
// // 1M 4m43s
// // 2M
// $now = now()->format('Y-m-d H:i:s');
// $handle = fopen($filePath, 'r');
// fgets($handle); // skip header
// try {
// $pdo = DB::connection()->getPdo();
// $stmt = $pdo->prepare('
// INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
// VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
// ');
// while (($row = fgetcsv($handle)) !== false) {
// $stmt->execute([
// $row[0],
// $row[1],
// $row[2],
// $row[3],
// $row[4],
// $row[5],
// $row[6],
// $now,
// $now,
// ]);
// }
// } finally {
// fclose($handle);
// }
// }
// private function import10PDOPreparedChunked(string $filePath): void
// {
// // Direct database connection with prepared statements
// // 100 12ms / 0.15MB
// // 1K 49ms / 0.74MB
// // 10K 222ms / 0.74MB
// // 100K 1.5s / 0.74MB
// // 1M 15.3s / 0.74MB
// // 2M 24s / 0.74MN
// $now = now()->format('Y-m-d H:i:s');
// $handle = fopen($filePath, 'r');
// fgetcsv($handle); // skip header
// $chunkSize = 500;
// $chunks = [];
// try {
// $stmt = $this->prepareChunkedStatement($chunkSize);
// while (($row = fgetcsv($handle)) !== false) {
// $chunks = array_merge($chunks, [
// $row[0], $row[1], $row[2], $row[3], $row[4],
// $row[5], $row[6], $now, $now,
// ]);
// if (count($chunks) === $chunkSize * 9) { // 9 columns
// $stmt->execute($chunks);
// $chunks = [];
// }
// }
// // Handle remaining records
// if (! empty($chunks)) {
// $remainingRows = count($chunks) / 9;
// $stmt = $this->prepareChunkedStatement($remainingRows);
// $stmt->execute($chunks);
// }
// } finally {
// fclose($handle);
// }
// }
// private function import11Concurrent(string $filePath): void
// {
// // 100 168ms
// // 1K 172ms
// // 10K 234ms
// // 100K 595ms
// // 1M 4.36s
// // 2M 8.8s
// $now = now()->format('Y-m-d H:i:s');
// $numberOfProcesses = 10;
// $tasks = [];
// for ($i = 0; $i < $numberOfProcesses; $i++) {
// $tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
// DB::reconnect();
// $handle = fopen($filePath, 'r');
// fgets($handle); // Skip header
// $currentLine = 0;
// $customers = [];
// while (($line = fgets($handle)) !== false) {
// // Each process takes every Nth line
// if ($currentLine++ % $numberOfProcesses !== $i) {
// continue;
// }
// $row = str_getcsv($line);
// $customers[] = [
// 'custom_id' => $row[0],
// 'name' => $row[1],
// 'email' => $row[2],
// 'company' => $row[3],
// 'city' => $row[4],
// 'country' => $row[5],
// 'birthday' => $row[6],
// 'created_at' => $now,
// 'updated_at' => $now,
// ];
// if (count($customers) === 1000) {
// DB::table('customers')->insert($customers);
// $customers = [];
// }
// }
// if (! empty($customers)) {
// DB::table('customers')->insert($customers);
// }
// fclose($handle);
// return true;
// };
// }
// Concurrency::run($tasks);
// }
// private function import12LoadDataInfile(string $filePath): void
// {
// // MySQL specific, fastest approach
// // 100 10ms / 0MB
// // 1K 29ms / 0MB
// // 10K 115ms / 0MB
// // 100K 567ms / 0MB
// // 1M 5s / 0MB
// // 2M 11s / 0MB
// $pdo = DB::connection()->getPdo();
// $pdo->setAttribute(PDO::MYSQL_ATTR_LOCAL_INFILE, true);
// $filepath = str_replace('\\', '/', $filePath);
// $query = <<<SQL
// LOAD DATA LOCAL INFILE '$filepath'
// INTO TABLE customers
// FIELDS TERMINATED BY ','
// ENCLOSED BY '"'
// LINES TERMINATED BY '\n'
// IGNORE 1 LINES
// (@col1, @col2, @col3, @col4, @col5, @col6, @col7)
// SET
// custom_id = @col1,
// name = @col2,
// email = @col3,
// company = @col4,
// city = @col5,
// country = @col6,
// birthday = @col7,
// created_at = NOW(),
// updated_at = NOW()
// SQL;
// $pdo->exec($query);
// }
// private function prepareChunkedStatement($chunkSize): PDOStatement
// {
// $rowPlaceholders = '(?, ?, ?, ?, ?, ?, ?, ?, ?)';
// $placeholders = implode(',', array_fill(0, $chunkSize, $rowPlaceholders));
// return DB::connection()->getPdo()->prepare("
// INSERT INTO customers (custom_id, name, email, company, city, country, birthday, created_at, updated_at)
// VALUES {$placeholders}
// ");
// }
}