salam/nuhodds_fi4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add daily no break grapper

Browse Source
This commit is contained in:
salam committed 2026-09-27 03:42:05 +02:00
1 parent 5188d1f171
commit 17de06cf6b
9 files changed
+325 -177
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Traits/Sofascore/CurlSofa.php
+87
View File
@@ -0,0 +1,87 @@
<?php
namespace App\Traits\Sofascore;
trait CurlSofa
{
protected function curl(string $url)
{
$headers = [
'User-Agent: '.$this->useragent(),
'Accept: */*',
'Accept-Language: de,en-US;q=0.7,en;q=0.8',
'Accept-Encoding: gzip, deflate, br, zstd',
'Referer: https://www.sofascore.com/',
'DNT: 1',
'Sec-GPC: 1',
'Connection: keep-alive',
'Sec-Fetch-Dest: empty',
'Sec-Fetch-Mode: cors',
'Sec-Fetch-Site: same-origin',
'Priority: u=4',
'Pragma: no-cache',
'Cache-Control: no-cache',
'TE: trailers',
];
// 10.208.40.145
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
// This line magically handles all decompression out-of-the-box:
curl_setopt($ch, CURLOPT_ENCODING, '');
$data = curl_exec($ch);
$data = json_decode($data, true);
return $data;
}
protected function useragent()
{
$array = [];
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0';
$array[] = 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
$array[] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.1 Safari/605.1.15';
$array[] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0';
$array[] = 'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36';
return $array[array_rand($array)];
}
protected function sleep($speed = false)
{
switch ($speed) {
case 1:
// 5 -10 sec
$sleep = rand(5000000, 10000000);
break;
case 2:
// 10 - 20 sec
$sleep = rand(10000000, 20000000);
break;
case 3:
// 30 - 50 sec
$sleep = rand(30000000, 50000000);
break;
default:
// 1 -1.5 sec
$sleep = rand(100000, 150000);
break;
}
usleep($sleep);
}
}
app/Traits/Sofascore/GrapSofascore.php
+250
View File
@@ -0,0 +1,250 @@
<?php
namespace App\Traits\Sofascore;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use function Laravel\Prompts\select;
use function Laravel\Prompts\text;
trait GrapSofascore
{
use GrapperEvent;
use GrappingHelper;
use GrappingMail;
use GrappingToExcel;
protected int $debug;
protected int $restart;
protected int $force;
protected int $task_id;
protected int $days_back = 0;
protected string $base_dir = 'sofascore_data';
protected string $data_dir = '';
protected string $base_url = 'https://www.sofascore.com/';
protected array $tournaments_requested = [];
protected array $days = [];
public function handle(): void
{
error_reporting(E_ALL);
ini_set('display_errors', 1);
$this->debug = $this->options()['d'] ?? 0;
$this->restart = $this->options()['r'] ?? 0;
$this->force = $this->options()['f'] ?? 0;
switch ($this->getName()) {
case 'grp:daily':
$this->daily_grapper_nobreak();
break;
default:
// code...
$grapperSelected = $this->select_grapper();
$this->set_base_dir($grapperSelected);
switch ($grapperSelected) {
case 'daily':
$this->task_id = 1;
$this->days_back = (int) $this->select_day();
$this->run_daily($grapperSelected);
break;
case 'range':
$this->task_id = 3;
$this->run_range($grapperSelected);
break;
case 'settings':
$this->settings_to_local();
break;
case 'excel':
$folder = $this->select_folder();
$this->import_to_excel($folder);
break;
case 'mail':
$folder = $this->select_folder();
$this->send($folder);
break;
case 'clear':
$folder = $this->select_folder();
$this->clear_folder($folder);
break;
}
break;
}
}
/**
* Summary of daily_grapper_nobreak
*
* @return void
*/
protected function daily_grapper_nobreak()
{
$folder = 'daily';
$this->task_id = 1;
$this->set_base_dir($folder);
$this->load_tournaments_requested($folder);
/**
* day 1
*/
$this->clear_folder($folder, 'all');
$day = Carbon::now()->subDay()->format('Y-m-d');
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
$this->import_to_excel($folder, "Daily-$day");
$this->clear_folder($folder, 'data');
/**
* day 5
*/
$day = Carbon::now()->subDays(5)->format('Y-m-d');
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
$this->import_to_excel($folder, "Daily-$day");
$this->send($folder);
}
/**
* Summary of run_daily
*/
protected function run_daily(string $grapperSelected)
{
$this->load_tournaments_requested($grapperSelected);
$this->load_days($grapperSelected);
if ($this->confirm($this->days[0], true)) {
foreach ($this->days as $day) {
$this->grap_tournaments_for_day($day);
foreach ($this->tournaments_requested as $tournament) {
$this->get_event($day, $tournament);
}
}
if ($this->debug) {
Notice::note('Day grapping compleated', 'g');
}
} else {
Notice::note('day grapping cancelled', 'r');
}
}
/**
* Summary of run_range
*
* @return never
*/
protected function run_range(string $grapperSelected)
{
$this->load_tournaments_requested($grapperSelected);
$this->load_days($grapperSelected);
dd($this->days);
}
/**
* Summary of select_grapper
*/
protected function select_grapper(): string
{
return select(
label: 'What task should run?',
scroll: 10,
options: [
'daily' => 'Select day back',
'range' => 'Range',
'rose' => 'Rose',
'player' => 'Player',
'settings' => 'Settings',
'excel' => 'Excel',
'mail' => 'Send Mail',
'clear' => 'Empty Folder',
]
);
}
/**
* Summary of select_folder
*/
protected function select_folder(): string
{
return select(
label: 'What folder should be used?',
scroll: 10,
options: [
'daily' => 'Daily',
'range' => 'Range',
'rose' => 'Rose',
'player' => 'Player',
]
);
}
protected function select_day(): string
{
return text(
label: 'How many days back?',
placeholder: 'z.B. 1',
default: '1',
);
}
protected function select_what_delete(): string
{
return select(
label: 'What files?',
scroll: 10,
options: [
'all' => 'All',
'data' => 'Data',
]
);
}
}
app/Traits/Sofascore/GrapperEvent.php
+161
View File
@@ -0,0 +1,161 @@
<?php
namespace App\Traits\Sofascore;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreTaskDay;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\File;
trait GrapperEvent
{
protected function get_event(string $day, array $tournament)
{
$res = SofaScoreDayTournament::whereDate('day', '=', $day)->where('tournament_id', $tournament['sofa_score_tournament_id'])->first();
if ($res) {
if ($this->debug) {
Notice::note("$day - ".$tournament['tournament']['name'].' - '.$tournament['sofa_score_tournament_id'], 'b');
}
$this->sleep(1);
$url = $this->base_url.'api/v1/unique-tournament/'.$tournament['sofa_score_tournament_id']."/scheduled-events/$day";
if ($this->debug) {
Notice::note($url, 'bb');
}
$this->sleep(1);
$cloudflareService = new CloudflareSolverService;
$data = $cloudflareService->getJsonFromUrl($url);
if (isset($data['events'])) {
foreach ($data['events'] as $event) {
if (! $this->force && SofaScoreEventGrapped::where('grapper_id', $this->task_id)->where('id', (int) $event['id'])->exists()) {
if ($this->debug) {
Notice::note('DB - Allready grapped at '.$day.' - '.$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'b');
}
} else {
if ($this->debug) {
Notice::note("$day - ".$event['homeTeam']['name'].' vs '.$event['awayTeam']['name'], 'g');
}
$folder = strtolower("$this->data_dir/$day/$day-".$event['tournament']['category']['slug'].'-'.$event['tournament']['slug'].'/'.$event['slug']);
File::makeDirectory($folder, 0775, true, true);
/*********************************
* EVENT event
*********************************/
File::put("$folder/event.json", json_encode($event));
/*********************************
* EVENT statistics
*********************************/
$event_id = $event['id'];
$url = $this->base_url."api/v1/event/$event_id/statistics";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/statistics.json", json_encode($data));
/*********************************
* EVENT lineups
*********************************/
$url = $this->base_url."api/v1/event/$event_id/lineups";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/lineups.json", json_encode($data));
/*********************************
* EVENT odds
*********************************/
$url = $this->base_url."api/v1/event/$event_id/odds/1/all";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/odds.json", json_encode($data));
/*********************************
* EVENT incidents
*********************************/
$url = $this->base_url."api/v1/event/$event_id/incidents";
if ($this->debug) {
Notice::note($url, 'b');
}
$this->sleep();
$data = $cloudflareService->getJsonFromUrl($url);
$data['id'] = $event_id;
File::put("$folder/incidents.json", json_encode($data));
$starttime = Carbon::createFromTimestamp($event['startTimestamp']);
SofaScoreEventGrapped::updateOrCreate([
'id' => (int) $event['id'],
], [
'grapper_id' => $this->task_id,
'day' => $day,
'liga' => $event['tournament']['name'],
'name' => $event['homeTeam']['name'].' vs '.$event['awayTeam']['name'],
'url' => $url,
'starttime' => $starttime->toDateTimeString(),
]);
}
}
}
}
SofaScoreTaskDay::updateOrCreate([
'grapper_id' => (int) $this->task_id,
'day' => $day,
], [
'done' => 1,
]);
}
}
app/Traits/Sofascore/GrappingHelper.php
+423
View File
@@ -0,0 +1,423 @@
<?php
namespace App\Traits\Sofascore;
use App\Models\Country;
use App\Models\SofaScoreCategory;
use App\Models\SofaScoreDayTournament;
use App\Models\SofaScoreEventGrapped;
use App\Models\SofaScoreRosa;
use App\Models\SofaScoreTask;
use App\Models\SofaScoreTaskDay;
use App\Models\SofaScoreTaskTournament;
use App\Models\SofaScoreTournament;
use App\Services\CloudflareSolverService;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\File;
trait GrappingHelper
{
use CurlSofa;
/**
* Summary of load_days
*
* @return void
*/
protected function load_days(string $grapperSelected)
{
$this->days = [];
$task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first();
switch ($grapperSelected) {
case 'daily':
$this->days[] = Carbon::now()->subDays($this->days_back)->format('Y-m-d');
break;
case 'range':
$task = SofaScoreTask::with('sport')->where('id', $this->task_id)->first();
$from = $this->validate_date($task->day_from);
$from = Carbon::createFromDate($from);
$to = $this->validate_date($task->day_to);
$to = Carbon::createFromDate($to);
$lengthdates = $from->diffInDays($to);
if ($lengthdates < 1) {
Notice::note('start must be before end', 'r');
}
if ($this->restart) {
while ($from <= $to) {
$this->days[] = $from->format('Y-m-d');
$from->addDay();
}
} else {
while ($from <= $to) {
$res = SofaScoreTaskDay::firstOrNew(['grapper_id' => (int) 3, 'day' => $from->format('Y-m-d')]);
if (! $res->done) {
$this->days[] = $from->format('Y-m-d');
}
$from->addDay();
}
}
shuffle($this->days);
break;
}
if ($this->restart) {
SofaScoreEventGrapped::where('grapper_id', $this->task_id)->delete();
foreach ($this->days as $day) {
// if restarting find position
SofaScoreTaskDay::where('grapper_id', 1)
->where('day', $day)
->update(['done' => 0]);
}
if ($this->debug) {
Notice::note('Grapper restarted', 'r', 1);
}
}
$this->days = array_unique($this->days, SORT_REGULAR);
}
/**
* @return void
*/
protected function set_base_dir(string $grapperSelected)
{
switch ($grapperSelected) {
case 'daily':
$this->data_dir = storage_path($this->base_dir).'/daily';
break;
case 'range':
$this->data_dir = storage_path($this->base_dir).'/range';
break;
case 'rose':
$this->data_dir = storage_path($this->base_dir).'/rose';
break;
case 'player':
$this->data_dir = storage_path($this->base_dir).'/player';
break;
case 'excel':
case 'mail':
case 'clear':
$this->data_dir = storage_path($this->base_dir);
break;
}
File::makeDirectory($this->data_dir, 0775, true, true);
}
/**
* Summary of load_tournaments_requested
*
* @return void
*/
protected function load_tournaments_requested(string $grapperSelected)
{
switch ($grapperSelected) {
case 'daily':
case 'range':
$this->tournaments_requested = SofaScoreTaskTournament::with('tournament')->where('sofa_score_task_id', 1)->where('disabled', false)->get()->toArray();
break;
}
}
/**
* Summary of selectGrapper
*/
public function settings_to_local(): void
{
if ($this->debug) {
Notice::note('⚽ settings to local');
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
if ($this->debug) {
Notice::note('⚽ open vpn');
}
exec('ssh -f -N -L 13306:127.0.0.1:3306 -p 58371 root@162.55.242.190');
/**
* SofaScoreRosa
*/
if ($this->debug) {
Notice::note('SofaScoreRosa', 'g');
}
SofaScoreRosa::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_rosas')->get();
foreach ($rows as $row) {
SofaScoreRosa::insert((array) $row);
}
/**
* SofaScoreTask
*/
if ($this->debug) {
Notice::note('SofaScoreTask', 'g');
}
SofaScoreTask::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_tasks')->get();
foreach ($rows as $row) {
SofaScoreTask::insert((array) $row);
}
/**
* SofaScoreTaskTournament
*/
if ($this->debug) {
Notice::note('SofaScoreTaskTournament', 'g');
}
SofaScoreTaskTournament::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_task_tournaments')->get();
foreach ($rows as $row) {
SofaScoreTaskTournament::insert((array) $row);
}
/**
* SofaScoreTournament
*/
if ($this->debug) {
Notice::note('SofaScoreTournament', 'g');
}
SofaScoreTournament::truncate();
$rows = DB::connection('remote_mariadb')->table('sofa_score_tournaments')->get();
foreach ($rows as $row) {
SofaScoreTournament::insert((array) $row);
}
if ($this->debug) {
Notice::note('⚽ close all vpn');
}
exec('killall ssh');
}
/**
* Summary of validate_date
*
* @return string
*/
protected function validate_date(string $date)
{
if (Carbon::createFromDate($date) !== false) {
return $date;
} else {
Notice::note('Wrong date format', 'r', 1);
}
}
/**
* a Grap --d --f 67 --fun=4 --task=1
*
* Ita Serie-A-Saison 23-08-2025 - 24-05-2026
* Ger Bundesliga 22-08-2025 – 16-05-2026
* Eng Premier league 15-08-2025 - 24-05-2026
* Spa La Liga 15-08-2025 - 24-05-2026
* Fra Liggue 15-08-2025 - 17-05-2026
*/
protected function grap_tournaments_for_day(string $day)
{
$tournaments_counter = 0;
$sport_id = 1;
$has = SofaScoreDayTournament::where('day', $day)->first();
$slug = 'football';
if (! $this->force && $has) {
if ($this->debug) {
Notice::note("tournaments $day for ".strtoupper($slug).' allready grapped', 'bb');
}
} else {
if ($this->debug) {
Notice::note("grappig tournaments for $day", 'bb');
}
$page = 1;
while ($page) {
// $data = $this->curl($url);
$this->sleep(1);
$url = $this->base_url."api/v1/sport/$slug/scheduled-tournaments/$day/page/$page";
$cloudflareService = new CloudflareSolverService;
$data = $cloudflareService->getJsonFromUrl($url);
if ($data['hasNextPage']) {
$page++;
} else {
$page = false;
}
foreach ($data['scheduled'] as $row) {
if (isset($row['tournament']['uniqueTournament'])) {
$tournament = $row['tournament']['uniqueTournament'];
} else {
$tournament = $row['tournament'];
}
$category = $tournament['category'];
if ($sport_id == 5) {
$country_id = 470;
} else {
/**
* CATEGORY
*/
$find = [
'Amateur',
'Bosnia & Herzegovina',
'Czechia',
'DR Congo',
'Netherlands',
'North & Central America',
'Bonaire',
'Sao Tome and Principe',
'Cape Verde',
'Macao',
'Saint Kitts and Nevis',
'Eswatini',
'Timor Leste',
'Laos',
'Congo Republic',
];
$replace = [
'',
'Bosnia Herzegovina',
'Czech Republic',
'Congo DR',
'Holland',
'North America',
'Bonaire, Sint Eustatius and Saba',
'Sao Tome e Principe',
'Cape Verde Islands',
'Macau',
'St. Kitts and Nevis',
'Swaziland',
'East Timor',
'Lao Peoples\'s Democratic Republic',
'Congo DR',
];
$country_id = Country::where('name', trim(str_replace($find, $replace, $category['name'])))->value('id');
}
if (! $country_id) {
// platzhalter fix later sofa_score_categories => txodds countries
// CATEGORY find <-> replace
// Log::channel('sofascore')->info($tournament);
$sofa_score_category_id = 10000000;
} else {
$call = SofaScoreCategory::updateOrCreate([
'id' => (int) $category['id'],
'country_id' => (int) $country_id,
'slug' => $category['slug'],
], [
'name' => $category['name'],
]);
$sofa_score_category_id = $call->id;
}
/**
* TOURNAMENT
*/
SofaScoreTournament::updateOrCreate([
'sport_id' => (int) $sport_id,
'tournament_id' => (int) $tournament['id'],
'sofa_score_category_id' => (int) $sofa_score_category_id,
], [
'name' => $tournament['name'],
'slug' => $tournament['slug'],
]);
$tournaments_counter++;
// if ($debug) {
// Notice::note('tournament: ' . $tournament['name'], 'p');
// }
/**
* DAY TOURNAMENT
*/
SofaScoreDayTournament::updateOrCreate([
'tournament_id' => (int) $tournament['id'],
'sport_id' => (int) $sport_id,
'day' => $day,
]);
}
if ($this->debug) {
Notice::note("tournaments: $tournaments_counter", 'p');
}
}
}
if ($this->debug) {
Notice::note('days tournament done', 'g');
}
}
protected function get_tree(string $dir, &$results = [])
{
$files = scandir($dir);
foreach ($files as $value) {
$path = realpath($dir.DIRECTORY_SEPARATOR.$value);
if (! is_dir($path)) {
$results[] = $path;
} elseif ($value != '.' && $value != '..') {
self::get_tree($path, $results);
$results[] = $path;
}
}
return $results;
}
protected function clean_string(string $string)
{
$string = str_replace(' ', '-', $string); // Replaces all spaces with hyphens.
$string = preg_replace('/[^A-Za-z0-9\-]/', '', $string); // Removes special chars.
$string = strtolower($string);
return preg_replace('/-+/', '-', $string); // Replaces multiple hyphens with single one.
}
protected function clear_folder(string $folder, $what = '')
{
$mainFolder = storage_path($this->base_dir.'/'.$folder);
if (! $what) {
$what = $this->select_what_delete();
}
switch ($what) {
case 'all':
if (File::exists($mainFolder)) {
File::deleteDirectory($mainFolder);
File::makeDirectory($mainFolder, 0755, true);
Notice::note("folder $folder empty", 'bb');
}
break;
case 'data':
if (File::exists($mainFolder)) {
$subFolders = File::directories($mainFolder);
foreach ($subFolders as $subfolder) {
File::deleteDirectory($subfolder);
}
Notice::note("data folders from $folder removed", 'bb');
}
break;
}
}
}
app/Traits/Sofascore/GrappingMail.php
+35
View File
@@ -0,0 +1,35 @@
<?php
namespace App\Traits\Sofascore;
use App\Mail\SofaScoreDataMail;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Mail;
trait GrappingMail
{
public function send(string $folder)
{
ini_set('memory_limit', -1);
if ($this->debug) {
Notice::note("Eend excel files from folder $folder", 'b');
}
$files = $this->get_tree("$this->data_dir/$folder");
$sendfiles = $this->filter($files, ['xlsx']);
$header = ucfirst($folder).' '.Carbon::now()->format('Y-m-d');
Mail::to('hermann.telser@gmail.com')
->cc('ubaldo.perrucci@gmail.com')
->send(
new SofaScoreDataMail($header, $sendfiles)
);
if ($this->debug) {
Notice::note('✉ Mail Send', 'bg');
}
}
}
app/Traits/Sofascore/GrappingToExcel.php
+88
View File
@@ -0,0 +1,88 @@
<?php
namespace App\Traits\Sofascore;
use App\Utilities\Notice;
use Illuminate\Support\Carbon;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Spreadsheet;
trait GrappingToExcel
{
public static $max = 2000;
public function import_to_excel(string $folder, $title = '')
{
ini_set('memory_limit', -1);
if ($this->debug) {
Notice::note("import $folder to excel", 'g');
}
if (! $title) {
$titel = ucfirst($folder).'_'.Carbon::now()->format('Y-m-d');
}
$files = $this->get_tree("$this->data_dir/$folder");
$files = array_chunk($this->filter($files), self::$max);
foreach ($files as $key => $value) {
$has = count($value);
if (! $key && $has < self::$max) {
$count_from = 1;
$count_to = $has;
} elseif (! $key) {
$count_from = 1;
$key++;
$count_to = $key * self::$max;
} elseif ($has == self::$max) {
$count_from = $key * self::$max + 1;
$key++;
$count_to = $key * self::$max;
} else {
$count_from = $key * self::$max + 1;
$count_to = $key * self::$max + $has;
}
$spreadsheet = new Spreadsheet;
$spreadsheet->getProperties()->setTitle("$title-$count_from-$count_to");
$activeWorksheet = $spreadsheet->getActiveSheet();
$column = 1;
foreach ($value as $file) {
$data = file_get_contents($file);
$data = json_decode($data, true);
$json_string = json_encode($data, JSON_PRETTY_PRINT);
$arr = explode(PHP_EOL, $json_string);
$row = 0;
foreach ($arr as $string) {
$row++;
$replacedString = preg_replace('/\\\\u([0-9abcdef]{4})/', '&#x$1;', $string);
$utf8string = mb_convert_encoding($replacedString, 'UTF-8', 'HTML-ENTITIES');
$activeWorksheet->setCellValue([$column, $row], $utf8string);
}
$column++;
}
$spreadsheet->setActiveSheetIndex(0);
$writer = IOFactory::createWriter($spreadsheet, 'Xlsx');
$writer->save("$this->data_dir/$folder/$titel-$count_from-$count_to.xlsx");
}
}
protected function filter(array $files, array $types = ['json'])
{
$filtered = [];
foreach ($files as $file) {
$path_parts = pathinfo($file);
if (isset($path_parts['extension']) && in_array($path_parts['extension'], $types)) {
$filtered[] = $file;
}
}
return $filtered;
}
}