add daily no break grapper
This commit is contained in:
salam committed
1 parent
5188d1f171
commit
17de06cf6b
9 files changed
+325
-177
No files matched your search
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Traits\GrapSofascore;
|
||||
use App\Traits\Sofascore\GrapSofascore;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class Grapper extends Command
|
||||
@@ -11,10 +11,10 @@ class Grapper extends Command
|
||||
|
||||
protected $signature = 'grp:sofa {--d} {--r} {--f}';
|
||||
|
||||
protected $description = 'grap sofascore';
|
||||
protected $description = 'grap sofascore selecting';
|
||||
|
||||
public function handleUpdate(string $grapperSelected): void
|
||||
public function handleGrapper(): void
|
||||
{
|
||||
$this->runGrapper($grapperSelected);
|
||||
$this->runGrapper();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Traits\Sofascore\GrapSofascore;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class GrapperDaily extends Command
|
||||
{
|
||||
use GrapSofascore;
|
||||
|
||||
protected $signature = 'grp:daily {--d} {--r} {--f}';
|
||||
|
||||
protected $description = 'grap sofascore daily';
|
||||
|
||||
public function handleGrapper(): void
|
||||
{
|
||||
$this->runGrapper();
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Traits;
|
||||
|
||||
use App\Utilities\Notice;
|
||||
|
||||
use function Laravel\Prompts\select;
|
||||
|
||||
trait GrapSofascore
|
||||
{
|
||||
use GrapperEvent;
|
||||
use GrappingHelper;
|
||||
use GrappingMail;
|
||||
use GrappingToExcel;
|
||||
|
||||
protected int $debug;
|
||||
|
||||
protected int $restart;
|
||||
|
||||
protected int $force;
|
||||
|
||||
protected int $task_id;
|
||||
|
||||
protected array $daily_days = [1, 5];
|
||||
|
||||
protected string $base_dir = 'sofascore_data';
|
||||
|
||||
protected string $data_dir = '';
|
||||
|
||||
protected string $base_url = 'https://www.sofascore.com/';
|
||||
|
||||
protected array $tournaments_requested = [];
|
||||
|
||||
protected array $days = [];
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
error_reporting(E_ALL);
|
||||
|
||||
ini_set('display_errors', 1);
|
||||
|
||||
$this->debug = $this->options()['d'] ?? 0;
|
||||
|
||||
$this->restart = $this->options()['r'] ?? 0;
|
||||
|
||||
$this->force = $this->options()['f'] ?? 0;
|
||||
|
||||
$grapperSelected = $this->select_grapper();
|
||||
|
||||
$this->set_base_dir($grapperSelected);
|
||||
|
||||
switch ($grapperSelected) {
|
||||
case 'daily':
|
||||
$this->task_id = 1;
|
||||
$this->run_daily($grapperSelected);
|
||||
break;
|
||||
|
||||
case 'range':
|
||||
$this->task_id = 3;
|
||||
$this->run_range($grapperSelected);
|
||||
break;
|
||||
|
||||
case 'settings':
|
||||
$this->settings_to_local();
|
||||
break;
|
||||
case 'excel':
|
||||
$folder = $this->select_folder();
|
||||
|
||||
$this->import_to_excel($folder);
|
||||
break;
|
||||
case 'mail':
|
||||
$folder = $this->select_folder();
|
||||
|
||||
$this->send($folder);
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of run_daily
|
||||
*/
|
||||
protected function run_daily(string $grapperSelected)
|
||||
{
|
||||
$this->load_tournaments_requested($grapperSelected);
|
||||
|
||||
$this->load_days($grapperSelected);
|
||||
|
||||
foreach ($this->days as $day) {
|
||||
|
||||
$this->grap_tournaments_for_day($day);
|
||||
|
||||
foreach ($this->tournaments_requested as $tournament) {
|
||||
$this->get_event($day, $tournament);
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->debug) {
|
||||
Notice::note('Day grapping compleated', 'g');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of run_range
|
||||
*
|
||||
* @return never
|
||||
*/
|
||||
protected function run_range(string $grapperSelected)
|
||||
{
|
||||
$this->load_tournaments_requested($grapperSelected);
|
||||
|
||||
$this->load_days($grapperSelected);
|
||||
|
||||
dd($this->days);
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of select_grapper
|
||||
*/
|
||||
protected function select_grapper(): string
|
||||
{
|
||||
return select(
|
||||
label: 'What task should run?',
|
||||
scroll: 10,
|
||||
options: [
|
||||
'daily' => 'Daily',
|
||||
'range' => 'Range',
|
||||
'rose' => 'Rose',
|
||||
'player' => 'Player',
|
||||
'settings' => 'Settings',
|
||||
'excel' => 'Excel',
|
||||
'mail' => 'Send Mail',
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of select_folder
|
||||
*/
|
||||
protected function select_folder(): string
|
||||
{
|
||||
return select(
|
||||
label: 'What folder should be used?',
|
||||
scroll: 10,
|
||||
options: [
|
||||
'daily' => 'Daily',
|
||||
'range' => 'Range',
|
||||
'rose' => 'Rose',
|
||||
'player' => 'Player',
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Traits;
|
||||
namespace App\Traits\Sofascore;
|
||||
|
||||
trait CurlSofa
|
||||
{
|
||||
@@ -0,0 +1,250 @@
|
||||
<?php
|
||||
|
||||
namespace App\Traits\Sofascore;
|
||||
|
||||
use App\Utilities\Notice;
|
||||
use Illuminate\Support\Carbon;
|
||||
|
||||
use function Laravel\Prompts\select;
|
||||
use function Laravel\Prompts\text;
|
||||
|
||||
trait GrapSofascore
|
||||
{
|
||||
use GrapperEvent;
|
||||
use GrappingHelper;
|
||||
use GrappingMail;
|
||||
use GrappingToExcel;
|
||||
|
||||
protected int $debug;
|
||||
|
||||
protected int $restart;
|
||||
|
||||
protected int $force;
|
||||
|
||||
protected int $task_id;
|
||||
|
||||
protected int $days_back = 0;
|
||||
|
||||
protected string $base_dir = 'sofascore_data';
|
||||
|
||||
protected string $data_dir = '';
|
||||
|
||||
protected string $base_url = 'https://www.sofascore.com/';
|
||||
|
||||
protected array $tournaments_requested = [];
|
||||
|
||||
protected array $days = [];
|
||||
|
||||
public function handle(): void
|
||||
{
|
||||
error_reporting(E_ALL);
|
||||
|
||||
ini_set('display_errors', 1);
|
||||
|
||||
$this->debug = $this->options()['d'] ?? 0;
|
||||
|
||||
$this->restart = $this->options()['r'] ?? 0;
|
||||
|
||||
$this->force = $this->options()['f'] ?? 0;
|
||||
|
||||
switch ($this->getName()) {
|
||||
case 'grp:daily':
|
||||
$this->daily_grapper_nobreak();
|
||||
break;
|
||||
|
||||
default:
|
||||
// code...
|
||||
|
||||
$grapperSelected = $this->select_grapper();
|
||||
|
||||
$this->set_base_dir($grapperSelected);
|
||||
|
||||
switch ($grapperSelected) {
|
||||
case 'daily':
|
||||
$this->task_id = 1;
|
||||
|
||||
$this->days_back = (int) $this->select_day();
|
||||
|
||||
$this->run_daily($grapperSelected);
|
||||
break;
|
||||
|
||||
case 'range':
|
||||
$this->task_id = 3;
|
||||
$this->run_range($grapperSelected);
|
||||
break;
|
||||
|
||||
case 'settings':
|
||||
$this->settings_to_local();
|
||||
break;
|
||||
case 'excel':
|
||||
$folder = $this->select_folder();
|
||||
|
||||
$this->import_to_excel($folder);
|
||||
break;
|
||||
case 'mail':
|
||||
$folder = $this->select_folder();
|
||||
|
||||
$this->send($folder);
|
||||
break;
|
||||
case 'clear':
|
||||
$folder = $this->select_folder();
|
||||
|
||||
$this->clear_folder($folder);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of daily_grapper_nobreak
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
protected function daily_grapper_nobreak()
|
||||
{
|
||||
$folder = 'daily';
|
||||
|
||||
$this->task_id = 1;
|
||||
|
||||
$this->set_base_dir($folder);
|
||||
|
||||
$this->load_tournaments_requested($folder);
|
||||
|
||||
/**
|
||||
* day 1
|
||||
*/
|
||||
$this->clear_folder($folder, 'all');
|
||||
|
||||
$day = Carbon::now()->subDay()->format('Y-m-d');
|
||||
|
||||
$this->grap_tournaments_for_day($day);
|
||||
|
||||
foreach ($this->tournaments_requested as $tournament) {
|
||||
$this->get_event($day, $tournament);
|
||||
}
|
||||
|
||||
$this->import_to_excel($folder, "Daily-$day");
|
||||
|
||||
$this->clear_folder($folder, 'data');
|
||||
|
||||
/**
|
||||
* day 5
|
||||
*/
|
||||
$day = Carbon::now()->subDays(5)->format('Y-m-d');
|
||||
|
||||
$this->grap_tournaments_for_day($day);
|
||||
|
||||
foreach ($this->tournaments_requested as $tournament) {
|
||||
$this->get_event($day, $tournament);
|
||||
}
|
||||
|
||||
$this->import_to_excel($folder, "Daily-$day");
|
||||
|
||||
$this->send($folder);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of run_daily
|
||||
*/
|
||||
protected function run_daily(string $grapperSelected)
|
||||
{
|
||||
$this->load_tournaments_requested($grapperSelected);
|
||||
|
||||
$this->load_days($grapperSelected);
|
||||
|
||||
if ($this->confirm($this->days[0], true)) {
|
||||
|
||||
foreach ($this->days as $day) {
|
||||
|
||||
$this->grap_tournaments_for_day($day);
|
||||
|
||||
foreach ($this->tournaments_requested as $tournament) {
|
||||
$this->get_event($day, $tournament);
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->debug) {
|
||||
Notice::note('Day grapping compleated', 'g');
|
||||
}
|
||||
} else {
|
||||
Notice::note('day grapping cancelled', 'r');
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of run_range
|
||||
*
|
||||
* @return never
|
||||
*/
|
||||
protected function run_range(string $grapperSelected)
|
||||
{
|
||||
$this->load_tournaments_requested($grapperSelected);
|
||||
|
||||
$this->load_days($grapperSelected);
|
||||
|
||||
dd($this->days);
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of select_grapper
|
||||
*/
|
||||
protected function select_grapper(): string
|
||||
{
|
||||
return select(
|
||||
label: 'What task should run?',
|
||||
scroll: 10,
|
||||
options: [
|
||||
'daily' => 'Select day back',
|
||||
'range' => 'Range',
|
||||
'rose' => 'Rose',
|
||||
'player' => 'Player',
|
||||
'settings' => 'Settings',
|
||||
'excel' => 'Excel',
|
||||
'mail' => 'Send Mail',
|
||||
'clear' => 'Empty Folder',
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Summary of select_folder
|
||||
*/
|
||||
protected function select_folder(): string
|
||||
{
|
||||
return select(
|
||||
label: 'What folder should be used?',
|
||||
scroll: 10,
|
||||
options: [
|
||||
'daily' => 'Daily',
|
||||
'range' => 'Range',
|
||||
'rose' => 'Rose',
|
||||
'player' => 'Player',
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
protected function select_day(): string
|
||||
{
|
||||
return text(
|
||||
label: 'How many days back?',
|
||||
placeholder: 'z.B. 1',
|
||||
default: '1',
|
||||
);
|
||||
}
|
||||
|
||||
protected function select_what_delete(): string
|
||||
{
|
||||
return select(
|
||||
label: 'What files?',
|
||||
scroll: 10,
|
||||
options: [
|
||||
'all' => 'All',
|
||||
'data' => 'Data',
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Traits;
|
||||
namespace App\Traits\Sofascore;
|
||||
|
||||
use App\Models\SofaScoreDayTournament;
|
||||
use App\Models\SofaScoreEventGrapped;
|
||||
@@ -30,6 +30,8 @@ trait GrapperEvent
|
||||
Notice::note($url, 'bb');
|
||||
}
|
||||
|
||||
$this->sleep(1);
|
||||
|
||||
$cloudflareService = new CloudflareSolverService;
|
||||
|
||||
$data = $cloudflareService->getJsonFromUrl($url);
|
||||
@@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Traits;
|
||||
namespace App\Traits\Sofascore;
|
||||
|
||||
use App\Models\Country;
|
||||
use App\Models\SofaScoreCategory;
|
||||
@@ -35,9 +35,7 @@ trait GrappingHelper
|
||||
switch ($grapperSelected) {
|
||||
case 'daily':
|
||||
|
||||
foreach ($this->daily_days as $dayBack) {
|
||||
$this->days[] = Carbon::now()->subDays($dayBack)->format('Y-m-d');
|
||||
}
|
||||
$this->days[] = Carbon::now()->subDays($this->days_back)->format('Y-m-d');
|
||||
|
||||
break;
|
||||
case 'range':
|
||||
@@ -110,6 +108,7 @@ trait GrappingHelper
|
||||
break;
|
||||
case 'excel':
|
||||
case 'mail':
|
||||
case 'clear':
|
||||
$this->data_dir = storage_path($this->base_dir);
|
||||
break;
|
||||
}
|
||||
@@ -393,4 +392,32 @@ trait GrappingHelper
|
||||
|
||||
return preg_replace('/-+/', '-', $string); // Replaces multiple hyphens with single one.
|
||||
}
|
||||
|
||||
protected function clear_folder(string $folder, $what = '')
|
||||
{
|
||||
$mainFolder = storage_path($this->base_dir.'/'.$folder);
|
||||
|
||||
if (! $what) {
|
||||
$what = $this->select_what_delete();
|
||||
}
|
||||
switch ($what) {
|
||||
case 'all':
|
||||
if (File::exists($mainFolder)) {
|
||||
File::deleteDirectory($mainFolder);
|
||||
File::makeDirectory($mainFolder, 0755, true);
|
||||
Notice::note("folder $folder empty", 'bb');
|
||||
}
|
||||
break;
|
||||
|
||||
case 'data':
|
||||
if (File::exists($mainFolder)) {
|
||||
$subFolders = File::directories($mainFolder);
|
||||
foreach ($subFolders as $subfolder) {
|
||||
File::deleteDirectory($subfolder);
|
||||
}
|
||||
Notice::note("data folders from $folder removed", 'bb');
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Traits;
|
||||
namespace App\Traits\Sofascore;
|
||||
|
||||
use App\Mail\SofaScoreDataMail;
|
||||
use App\Utilities\Notice;
|
||||
@@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Traits;
|
||||
namespace App\Traits\Sofascore;
|
||||
|
||||
use App\Utilities\Notice;
|
||||
use Illuminate\Support\Carbon;
|
||||
@@ -11,7 +11,7 @@ trait GrappingToExcel
|
||||
{
|
||||
public static $max = 2000;
|
||||
|
||||
public function import_to_excel(string $folder)
|
||||
public function import_to_excel(string $folder, $title = '')
|
||||
{
|
||||
ini_set('memory_limit', -1);
|
||||
|
||||
@@ -19,7 +19,9 @@ trait GrappingToExcel
|
||||
Notice::note("import $folder to excel", 'g');
|
||||
}
|
||||
|
||||
$titel = ucfirst($folder).'_'.Carbon::now()->format('Y-m-d');
|
||||
if (! $title) {
|
||||
$titel = ucfirst($folder).'_'.Carbon::now()->format('Y-m-d');
|
||||
}
|
||||
|
||||
$files = $this->get_tree("$this->data_dir/$folder");
|
||||
|
||||
@@ -29,23 +31,23 @@ trait GrappingToExcel
|
||||
|
||||
$has = count($value);
|
||||
if (! $key && $has < self::$max) {
|
||||
$from = 1;
|
||||
$to = $has;
|
||||
$count_from = 1;
|
||||
$count_to = $has;
|
||||
} elseif (! $key) {
|
||||
$from = 1;
|
||||
$count_from = 1;
|
||||
$key++;
|
||||
$to = $key * self::$max;
|
||||
$count_to = $key * self::$max;
|
||||
} elseif ($has == self::$max) {
|
||||
$from = $key * self::$max + 1;
|
||||
$count_from = $key * self::$max + 1;
|
||||
$key++;
|
||||
$to = $key * self::$max;
|
||||
$count_to = $key * self::$max;
|
||||
} else {
|
||||
$from = $key * self::$max + 1;
|
||||
$to = $key * self::$max + $has;
|
||||
$count_from = $key * self::$max + 1;
|
||||
$count_to = $key * self::$max + $has;
|
||||
}
|
||||
|
||||
$spreadsheet = new Spreadsheet;
|
||||
$spreadsheet->getProperties()->setTitle("$titel-$from-$to");
|
||||
$spreadsheet->getProperties()->setTitle("$title-$count_from-$count_to");
|
||||
$activeWorksheet = $spreadsheet->getActiveSheet();
|
||||
|
||||
$column = 1;
|
||||
@@ -67,7 +69,7 @@ trait GrappingToExcel
|
||||
|
||||
$spreadsheet->setActiveSheetIndex(0);
|
||||
$writer = IOFactory::createWriter($spreadsheet, 'Xlsx');
|
||||
$writer->save("$this->data_dir/$folder/$titel-$from-$to.xlsx");
|
||||
$writer->save("$this->data_dir/$folder/$titel-$count_from-$count_to.xlsx");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user