wip import
This commit is contained in:
salam committed
1 parent
048989fed8
commit
4fa4e1a611
11 files changed
+48902
-141
No files matched your search
@@ -4,11 +4,10 @@ namespace App\Filament\Imports;
|
||||
|
||||
use App\Models\StatistikoPlayer;
|
||||
use Filament\Notifications\Notification;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\HtmlString;
|
||||
use Illuminate\Support\LazyCollection;
|
||||
use Illuminate\Support\Str;
|
||||
use Spatie\SimpleExcel\SimpleExcelReader;
|
||||
|
||||
class StatistikoImporter
|
||||
@@ -21,25 +20,29 @@ class StatistikoImporter
|
||||
|
||||
public static int $startQueries;
|
||||
|
||||
public static function run($mode, $record): void
|
||||
public static function run($record): void
|
||||
{
|
||||
error_reporting(E_ALL);
|
||||
|
||||
ini_set('display_errors', 1);
|
||||
ini_set('max_execution_time', -1);
|
||||
|
||||
$table = $record['data_type'];
|
||||
|
||||
$filePath = storage_path().'/app/private/'.$record['file_name'];
|
||||
$filePath = $record['file_name'];
|
||||
|
||||
self::startBenchmark($table);
|
||||
|
||||
try {
|
||||
self::handleImport($filePath, $table, $mode);
|
||||
} catch (\Exception $e) {
|
||||
dd(get_class($e).' '.Str::of($e->getMessage())->limit(100)->value());
|
||||
switch ($record['data_type']) {
|
||||
case 'player':
|
||||
self::importPlayer($filePath);
|
||||
break;
|
||||
case 'events':
|
||||
self::importEvents($filePath);
|
||||
break;
|
||||
case 'events_player':
|
||||
break;
|
||||
}
|
||||
|
||||
self::endBenchmark();
|
||||
self::endBenchmark($table);
|
||||
}
|
||||
|
||||
protected static function startBenchmark(string $table): void
|
||||
@@ -50,7 +53,7 @@ class StatistikoImporter
|
||||
DB::connection('mongodb')->enableQueryLog();
|
||||
}
|
||||
|
||||
protected static function endBenchmark(string $table = 'customers'): void
|
||||
protected static function endBenchmark(string $table): void
|
||||
{
|
||||
$executionTime = microtime(true) - self::$benchmarkStartTime;
|
||||
$memoryUsage = round((memory_get_usage() - self::$benchmarkStartMemory) / 1024 / 1024, 2);
|
||||
@@ -73,6 +76,7 @@ class StatistikoImporter
|
||||
$str .= '<br/>ROWS: '.number_format($rowDiff);
|
||||
|
||||
$recipient = auth()->user();
|
||||
|
||||
Notification::make()
|
||||
->success()
|
||||
->title('Import compleated')
|
||||
@@ -80,110 +84,95 @@ class StatistikoImporter
|
||||
->sendToDatabase($recipient);
|
||||
}
|
||||
|
||||
private static function handleImport(string $filePath, $table, $mode): void
|
||||
private static function importPlayer(string $filePath): void
|
||||
{
|
||||
$now = now()->format('Y-m-d H:i:s');
|
||||
|
||||
$chunkSize = 1000;
|
||||
|
||||
SimpleExcelReader::create($filePath)
|
||||
SimpleExcelReader::create(storage_path('app/private/'.$filePath))
|
||||
->getRows()
|
||||
->chunk($chunkSize)
|
||||
->each(function ($rowProperties) {
|
||||
dd($rowProperties);
|
||||
// processing rows
|
||||
->chunk(1000)
|
||||
->each(function (LazyCollection $chunk) use ($now) {
|
||||
|
||||
$data = [];
|
||||
|
||||
foreach ($chunk as $row) {
|
||||
|
||||
$row = array_values($row);
|
||||
|
||||
// check if name exists
|
||||
if (is_string(preg_replace("'\W'", '', $row[1]))) {
|
||||
$data[] = [
|
||||
'player_id' => $row[0],
|
||||
'name' => $row[1],
|
||||
'birthdate' => Carbon::parse($row[2])->toDateString(),
|
||||
'height' => $row[3],
|
||||
'nationality' => $row[4],
|
||||
'position' => $row[5],
|
||||
'sport_radar_id' => $row[6],
|
||||
'id_squadra_tm' => $row[7],
|
||||
'full_name' => $row[8],
|
||||
'no_specialities' => $row[9],
|
||||
'rigorist' => $row[10],
|
||||
'national_team_id' => $row[11],
|
||||
'last_update' => Carbon::parse($row[12])->toDateString(),
|
||||
'league_code' => $row[13],
|
||||
'team_code' => $row[14],
|
||||
'team_name' => $row[15],
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
];
|
||||
}
|
||||
}
|
||||
StatistikoPlayer::upsert($data, 'player_id');
|
||||
});
|
||||
|
||||
// LazyCollection::make(function () use ($filePath) {
|
||||
// $handle = fopen($filePath, 'r');
|
||||
// fgets($handle); // skip header
|
||||
|
||||
// while (($line = fgets($handle)) !== false) {
|
||||
// yield str_getcsv($line);
|
||||
// }
|
||||
// fclose($handle);
|
||||
// })->each(function ($row) {
|
||||
// dd($row);
|
||||
// });
|
||||
// ->map(fn ($row) => [
|
||||
// 'custom_id' => $row[0],
|
||||
// 'name' => $row[1],
|
||||
// 'email' => $row[2],
|
||||
// 'company' => $row[3],
|
||||
// 'city' => $row[4],
|
||||
// 'country' => $row[5],
|
||||
// 'birthday' => $row[6],
|
||||
// 'created_at' => $now,
|
||||
// 'updated_at' => $now,
|
||||
// ])
|
||||
// ->chunk($chunkSize)
|
||||
// ->each(fn ($chunk) => $result = StatistikoPlayer::upsert(
|
||||
// [
|
||||
// ['title' => 'Inspector Maigret', 'recommended' => false, 'runtime' => 128],
|
||||
// ['title' => 'Petit Maman', 'recommended' => true, 'runtime' => 72],
|
||||
// ],
|
||||
// 'title',
|
||||
// 'recommended',
|
||||
// ));
|
||||
}
|
||||
|
||||
private static function import11Concurrent(string $filePath): void
|
||||
private static function importEvents(string $filePath): void
|
||||
{
|
||||
// 100 168ms
|
||||
// 1K 172ms
|
||||
// 10K 234ms
|
||||
// 100K 595ms
|
||||
// 1M 4.36s
|
||||
// 2M 8.8s
|
||||
$now = now()->format('Y-m-d H:i:s');
|
||||
$numberOfProcesses = 10;
|
||||
|
||||
$tasks = [];
|
||||
for ($i = 0; $i < $numberOfProcesses; $i++) {
|
||||
$tasks[] = function () use ($filePath, $i, $numberOfProcesses, $now) {
|
||||
DB::reconnect();
|
||||
SimpleExcelReader::create(storage_path('app/private/'.$filePath))
|
||||
->getRows()
|
||||
->chunk(1000)
|
||||
->each(function (LazyCollection $chunk) use ($now) {
|
||||
|
||||
$handle = fopen($filePath, 'r');
|
||||
fgets($handle); // Skip header
|
||||
$currentLine = 0;
|
||||
$customers = [];
|
||||
$data = [];
|
||||
|
||||
while (($line = fgets($handle)) !== false) {
|
||||
// Each process takes every Nth line
|
||||
if ($currentLine++ % $numberOfProcesses !== $i) {
|
||||
continue;
|
||||
}
|
||||
foreach ($chunk as $row) {
|
||||
|
||||
$row = str_getcsv($line);
|
||||
$customers[] = [
|
||||
'custom_id' => $row[0],
|
||||
'name' => $row[1],
|
||||
'email' => $row[2],
|
||||
'company' => $row[3],
|
||||
'city' => $row[4],
|
||||
'country' => $row[5],
|
||||
'birthday' => $row[6],
|
||||
$row = array_values($row);
|
||||
dd($row);
|
||||
|
||||
$data[] = [
|
||||
'sport_id' => 1,
|
||||
'league_id' => $row[1],
|
||||
'home_id' => $row[2],
|
||||
'away_id' => $row[3],
|
||||
'starttime' => Carbon::parse($row[4])->toDateTimeString(),
|
||||
'season' => $row[5],
|
||||
'gls_h' => $row[6],
|
||||
'gls_a' => $row[7],
|
||||
'crn_h' => $row[8],
|
||||
'fls_h' => $row[9],
|
||||
'ofs_h' => $row[10],
|
||||
'snt_h' => $row[11],
|
||||
'tsh_h' => $row[12],
|
||||
'crd_h' => $row[13],
|
||||
'crn_a' => $row[14],
|
||||
'fls_a' => $row[15],
|
||||
'ofs_a' => $row[16],
|
||||
'snt_a' => $row[17],
|
||||
'tsh_a' => $row[18],
|
||||
'crd_a' => $row[19],
|
||||
'expgoalshome_value' => $row[20],
|
||||
'expgoalsaway_value' => $row[21],
|
||||
'exp_handicap' => $row[22],
|
||||
'exp_total' => $row[23],
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
];
|
||||
|
||||
if (count($customers) === 1000) {
|
||||
DB::table('customers')->insert($customers);
|
||||
$customers = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($customers)) {
|
||||
DB::table('customers')->insert($customers);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
|
||||
return true;
|
||||
};
|
||||
}
|
||||
|
||||
Concurrency::run($tasks);
|
||||
|
||||
StatistikoPlayer::upsert($data, 'player_id');
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user