drkthunder02/w4rpservices
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added jobs back in for the fetching and processing the mining ledgers

Browse Source 
those jobs were taking extremely long times to complete, and jobs are better suited for them.
This commit is contained in:
drkthunder02
2021-03-15 19:38:06 +09:00
parent 4c281934ff
commit 0ee3699b44
6 changed files with 238 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
app/Console/Commands/MiningTaxes/MiningTaxesDataCleanup.php Normal file
View File

@@ -0,0 +1,61 @@
<?php
namespace App\Console\Commands\MiningTaxes;
use Illuminate\Console\Command;
use Log;
use Carbon\Carbon;
//Application Library
use Commands\Library\CommandHelper;
//Models
use app\Models\MiningTax\Ledger;
class MiningTaxesDataCleanup extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'MiningTaxes:CleanupData';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Cleanup old ledger entries for mining taxes.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
//Create the command helper container
$task = new CommandHelper('MiningTaxesDataCleanup');
//Set the task as started
$task->SetStartStatus();
//Clean up old data
Ledger::where(['updated_at', '<', Carbon::now()->subDays(120)])->delete();
//Set the task as finished
$task->SetStopStatus();
return 0;
}
}

85
app/Console/Commands/MiningTaxes/MiningTaxesLedgers.php
View File

@@ -9,18 +9,9 @@ use Carbon\Carbon;
//Application Library
use Commands\Library\CommandHelper;
use Seat\Eseye\Exceptions\RequestFailedException;
use App\Library\Esi\Esi;
use App\Library\Helpers\LookupHelper;
use App\Library\Moons\MoonCalc;
//Models
use App\Models\MiningTax\Observer;
use App\Models\MiningTax\Ledger;
use App\Models\Moon\MineralPrice;
use App\Models\Moon\ItemComposition;
use App\Models\Esi\EsiToken;
use App\Models\Esi\EsiScope;
//Jobs
//use App\Jobs\Commands\MiningTaxes\FetchMiningTaxesLedgersJob;
@@ -63,88 +54,20 @@ class MiningTaxesLedgers extends Command
//Set the task as started
$task->SetStartStatus();
//Get the current time to mark when the process started
$startTime = time();
//Get the site configuration which holds some data we need
$config = config('esi');
//Get the observers from the database
$observers = Observer::all();
//Job Variables to be moved later
$esiHelper = new Esi;
$lookup = new LookupHelper;
$mHelper = new MoonCalc;
$esiHelper = new Esi;
/*
//For each of the observers, send a job to fetch the mining ledger
foreach($observers as $obs) {
//Dispatch the mining taxes ledger jobs
FetchMiningTaxesLedgersJob::dispatch($config['primary'], $config['corporation'], $obs->observer_id)->onQueue('miningtaxes');
}
*/
$refreshToken = $esiHelper->GetRefreshToken($config['primary']);
$esi = $esiHelper->SetupEsiAuthentication($refreshToken);
foreach($observers as $obs) {
$startObserverTime = time();
try {
$response = $esi->invoke('get', '/corporation/{corporation_id}/mining/observers/{observer_id}/', [
'corporation_id' => $config['corporation'],
'observer_id' => $obs->observer_id,
]);
} catch(RequestFailedException $e) {
Log::warning('Failed to get the mining ledger in FetchMiningTaxesLedgersCommand for observer id: ' . $this->observerId);
return null;
}
$ledgers = json_decode($response->raw);
foreach($ledgers as $ledger) {
$startLedgerTime = time();
//Get some basic information we need to work with
$charName = $lookup->CharacterIdToName($ledger->character_id);
//Get the type name from the ledger ore stuff
$typeName = $lookup->ItemIdToName($ledger->type_id);
//Get the price from the helper function
$price = $mHelper->CalculateOrePrice($ledger->type_id);
//Calculate the total price based on the amount
$amount = $price * $ledger->quantity;
//Insert or update the entry in the database
$item = Ledger::updateOrCreate([
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $obs->observer_id,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'amount' => $amount,
], [
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $obs->observer_id,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'amount' => $amount,
]);
printf("Current cycle time for this ledger entry is: " . (time() - $startLedgerTime) . "s.\r\n");
}
printf("Current time taken is: " . (time() - $startObserverTime) . "s.\r\n");
}
//Clean up old data
//Ledger::where(['updated_at', '<', Carbon::now()->subDays(120)])->delete();
printf("Total time taken is: " . (time() - $startTime) . "s.\r\n");
//Set the task as finished
$task->SetStopStatus();
//Return 0
return 0;
}

5
app/Console/Commands/MiningTaxes/MiningTaxesObservers.php
View File

@@ -107,9 +107,10 @@ class MiningTaxesObservers extends Command
$date = Carbon::now()->subDays(60);
Observer::where('last_updated', '<', $date)->delete();
//Set the task as completed
$task->SetStopStatus();
//Set the end time for debugging and printint out to the screen
$endTime = time();
printf("Time to complete: " . ($endTime - $startTime) . "\n\r");

9
app/Console/Kernel.php
View File

@@ -36,6 +36,7 @@ class Kernel extends ConsoleKernel
Commands\MiningTaxes\MiningTaxesLedgers::class,
Commands\MiningTaxes\MiningTaxesObservers::class,
Commands\MiningTaxes\MiningTaxesPayments::class,
Commands\MiningTaxes\MiningTaxesDataCleanup::class,
];
/**
@@ -52,7 +53,8 @@ class Kernel extends ConsoleKernel
* Purge Data Schedule
*/
$schedule->command('data:CleanData')
->weekly(7, '11:00');
->weekly(7, '11:00')
->withoutOverlapping();
$schedule->command('data:PurgeCorpLedgers')
->monthly();
$schedule->command('data:PurgeUsers')
@@ -86,7 +88,10 @@ class Kernel extends ConsoleKernel
->withoutOverlapping();
$schedule->command('MiningTaxes:Payments')
->hourlyAt('15')
->withoutOverlapping();
->withoutOverlapping();
$schedule->command('MiningTaxes:CleanupData')
->weekly(7, '11:15')
->withoutOverlapping();
}
/**

119
app/Jobs/Commands/MiningTaxes/FetchMiningTaxesLedgersJob.php
View File

@@ -16,6 +16,9 @@ use App\Library\Esi\Esi;
use App\Library\Helpers\LookupHelper;
use App\Library\Moons\MoonCalc;
//Jobs
use App\Jobs\Commands\MiningTaxes\ProcessMiningTaxesLedgersJob;
//App Models
use App\Models\MiningTax\Observer;
use App\Models\MiningTax\Ledger;
@@ -48,7 +51,6 @@ class FetchMiningTaxesLedgersJob implements ShouldQueue
private $charId;
private $corpId;
private $observerId;
private $esi;
/**
* Create a new job instance.
@@ -64,17 +66,6 @@ class FetchMiningTaxesLedgersJob implements ShouldQueue
$this->charId = $charId;
$this->corpId = $corpId;
$this->observerId = $observerId;
//Setup the ESI stuff
$esiHelper = new Esi;
//Setup the private esi variables
if(!$esiHelper->haveEsiScope($this->charId, 'esi-industry.read_corporation_mining.v1')) {
Log::critical('Character: ' . $this->charId . ' did not have the correct esi scope in FetchMiningTaxesLedgersJob.');
return null;
}
$refreshToken = $esiHelper->GetRefreshToken($this->charId);
$this->esi = $esiHelper->SetupEsiAuthentication($refreshToken);
}
/**
@@ -88,55 +79,75 @@ class FetchMiningTaxesLedgersJob implements ShouldQueue
$lookup = new LookupHelper;
$mHelper = new MoonCalc;
$esiHelper = new Esi;
$pageFailed = false;
//Get the ledger from ESI
try {
$response = $this->esi->invoke('get', '/corporation/{corporation_id}/mining/observers/{observer_id}/', [
'corporation_id' => $this->corpId,
'observer_id' => $this->observerId,
]);
} catch(RequestFailedException $e) {
Log::warning('Failed to get the mining ledger in FetchMiningTaxesLedgersJob for observer id: ' . $this->observerId);
//Check for the correct scope
if(!$esiHelper->haveEsiScope($this->charId, 'esi-industry.read_corporation_mining.v1')) {
Log::critical('Character: ' . $this->charId . ' did not have the correct esi scope in FetchMiningTaxesLedgersJob.');
return null;
}
$ledgers = json_decode($response);
//Get the esi token in order to pull data from esi
$refreshToken = $esiHelper->GetRefreshToken($this->charId);
$esi = $esiHelper->SetupEsiAuthentication($refreshToken);
//Sort through the array, and create the variables needed for database entries
foreach($ledgers as $ledger) {
//Get some basic information we need to work with
$charName = $lookup->CharacterIdToName($ledger->character_id);
//Get the type name from the ledger ore stuff
$typeName = $lookup->ItemIdToName($ledger->type_id);
//Decode the date and store it.
//$updated = $esiHelper->DecodeDate($ledger->last_updated);
//Set the current page
$currentPage = 1;
$totalPages = 1;
$price = $mHelper->CalculateOrePrice($ledger->type_id);
$amount = $price * $ledger->quantity;
//Setup a do-while loop to sort through the ledgers by pages
do {
/**
* During the course of the operation, we want to ensure our token stays valid.
* If the token, expires, then we want to refresh the token through the esi helper
* library functionality.
*/
if($esiHelper->TokenExpired($refreshToken)) {
$refreshToken = $esiHelper->GetRefreshToken($charId);
$esi = $esiHelper->SetupEsiAuthentication($refreshToken);
}
//Insert or update the entry in the database
$item = Ledger::updateOrCreate([
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $this->observerId,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'price' => $amount,
], [
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $this->observerId,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'price' => $amount,
]);
}
/**
* Attempt to get the data from the esi api. If it fails, we skip the page
*/
try {
$response = $esi->page($currentPage)
->invoke('get', '/corporation/{corporation_id}/mining/observers/{observer_id}/', [
'corporation_id' => $config['corporation'],
'observer_id' => $obs->observer_id,
]);
} catch(RequestFailedException $e) {
Log::warning('Failed to get the mining ledger in FetchMiningTaxesLedgersCommand for observer id: ' . $this->observerId);
$pageFailed = true;
}
//Clean up old data
Ledger::where(['updated_at', '<', Carbon::now()->subDays(120)])->delete();
/**
* If the current page is the first one and the page didn't fail, then update the total pages.
* If the first page failed, just return as we aren't going to be able to get the total amount of data needed.
*/
if($currentPage == 1 && $pageFailed == false) {
$totalPages = $response->pages;
} else if($currentPage == 1 && $pageFailed == true) {
return null;
}
if($pageFailed == true) {
//If the page failed, then reset the variable, and skip the current iteration
//of creating the jobs.
$pageFailed = false;
} else {
//Decode the json response from the ledgers
$ledgers = json_decode($response->raw);
//Dispatch jobs to process each of the mining ledger entries
foreach($ledgers as $ledger) {
ProcessMiningTaxesLedgersJob::dispatch($ledger)->onQueue('miningtaxes');
}
}
//Increment the current pages
$currentPage++;
} while($currentPage <= $totalPages);
}
}

98
app/Jobs/Commands/MiningTaxes/ProcessMiningTaxesLedgersJob.php Normal file
View File

@@ -0,0 +1,98 @@
<?php
namespace App\Jobs\Commands\MiningTaxes;
//Internal Library
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
//App Library
use App\Library\Helpers\LookupHelper;
use App\Library\Moons\MoonCalc;
//Models
use App\Models\MiningTax\Ledger;
use App\Models\Moon\MineralPrice;
use App\Models\Moon\ItemComposition;
class ProcessMiningTaxesLedgersJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Timeout in seconds
*
* @var int
*/
public $timeout = 3600;
/**
* Number of job retries
*
* @var int
*/
public $tries = 3;
/**
* Job Variables
*/
private $ledger;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($ledger)
{
//Set the connection for the job
$this->connection = 'redis';
//Import variables from the calling function
$this->ledger = $ledger;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$lookup = new LookupHelper;
$mHelper = new MoonCalc;
//Get some of the basic information we need to work with
$charName = $lookup->CharacterIdToName($this->ledger->character_id);
//Get the type name from the ledger ore
$typeName = $lookup->ItemIdToName($this->ledger->type_id);
//Get the price from the helper function
$price = $mHelper->CalculateOrePrice($this->ledger->type_id);
//Calculate the total price based on the amount
$amount = $price * $this->ledger->quantity;
//Insert or update the entry in the database
$item = Ledger::updateOrCreate([
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $obs->observer_id,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'amount' => $amount,
], [
'character_id' => $ledger->character_id,
'character_name' => $charName,
'observer_id' => $obs->observer_id,
'last_updated' => $ledger->last_updated,
'type_id' => $ledger->type_id,
'ore_name' => $typeName,
'quantity' => $ledger->quantity,
'amount' => $amount,
]);
}
}