Commit 95745322 by Vladislav

Merge branch 'master' into ticket_19472

2 parents baafb13c 98739732
......@@ -43,15 +43,11 @@ class CampaignsAdd extends Command
public function handle()
{
$tokens = Tokens::whereHas('dictionaryCampaignsEnabledForNotExternal')
->with([
'dictionaryCampaignsEnabledForNotExternal' => function (HasManyThrough $query) {
return $query->limit(10);
},
'dictionaryCampaignsEnabledForNotExternal.campaign'
])
->with('dictionaryCampaignsEnabledForNotExternal.campaign')
->where('type', '!=', Tokens::MAIN)
->get();
foreach ($tokens as $token) {
$factory = APIRequest::getInstance(API::YANDEX);
$factory->setToken($token);
......
......@@ -63,11 +63,12 @@ class CampaignsResume extends Command
*/
$tokens = Tokens::whereHas('dictionaryCampaignsEnabledForExternalDisabled')
->with([
'dictionaryCampaignsEnabledForExternalDisabled' => function (HasManyThrough $query) {
return $query->limit(1000);
},
])->where('type', '!=', Tokens::MAIN)
// ->with([
// 'dictionaryCampaignsEnabledForExternalDisabled' => function (HasManyThrough $query) {
// return $query->limit(1000);
// },
// ])
->where('type', '!=', Tokens::MAIN)
->get();
foreach ($tokens as $token) {
......
......@@ -62,11 +62,12 @@ class CampaignsSuspend extends Command
*/
$tokens = Tokens::whereHas('dictionaryCampaignsNotEnabledForExternalNotDisabled')
->with([
'dictionaryCampaignsNotEnabledForExternalNotDisabled' => function (HasManyThrough $query) {
return $query->limit(1000);
},
])->where('type', '!=', Tokens::MAIN)
// ->with([
// 'dictionaryCampaignsNotEnabledForExternalNotDisabled' => function (HasManyThrough $query) {
// return $query->limit(1000);
// },
// ])
->where('type', '!=', Tokens::MAIN)
->get();
foreach ($tokens as $token) {
......
......@@ -43,12 +43,13 @@ class CampaignsUpdate extends Command
public function handle()
{
$tokens = Tokens::whereHas('dictionaryCampaignsEnabledForExternalNeedUpdated')
->with([
'dictionaryCampaignsEnabledForExternalNeedUpdated' => function (HasManyThrough $query) {
return $query->limit(10);
},
'dictionaryCampaignsEnabledForExternalNeedUpdated.campaign'
])->where('type', '!=', Tokens::MAIN)
// ->with([
// 'dictionaryCampaignsEnabledForExternalNeedUpdated' => function (HasManyThrough $query) {
// return $query->limit(10);
// },
// 'dictionaryCampaignsEnabledForExternalNeedUpdated.campaign'
// ])
->where('type', '!=', Tokens::MAIN)
->get();
foreach ($tokens as $token) {
......
......@@ -12,6 +12,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
class ProcessCallAPI implements ShouldQueue
{
......@@ -53,11 +54,17 @@ class ProcessCallAPI implements ShouldQueue
if ($maxObjects and $maxObjects !== $limits::NAN && $this->api->getObjectsCount() > $maxObjects) {
//те на которые не хватило баллов помещаем в очередь
if ($apiR = $this->api->slice($maxObjects)) {
dispatch(new ProcessCallLimitedAPI($apiR));
dispatch(new ProcessCallLimitedAPI($apiR))->onQueue('limits');
}
}
$response = $api->execute();
$limits->acceptRezerv($this->limitId, new HeaderLimits($response->headers()));
try{
$limits->acceptRezerv($this->limitId, new HeaderLimits($response->headers()));
}catch(\Exception $e){
Log::debug($e);
Log::debug($response->headers());
}
//TODO: обработать результат.
// если не хватило баллов на все что хотели запросить, то в очередь отправляем новый запрос на получение новых данных
AdsHandler::getInstance($this->api)->handle($response);
......
......@@ -75,6 +75,6 @@ class ProcessCallLimitedAPI implements ShouldQueue//, ShouldBeUnique
private function reRunHour(){
$process = new ProcessCallLimitedAPI($this->api);
dispatch( $process )
->delay(now()->addMinutes(60-date("i")))->onQueue('limits');
->delay(now()->addMinutes(60-date("i")));
}
}
......@@ -20,5 +20,6 @@ interface APIRequest{
function call($params = null);
function handle($response);
function getObjectsCount();//возвращаем сколько объектов обрабатывается запросом. Для выборок, максимальное что может вернуться.
function getObjectsCount();
function getMaxCount();
}
......@@ -64,18 +64,28 @@ class Limits implements \App\Service\Contract\Limits {
function countObjectsLimit(\App\Service\Contract\APIRequest $request): int
{
$cost = $this->limitCosts->getCostObject($request);
$maxCount = $request->getMaxCount();
if ($this->limitCosts->getCostCall($request) > $this->current()){
return 0;
}
if ($cost==0){
if ($cost == 0 || $maxCount === self::NAN){
return self::NAN;
}
$maxCount = floor(($this->current() - $this->limitCosts->getCostCall($request)) / $cost);
$objects = $request->getObjectsCount();
if ($objects > $maxCount){
$objects = $maxCount;
$allowCount = floor(($this->current() - $this->limitCosts->getCostCall($request)) / $cost);
$objectsCount = $request->getObjectsCount();
if ($objectsCount > $maxCount) {
$objectsCount = $maxCount;
}
return $objects;
if ($objectsCount > $allowCount){
$objectsCount = $allowCount;
}
return $objectsCount;
}
/**
......@@ -182,6 +192,10 @@ class Limits implements \App\Service\Contract\Limits {
private function getSpent($objects, \App\Service\Contract\APIRequest $request): int
{
$cost = $this->limitCosts->getCostObject($request);
if ($objects === Limits::NAN)
return $this->limitCosts->getCostCall($request);
return $objects * $cost + $this->limitCosts->getCostCall($request);
}
}
......@@ -83,32 +83,17 @@ class APIRequest implements \App\Service\Contract\APIRequest
$replication = clone $this;
if (is_array($key)) {
$params = $this->changeInArray($params, $key, $maxObjects);
} else {
$params_by_key = $params[$key];
$params[$key] = $this->sliceArray($params_by_key, $maxObjects);
if (!is_array($key)) {
$key = [$key];
}
$replication->setParams($params);
$replication->setParams($this->changeInArray($params, $key, $maxObjects));
if (is_array($key)) {
$params = $this->changeInArray($this->getParams(), $key, 0, $maxObjects);
} else {
$params_by_key = $params[$key];
$params[$key] = $this->sliceArray($params_by_key, 0, $maxObjects);
}
$this->setParams($params);
$this->setParams($this->changeInArray($params, $key, 0, $maxObjects));
return $replication;
}
private function sliceArray($params, $offset, $length = null): array
{
return array_values(array_slice($params, $offset, $length));
}
private function changeInArray($params, $key, $offset, $maxObjects = null)
{
$params_i = &$params;
......@@ -120,14 +105,19 @@ class APIRequest implements \App\Service\Contract\APIRequest
return $params;
}
private function sliceArray($params, $offset, $length = null): array
{
return array_values(array_slice($params, $offset, $length));
}
function slice($count): ?\App\Service\Contract\APIRequest
{
throw new Exception('Я не знаю формата запрсов, чтобы его разбить');
throw new \Exception('Я не знаю формата запрсов, чтобы его разбить');
}
function next($count)
{
throw new Exception('Я не знаю формата запрсов, чтобы его разбить');
throw new \Exception('Я не знаю формата запрсов, чтобы его разбить');
}
function call($params = null)
......@@ -142,6 +132,11 @@ class APIRequest implements \App\Service\Contract\APIRequest
function getObjectsCount()
{
throw new Exception('Не задано сколкьо бъектов обрабатывается');
throw new \Exception('Не задано сколько объектов обрабатывается');
}
function getMaxCount()
{
throw new \Exception('Не задано сколько максимум объектов необходимо обработатывать');
}
}
......@@ -12,10 +12,11 @@ use App\Service\StrReplaceByVariables;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Facades\Log;
use function MongoDB\Driver\Monitoring\removeSubscriber;
class AddCampaigns extends DirectRequest
{
CONST MAX_COUNT = 10;
protected $max_count = 10;
protected $timestamp;
/* @var Collection|DictionaryCampaign[] $dictionaryCampaigns */
......@@ -32,10 +33,7 @@ class AddCampaigns extends DirectRequest
public function getObjectsCount()
{
$params = $this->getParams();
$cnt = count($params['Campaigns']);
return $cnt > self::MAX_COUNT ? self::MAX_COUNT : $cnt;
return count($this->getParams()['Campaigns']);
}
public function slice($maxObjects): ?APIRequest
......@@ -85,6 +83,7 @@ class AddCampaigns extends DirectRequest
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
......
......@@ -8,12 +8,13 @@ use App\Models\Pivots\DictionaryCampaign;
use App\Service\Requests\DirectRequest;
use Carbon\Carbon;
class CheckCampaignsChange extends DirectRequest
class CheckCampaignsChanges extends DirectRequest
{
CONST SELF = 'SELF';
CONST CHILDREN = 'CHILDREN';
CONST STAT = 'STAT';
protected $max_count = -1;
protected $next;
function call($params = null)
......
......@@ -14,6 +14,8 @@ use Illuminate\Support\Facades\Log;
class CheckChanges extends DirectRequest
{
protected $max_count = -1;
public function call($params = null)
{
$this->requestPrepare($params);
......
<?php
namespace App\Service\Requests\Direct;
use App\Jobs\ProcessCallLimitedAPI;
use App\Service\Requests\DirectRequest;
class CheckDictionariesChanges extends DirectRequest {
function call($params = null){
class CheckDictionariesChanges extends DirectRequest
{
protected $max_count = -1;
function call($params = null)
{
$this->requestPrepare($params);
$process = new ProcessCallLimitedAPI($this);
dispatch($process)->onQueue('limits');
}
function handle($response){
function handle($response)
{
}
private function requestPrepare($params){
private function requestPrepare($params)
{
$this->setService('Changes');
$this->setMethod('checkDictionaries');
}
......
......@@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Log;
class GetAdGroups extends DirectRequest
{
protected $max_count = 10000;
function call($params = null)
{
......@@ -98,6 +99,7 @@ class GetAdGroups extends DirectRequest
} catch (\Exception $e) {
DB::rollBack();
Log::debug($e);
throw $e;
}
}
......
......@@ -10,6 +10,7 @@ use Illuminate\Support\Facades\Log;
class GetCampaigns extends DirectRequest
{
protected $max_count = 10000;
protected $timestamp;
public function call($params = null)
......@@ -22,10 +23,10 @@ class GetCampaigns extends DirectRequest
public function getObjectsCount()
{
$params = $this->getParams();
if (isset($params['SelectionCriteria']['Ids']) && count($params['SelectionCriteria']['Ids']) < self::MAX_COUNT){
if (isset($params['SelectionCriteria']['Ids'])){
return count($params['SelectionCriteria']['Ids']);
}
return self::MAX_COUNT;
return parent::getObjectsCount();
}
public function handle($response)
......@@ -76,6 +77,7 @@ class GetCampaigns extends DirectRequest
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
......
......@@ -9,6 +9,8 @@ use Illuminate\Support\Facades\Log;
class GetDictionaries extends DirectRequest
{
protected $max_count = 10000;
function call($params = null)
{
$this->requestPrepare($params);
......@@ -41,6 +43,7 @@ class GetDictionaries extends DirectRequest
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
......
......@@ -11,7 +11,7 @@ use Illuminate\Support\Facades\Log;
class ResumeCampaigns extends DirectRequest
{
CONST MAX_COUNT = 1000;
protected $max_count = 1000;
public function call($params = null)
{
......@@ -22,10 +22,7 @@ class ResumeCampaigns extends DirectRequest
public function getObjectsCount()
{
$params = $this->getParams();
$cnt = count($params['SelectionCriteria']['Ids']);
return $cnt > self::MAX_COUNT ? self::MAX_COUNT : $cnt;
return count($this->getParams()['SelectionCriteria']['Ids']);
}
public function slice($maxObjects): ?APIRequest
......@@ -59,6 +56,7 @@ class ResumeCampaigns extends DirectRequest
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
......
......@@ -12,7 +12,7 @@ use Illuminate\Support\Facades\Log;
class SuspendCampaigns extends DirectRequest
{
CONST MAX_COUNT = 1000;
protected $max_count = 1000;
public function call($params = null)
{
......@@ -23,10 +23,7 @@ class SuspendCampaigns extends DirectRequest
public function getObjectsCount()
{
$params = $this->getParams();
$cnt = count($params['SelectionCriteria']['Ids']);
return $cnt > self::MAX_COUNT ? self::MAX_COUNT : $cnt;
return count($this->getParams()['SelectionCriteria']['Ids']);
}
public function slice($maxObjects): ?APIRequest
......@@ -60,6 +57,7 @@ class SuspendCampaigns extends DirectRequest
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
......
......@@ -14,7 +14,7 @@ use Illuminate\Support\Facades\Log;
class UpdateCampaigns extends DirectRequest
{
CONST MAX_COUNT = 10;
protected $max_count = 10;
protected $timestamp;
......@@ -27,10 +27,7 @@ class UpdateCampaigns extends DirectRequest
public function getObjectsCount()
{
$params = $this->getParams();
$cnt = count($params['Campaigns']);
return $cnt > self::MAX_COUNT ? self::MAX_COUNT : $cnt;
return count($this->getParams()['Campaigns']);
}
public function slice($maxObjects): ?APIRequest
......@@ -58,6 +55,7 @@ class UpdateCampaigns extends DirectRequest
}
} catch (\Exception $e) {
Log::debug($e);
throw $e;
}
}
......
<?php
namespace App\Service\Requests;
/**
......@@ -7,9 +8,10 @@ namespace App\Service\Requests;
*
*
*/
class DirectRequest extends APIRequest {
class DirectRequest extends APIRequest
{
CONST MAX_COUNT = 10000;
protected $max_count = -1;
private $url;
......@@ -25,27 +27,31 @@ class DirectRequest extends APIRequest {
return null;
}
function next($offset){
if (!isset($this->params['Page'])){
function next($offset)
{
if (!isset($this->params['Page'])) {
$this->params['Page'] = [];
}
$this->params['Page']['Limit'] = self::MAX_COUNT;
$this->params['Page']['Offset'] = $offset;
}
function getUrl(){
function getUrl()
{
return $this->url . strtolower($this->getService());
}
function getBody(){
function getBody()
{
$data = [
'method' => $this->getMethod(),
'params' => $this->getParams() ?? (object)[]
'params' => $this->getParams() ?? (object)[]
];
return json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
return json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
}
function getHeaders(){
function getHeaders()
{
return [
// "Authorization: Bearer " . $this->getToken()->token, // OAuth-токен. Использование слова Bearer обязательно
"Accept-Language: ru", // Язык ответных сообщений
......@@ -53,19 +59,26 @@ class DirectRequest extends APIRequest {
];
}
function getRequest($service, $method){
function getRequest($service, $method)
{
$ns = str_replace("DirectRequest", "Direct", self::class);
$class = $ns . "\\" . ucfirst($method) . ucfirst($service);
if (class_exists($class)){
if (class_exists($class)) {
$request = new $class($this->getApi());
$request->setToken($this->getToken());
return $request;
}
throw new Exception('Unknown Service or Method');
throw new \Exception('Unknown Service or Method');
return $this;
}
function getObjectsCount(){
return self::MAX_COUNT;
function getObjectsCount()
{
return $this->getMaxCount();
}
function getMaxCount()
{
return $this->max_count;
}
}
......@@ -19,8 +19,8 @@ class CreateDictionaryCampaignsTable extends Migration
$table->bigInteger('campaign_id')->unsigned();
$table->bigInteger('dictionary_id')->unsigned();
$table->string('name', 255)->nullable();
$table->text('negative_keywords')->nullable();
$table->text('excluded_sites')->nullable();
$table->json('negative_keywords')->nullable();
$table->json('excluded_sites')->nullable();
$table->boolean('synced')->default(1);
$table->boolean('updated')->default(1);
$table->boolean('enabled')->default(1);
......
......@@ -62,7 +62,7 @@ class AddCampaignsTest extends TestCase
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token)
->getRequest('campaigns', 'add');
->getRequest('Campaigns', 'add');
$this->actingAs($this->user)
->post(route('token.city.store', [
......
......@@ -62,7 +62,7 @@ class CallLimitedApiTest extends TestCase
$process->handle();
Queue::assertPushed(ProcessCallLimitedAPI::class, 3);
$this->token->limit=16;
$this->token->limit=15;
$this->request->setService('AdGroups');
$this->request->setMethod('get');
$process->handle();
......
......@@ -55,11 +55,11 @@ class CheckCampaignsTest extends TestCase
$this->request_main = APIRequest::getInstance(API::YANDEX)
->setToken($this->token_main)
->getRequest('change', 'CheckCampaigns');
->getRequest('Changes', 'checkCampaigns');
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token)
->getRequest('change', 'CheckCampaigns');
->getRequest('Changes', 'checkCampaigns');
}
......
......@@ -70,7 +70,7 @@ class CheckChangesAdGroupsTest extends TestCase
$this->request_main = APIRequest::getInstance(API::YANDEX)
->setToken($this->token_main)
->getRequest('changes', 'check');
->getRequest('Changes', 'check');
$this->request_main_params = [
'CampaignIds' => Campaigns::forUpdatedChildren()->pluck('external_id')->all(),
......@@ -82,7 +82,7 @@ class CheckChangesAdGroupsTest extends TestCase
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token)
->getRequest('changes', 'check');
->getRequest('Changes', 'check');
}
......
......@@ -38,7 +38,7 @@ class CheckDictionariesTest extends TestCase
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token)
->getRequest('change', 'CheckDictionaries');
->getRequest('Changes', 'CheckDictionaries');
}
......
......@@ -77,7 +77,7 @@ class GetAdGroupTest extends TestCase
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token)
->getRequest('adgroups', 'get');
->getRequest('AdGroups', 'get');
}
......
......@@ -42,7 +42,7 @@ class GetCampaignsTest extends TestCase
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token_main)
->getRequest('campaigns', 'get');
->getRequest('Campaigns', 'get');
}
......
......@@ -40,7 +40,7 @@ class GetDictionariesTest extends TestCase
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token)
->getRequest('dictionaries', 'get');
->getRequest('Dictionaries', 'get');
}
......
......@@ -123,16 +123,21 @@ class LimitsTest extends TestCase
{
$request = \App\Service\Requests\APIRequest::getInstance(API::YANDEX)
->setToken($this->token);
$request->setService('AdExtensions');
$request->setMethod('add');
$this->token->limit = 6;
$objects = $this->limitService->countObjectsLimit($request);
$this->assertEquals($objects, 1);
// Todo метод не описан, и нет максимального колл-во для загрузки, сейчас -1 без максимума
// $this->assertEquals($objects, 1);
$this->assertEquals($objects, -1);
$this->token->limit = 12;
$objects = $this->limitService->countObjectsLimit($request);
$this->assertEquals($objects, 7);
// Todo метод не описан, и нет максимального колл-во для загрузки, сейчас -1 без максимума
// $this->assertEquals($objects, 7);
$this->assertEquals($objects, -1);
$request->setService('Bids');
......@@ -140,11 +145,15 @@ class LimitsTest extends TestCase
$this->token->limit = 18;
$objects = $this->limitService->countObjectsLimit($request);
$this->assertEquals($objects, 2000);
// Todo метод не описан, и нет максимального колл-во для загрузки, сейчас -1 без максимума
// $this->assertEquals($objects, 2000);
$this->assertEquals($objects, -1);
$this->token->limit = 21;
$objects = $this->limitService->countObjectsLimit($request);
$this->assertEquals($objects, 4000);
// Todo метод не описан, и нет максимального колл-во для загрузки, сейчас -1 без максимума
// $this->assertEquals($objects, 4000);
$this->assertEquals($objects, -1);
Queue::fake();
......@@ -154,10 +163,10 @@ class LimitsTest extends TestCase
$objects = $this->limitService->countObjectsLimit($requestCmpgn);
$this->assertEquals($objects, 181);
$this->token->limit = DirectRequest::MAX_COUNT+100;
$this->token->limit = $requestCmpgn->getMaxCount() + 100;
$requestCmpgn->call();
$objects = $this->limitService->countObjectsLimit($requestCmpgn);
$this->assertEquals($objects, DirectRequest::MAX_COUNT);
$this->assertEquals($objects, $requestCmpgn->getMaxCount());
$requestCmpgn = $request->getRequest('Campaigns', 'update');
$this->token->limit = 191;
......
......@@ -66,7 +66,7 @@ class ProcessCallSliceTest extends TestCase
$this->request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token)
->getRequest('campaigns', 'add');
->getRequest('Campaigns', 'add');
$this->campaign->dictionaries()->syncWithoutDetaching(
$this->dictionaries->keyBy(Campaigns::getModel()->getKeyName())->transform(function (Dictionary $dictionary) {
......@@ -114,7 +114,7 @@ class ProcessCallSliceTest extends TestCase
$this->assertEquals(true, !!$requestR);
$this->assertEquals(2, $this->request->getObjectsCount());
$this->assertEquals($this->dictionaries_count - 2, count($requestR->getParams()['Campaigns']));
$this->assertEquals($this->dictionaries_count - 2, $requestR->getObjectsCount());
$limits = Limits::getInstance($this->request_resume->getToken());
......@@ -127,7 +127,7 @@ class ProcessCallSliceTest extends TestCase
$this->assertEquals(true, !!$requestR);
$this->assertEquals(2, $this->request_resume->getObjectsCount());
$this->assertEquals($this->dictionaries_count - 2, count($requestR->getParams()['SelectionCriteria']['Ids']));
$this->assertEquals($this->dictionaries_count - 2, $requestR->getObjectsCount());
}
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!