Skip to content

Письменов Дмитрий Иванович / yourroomads

Go to a project
Commit 5b1796fd by Vladislav
Options

Merge branch 'master' of https://gitlab.veeble.ru/pismenov/yourroomads

2 parents 24c3be4d 1b0bedca
Inline Side-by-side
Showing with 330 additions and 89 deletions
app/Console/Commands/KeywordsUpdate.php
...@@ -45,37 +45,23 @@ class KeywordsUpdate extends Command ...@@ -45,37 +45,23 @@ class KeywordsUpdate extends Command
*/ */
public function handle() public function handle()
{ {
$tokens = Tokens::whereHas('dictionaryCampaignsEnabledForExternalUpdated.goalKeywordsForNeedUpdated.keyword') $tokens = Tokens::/*whereHas('dictionaryCampaignsEnabledForExternalUpdated.goalKeywordsForNeedUpdated.keyword')
->where('type', '!=', Tokens::MAIN) ->*/where('type', '!=', Tokens::MAIN)
->get(); ->get();
foreach ($tokens as $token) { foreach ($tokens as $token) {
$token->load([ /*$token->load([
'dictionaryCampaignsEnabledForExternalSynchronized' => function (HasManyThrough $query) { 'dictionaryCampaignsEnabledForExternalSynchronized' => function (HasManyThrough $query) {
return $query->has('goalKeywordsForNeedUpdated.keyword'); return $query->has('goalKeywordsForNeedUpdated.keyword');
}, },
]); ]);*/
$factory = APIRequest::getInstance(API::YANDEX); $factory = APIRequest::getInstance(API::YANDEX);
$factory->setToken($token); $factory->setToken($token);
$goalKeywords = DB::table('goal_keywords') $goalKeywords = GoalKeyword::getForUpdate($token);
->join('keywords', 'goal_keywords.keyword_id', '=', 'keywords.id') if ($goalKeywords->isEmpty()) continue;
->whereNull('keywords.deleted_at')
->whereNull('keywords.reserve_update_at')
->whereNotNull('goal_advertisements.updated_need')
->whereNotNull('goal_keywords.goal_ad_group_external_id')
->whereNotNull('goal_keywords.dictionary_campaign_external_id')
->whereIn('goal_keywords.dictionary_campaign_id', $token->dictionaryCampaignsEnabledForExternalSynchronized->pluck('id'))
->select([
'goal_keywords.id as id',
'goal_keywords.dictionary_campaign_id as dictionary_campaign_id',
'keywords.keyword as keyword',
'keywords.user_param_1 as user_param_1',
'keywords.user_param_2 as user_param_2',
])
->get();
foreach (array_chunk($goalKeywords->pluck('id')->toArray(), 1000) as $items){ foreach (array_chunk($goalKeywords->pluck('id')->toArray(), 1000) as $items){
GoalKeyword::whereIn('id', $items) GoalKeyword::whereIn('id', $items)
......
app/Models/AdGroup.php
...@@ -172,7 +172,7 @@ class AdGroup extends Model ...@@ -172,7 +172,7 @@ class AdGroup extends Model
*/ */
public function scopeForUpdatedSelf($query) public function scopeForUpdatedSelf($query)
{ {
return $query->whereNotNull("{$query->getModel()->getTable()}.updated_self"); return $query->whereNotNull("updated_self");
} }
/** /**
......
app/Models/Keyword.php
...@@ -143,4 +143,15 @@ class Keyword extends Model ...@@ -143,4 +143,15 @@ class Keyword extends Model
return $this->belongsTo(Campaigns::class, 'campaign_id'); return $this->belongsTo(Campaigns::class, 'campaign_id');
} }
public function equals($data){
return $data['keyword'] == $this->keyword
&& $data['user_param_1'] == $this->user_param_1
&& $data['user_param_2'] == $this->user_param_2
&& $data['bid'] == $this->bid
&& $data['context_bid'] == $this->context_bid
&& $data['state'] == $this->state
&& $data['status'] == $this->status
&& $data['serving_status'] == $this->serving_status
;
}
} }
app/Models/Pivots/GoalKeyword.php
...@@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Builder; ...@@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Relations\Pivot; use Illuminate\Database\Eloquent\Relations\Pivot;
use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
/** /**
* App\Models\Pivots\GoalKeyword * App\Models\Pivots\GoalKeyword
...@@ -183,4 +184,25 @@ class GoalKeyword extends Pivot ...@@ -183,4 +184,25 @@ class GoalKeyword extends Pivot
return $this->belongsTo(DictionaryCampaign::class, 'dictionary_campaign_id'); return $this->belongsTo(DictionaryCampaign::class, 'dictionary_campaign_id');
} }
public static function getForUpdate($token){
return DB::table('goal_keywords')
->join('keywords', 'goal_keywords.keyword_id', '=', 'keywords.id')
->whereNull('keywords.deleted_at')
->whereNull('goal_keywords.reserve_update_at')
->whereNotNull('goal_keywords.updated_need')
->whereNotNull('goal_keywords.external_id')
->whereNotNull('goal_keywords.goal_ad_group_external_id')
->whereNotNull('goal_keywords.dictionary_campaign_external_id')
->whereIn('goal_keywords.dictionary_campaign_id', $token->dictionaryCampaignsEnabledForExternalSynchronized->pluck('id'))
->select([
'goal_keywords.id as id',
'goal_keywords.external_id',
'goal_keywords.dictionary_campaign_id as dictionary_campaign_id',
'keywords.keyword as keyword',
'keywords.user_param_1 as user_param_1',
'keywords.user_param_2 as user_param_2',
])
->get();
}
} }
app/Service/Requests/Direct/GetKeywords.php
...@@ -56,6 +56,10 @@ class GetKeywords extends DirectRequest ...@@ -56,6 +56,10 @@ class GetKeywords extends DirectRequest
function handle($response) function handle($response)
{ {
$connection = config('database.default');
$now = $connection=='sqlite'? DB::raw("strftime('%Y-%m-%d %H:%M:%S', datetime('now'))") : DB::raw('now()');
try { try {
$external_ids = []; $external_ids = [];
...@@ -90,6 +94,8 @@ class GetKeywords extends DirectRequest ...@@ -90,6 +94,8 @@ class GetKeywords extends DirectRequest
// $campaign_ids_synced_need = []; // $campaign_ids_synced_need = [];
$insertData = []; $insertData = [];
$cnt = 0;
$items = [];
foreach ($response['result']['Keywords'] as $keyword) { foreach ($response['result']['Keywords'] as $keyword) {
$ad_group = $ad_groups->get((string)$keyword['AdGroupId']); $ad_group = $ad_groups->get((string)$keyword['AdGroupId']);
...@@ -100,11 +106,6 @@ class GetKeywords extends DirectRequest ...@@ -100,11 +106,6 @@ class GetKeywords extends DirectRequest
$external_id = (string)$keyword['Id']; $external_id = (string)$keyword['Id'];
//идентификатор фразы у нас уникален. Если такая фраза уже есть, нет смысла ее менять, т.к. данные в общем то неизменны.
//если будет изменена фраза, то на самом деле будет фраза с новым идентификатором
//поэтому будем всегда новые добавлять и игонрить те что уже есть
$data = [ $data = [
'external_id' => $external_id, 'external_id' => $external_id,
'campaign_external_id' => $ad_group->campaign_external_id, 'campaign_external_id' => $ad_group->campaign_external_id,
...@@ -120,77 +121,43 @@ class GetKeywords extends DirectRequest ...@@ -120,77 +121,43 @@ class GetKeywords extends DirectRequest
'state' => $keyword['State'], 'state' => $keyword['State'],
'status' => $keyword['Status'], 'status' => $keyword['Status'],
'serving_status' => $keyword['ServingStatus'], 'serving_status' => $keyword['ServingStatus'],
'updated_at' => DB::raw('now()'), 'updated_at' => $now,
'deleted_at' => null//не забыть убрать признак удаления, если вдруг опять пришла удаленная ранее фраза 'deleted_at' => null//не забыть убрать признак удаления, если вдруг опять пришла удаленная ранее фраза
]; ];
$insertData[] = $data; $keyword_data = Keyword::where('external_id', $external_id)->get()->first();
// $keyword = Keyword::updateOrCreate([ if ($keyword_data){
// 'external_id' => $external_id if (!$keyword_data->equals($data)){
// ], $data); //если данные изменились, то надо обновить и в кеше и пометить на то что надо их выгрузить
$k_upd = Keyword::where('external_id', $external_id);
// if ($keyword->wasRecentlyCreated) { unset($data['external_id']);
// $campaign_ids_synced_need[$keyword->campaign_id] = true; $k_upd->update($data);
// } elseif ($keyword->wasChanged(['campaign_id'])) { $k_upd->get()->first()->goalKeywords()->update(['updated_need' => $now]);
// $campaign_ids_synced_need[$keyword->campaign_id] = true; }else{
// } elseif ($keyword->wasChanged($keyword::getPropertiesWatch()->toArray())) { $items[] = $external_id;
// $keyword->goalKeywords()->has('dictionaryCampaign')->forExternal()->update([ }
// 'updated_need' => Carbon::now(), } else{
// ]); //новый фразы
// } $insertData[$external_id] = $data;
}
} }
Keyword::whereIn('external_id', $items)->update([
'updated_at' => $now
]);
foreach (array_chunk($insertData, 1000) as $data) { foreach (array_chunk($insertData, 1000) as $data) {
$items = []; $items = [];
foreach ($data as $item){ foreach ($data as $item){
$items[] = $item['external_id']; $items[] = $item['external_id'];
} }
Keyword::insertOrIgnore($data); Keyword::insertOrIgnore($data);
Keyword::whereIn('external_id', $items)->update(['updated_at' => DB::raw('now()')]); Keyword::whereIn('external_id', $items)->update([
'updated_at' => $now
]);
} }
// if ($this->getToken()->isMain()) {
//
// if (isset($this->getParams()['SelectionCriteria']['AdGroupIds'])) {
// AdGroup::whereIn('external_id', $this->getParams()['SelectionCriteria']['AdGroupIds'])
// ->update([
// 'keywords_loaded_at' => Carbon::now(),
// ]);
// }
//
// if (count($campaign_ids_synced_need)) {
// Campaigns::findMany(array_keys($campaign_ids_synced_need))->each(function (Campaigns $campaign) {
// $campaign->dictionaryCampaigns()->update([
// 'synced_need' => Carbon::now(),
// ]);
// });
// }
//
// $keywordQuery = Keyword::query();
//
// if (isset($this->getParams()['SelectionCriteria']['AdGroupIds'])) {
// $keywordQuery->whereIn('ad_group_external_id', $this->getParams()['SelectionCriteria']['AdGroupIds']);
// } else {
// $keywordQuery->whereIn('ad_group_id', $ad_groups->pluck('id'));
// }
//
// } else {
// $keywordQuery = GoalKeyword::query()
// ->whereIn('goal_ad_group_id', $ad_groups->pluck('id'));
// }
//
// if (count($ids)) {
// $keywordQuery->whereNotIn('id', $ids);
// }
//
// $keywordQuery->get()->each(function ($goalKeyword) {
// /* @var $goalKeyword GoalKeyword|Keyword */
// $goalKeyword->delete();
// });
//удаление будет толко когда получаем изменения по группам, иначе это либо загрузка все фраз либо как то избранных //удаление будет толко когда получаем изменения по группам, иначе это либо загрузка все фраз либо как то избранных
if ( !empty($this->getParams()['SelectionCriteria']['AdGroupIds']) && if ( !empty($this->getParams()['SelectionCriteria']['AdGroupIds']) &&
!isset($response['result']['LimitedBy']) ){ !isset($response['result']['LimitedBy']) ){
...@@ -200,17 +167,23 @@ class GetKeywords extends DirectRequest ...@@ -200,17 +167,23 @@ class GetKeywords extends DirectRequest
//это означает что этой фразы не было в результатах и в БД она по этой причине не обновилась //это означает что этой фразы не было в результатах и в БД она по этой причине не обновилась
//надо такие пометить на удаление //надо такие пометить на удаление
//при удалении для всех таких надо будет удалить фразы из целевых и потом удалить их сами //при удалении для всех таких надо будет удалить фразы из целевых и потом удалить их сами
if ($connection=='sqlite'){
$sql = "UPDATE keywords k $sql = "UPDATE keywords
SET deleted_at = {$now->getValue()}
WHERE updated_at<=(SELECT ag.keywords_loaded_at FROM ad_groups ag WHERE keywords.ad_group_id=ag.id)
AND deleted_at is null
AND ad_group_external_id in (" . implode(", ", $ag_groups) . ")";
} else{
$sql = "UPDATE keywords k
INNER JOIN ad_groups ag ON k.ad_group_id=ag.id INNER JOIN ad_groups ag ON k.ad_group_id=ag.id
SET k.deleted_at = now() SET k.deleted_at = {$now->getValue()}
WHERE k.updated_at<=ag.keywords_loaded_at WHERE k.updated_at<=ag.keywords_loaded_at
AND k.deleted_at is null AND k.deleted_at is null
AND ag.external_id in (" . implode(", ", $ag_groups) . ")"; AND ag.external_id in (" . implode(", ", $ag_groups) . ")";
DB::update($sql); }
$sql = "UPDATE ad_groups SET keywords_loaded_at=now() WHERE external_id in (" . implode(", ", $ag_groups) . ")";
DB::update($sql); DB::update($sql);
AdGroup::whereIn('external_id', $ag_groups)->update(['keywords_loaded_at' => $now]);
} }
} catch (\Exception $e) { } catch (\Exception $e) {
Log::debug($e); Log::debug($e);
......
app/Service/Requests/Direct/UpdateKeywords.php
...@@ -100,7 +100,6 @@ class UpdateKeywords extends DirectRequest ...@@ -100,7 +100,6 @@ class UpdateKeywords extends DirectRequest
} else { } else {
$list = $lists[$goalKeyword->dictionary_campaign_id]; $list = $lists[$goalKeyword->dictionary_campaign_id];
} }
$data = [ $data = [
'Id' => $goalKeyword->external_id, 'Id' => $goalKeyword->external_id,
'Keyword' => StrReplaceByVariables::getInstance($goalKeyword->keyword, $list)->get(), 'Keyword' => StrReplaceByVariables::getInstance($goalKeyword->keyword, $list)->get(),
......
database/factories/KeywordsFactory.php 0 → 100644
<?php
use Faker\Generator as Faker;
$factory->define(App\Models\Keyword::class, function (Faker $faker) {
return [
'keyword' => $faker->name,
'external_id' => $faker->unique(true)->randomNumber(),
'bid' => $faker->randomNumber(),
'context_bid' => $faker->randomNumber(),
'strategy_priority' => $faker->randomElement(['LOW', 'NORMAL', 'HIGH']),
'state' => $faker->randomElement(['OFF', 'ON', 'SUSPENDED']),
'status' => $faker->randomElement(['ACCEPTED', 'DRAFT', 'REJECTED']),
'serving_status' => $faker->randomElement(['ELIGIBLE', 'RARELY_SERVED']),
];
});
tests/Unit/KeywordsTest.php 0 → 100644
<?php
namespace Tests\Unit;
use App\Jobs\ProcessCallLimitedAPI;
use App\Models\Account;
use App\Models\AdGroup;
use App\Models\Campaigns;
use App\Models\Dictionary;
use App\Models\Keyword;
use App\Models\Pivots\GoalKeyword;
use App\Models\Tokens;
use App\Models\User;
use App\Service\Contract\API;
use App\Service\Requests\APIRequest;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
use Illuminate\Foundation\Testing\RefreshDatabase;
class KeywordsTest extends TestCase
{
use RefreshDatabase;
private $request;
private $user;
private $token_main;
private $token;
private $campaign;
private $groups;
private $dictionary;
private $dicCampaign;
protected function setUp(): void
{
parent::setUp();
$account = Account::create(['name' => 'Acme Corporation']);
$this->user = factory(User::class)->create([
'account_id' => $account->id,
'first_name' => 'John',
'last_name' => 'Doe',
'email' => 'johndoe@example.com',
'owner' => true,
]);
$this->token = factory(Tokens::class)->create([
'created_by' => $this->user->getKey()
]);
$this->token_main = factory(Tokens::class)->create([
'type' => Tokens::MAIN,
'created_by' => $this->user->id,
]);
$this->campaign = factory(Campaigns::class)->create([
'external_id' => 1,
'manage' => true,
'token' => $this->token_main->getKey(),
]);
$this->dictionary = factory(Dictionary::class)->create([
'region_id' => 1,
'type' => Dictionary::CITY,
]);
$this->actingAs($this->user)
->post(route('token.city.store', [
$this->token->id,
$this->dictionary->getKey(),
]))
->assertStatus(302);
$this->dicCampaign = $this->campaign->dictionaryCampaigns()->first();
$this->dicCampaign->external_id = 12;
$this->dicCampaign->save();
//группы
$this->groups = factory(AdGroup::class)->create([
'external_id' => 1,
'campaign_id' => $this->campaign->getKey(),
'campaign_external_id' => $this->campaign->external_id,
]);
$connection = config('database.default');
$now = $connection=='sqlite'? DB::raw("strftime('%Y-%m-%d %H:%M:%S', datetime('now'))") : DB::raw('now()');
AdGroup::where('external_id', 1)->update(['keywords_loaded_at' => $now]);
//целевые группы
$this->goalGroups = $this->groups->goalGroups()->create([
'external_id' => 2,
'dictionary_campaign_external_id' => $this->dicCampaign->external_id,
'ad_group_id' => $this->groups->id,
'dictionary_campaign_id' => $this->dicCampaign->id,
'name' => $this->groups->name,
'negative_keywords' => $this->groups->negative_keywords
]);
//ключи
for($i=1; $i<11; $i++){
$this->groups->keywords()->create(
factory(Keyword::class)->make([
'campaign_id' => $this->campaign->getKey(),
'external_id' => $i,
'campaign_external_id' => $this->campaign->external_id,
'ad_group_external_id' => $this->groups->external_id,
])->toArray()
);
}
//целевые ключи
foreach ($this->groups->keywords as $keyword){
$keyword->goalKeywords()->create([
'external_id' => "101" . $keyword->external_id,
'dictionary_campaign_external_id' => $this->dicCampaign->external_id,
'goal_ad_group_external_id' => $this->goalGroups->external_id,
'dictionary_campaign_id' => $this->dicCampaign->id,
'goal_ad_group_id' => $this->goalGroups->id,
'keyword_id' => $keyword->id,
]);
}
}
public function testCallApi()
{
Queue::fake();
$request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token_main)
->getRequest('Keywords', 'get');
Queue::assertNothingPushed();
$ids_limit = [1,2,3];
$request->call(['AdGroupIds' => $ids_limit]);
Queue::assertPushed(ProcessCallLimitedAPI::class);
}
public function testHandleApi()
{
Queue::fake();
$request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token_main)
->getRequest('Keywords', 'get');
$ids_limit = [1];
$request->call(['AdGroupIds' => $ids_limit]);
Queue::assertPushed(ProcessCallLimitedAPI::class);
sleep(2);
//генерим новые 10 - слов
$keywords = factory(Keyword::class, 10)->make([
'campaign_id' => $this->campaign->getKey(),
'campaign_external_id' => $this->campaign->external_id,
'ad_group_external_id' => $this->groups->external_id,
]);
$data = [
'result' => [
'Keywords' => [
]
]
];
//добавляем новые слова
foreach ($keywords as $k=>$keyword){
$data['result']['Keywords'][] = [
'Id' => $k+100,
'AdGroupId' => $keyword->ad_group_external_id,
'Keyword' => $keyword->keyword,
'UserParam1' => null,
'UserParam2' => null,
'Bid' => $keyword->bid,
'ContextBid' => $keyword->context_bid,
'StrategyPriority' => $keyword->strategy_priority,
'State' => $keyword->state,
'Status' => $keyword->status,
'ServingStatus' => $keyword->serving_status,
];
}
//добавляем существующие. Из них 3 оставим как есть, 3 изменим, а 4 удалим
foreach ($this->groups->keywords as $k=>$keyword){
if ($k<4){
continue;//удаляем
}
if ($k<7){//оставляем как есть
$data['result']['Keywords'][] = [
'Id' => $keyword->external_id,
'AdGroupId' => $keyword->ad_group_external_id,
'Keyword' => $keyword->keyword,
'UserParam1' => null,
'UserParam2' => null,
'Bid' => $keyword->bid,
'ContextBid' => $keyword->context_bid,
'StrategyPriority' => $keyword->strategy_priority,
'State' => $keyword->state,
'Status' => $keyword->status,
'ServingStatus' => $keyword->serving_status,
];
continue;
}
//остальные меняем
$data['result']['Keywords'][] = [
'Id' => $keyword->external_id,
'AdGroupId' => $keyword->ad_group_external_id,
'Keyword' => $keyword->keyword . ' change',
'UserParam1' => null,
'UserParam2' => null,
'Bid' => $keyword->bid,
'ContextBid' => $keyword->context_bid,
'StrategyPriority' => $keyword->strategy_priority,
'State' => $keyword->state,
'Status' => $keyword->status,
'ServingStatus' => $keyword->serving_status,
];
}
$request->handle($data);
$this->groups->refresh();
$this->goalGroups->refresh();
//в результате у нас должно стать 20 фраз
//из них 10 новых
$this->assertCount( 20, $this->groups->keywords()->withTrashed()->get());
//3 помечены на изменение
$this->assertCount( 3, $this->goalGroups->goalKeywords()->whereNotNull('updated_need')->get());
//20 - 4 на удаление = 16
$this->assertCount( 16, $this->groups->keywords);
$goalKeywords = GoalKeyword::getForUpdate($this->token);
$this->assertCount( 3, $goalKeywords);
}
}
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!