Skip to content

Письменов Дмитрий Иванович / yourroomads

Go to a project
Commit 5b1796fd by Vladislav
Options

Merge branch 'master' of https://gitlab.veeble.ru/pismenov/yourroomads

2 parents 24c3be4d 1b0bedca
Inline Side-by-side
Showing with 329 additions and 88 deletions
app/Console/Commands/KeywordsUpdate.php
......@@ -45,37 +45,23 @@ class KeywordsUpdate extends Command
*/
public function handle()
{
$tokens = Tokens::whereHas('dictionaryCampaignsEnabledForExternalUpdated.goalKeywordsForNeedUpdated.keyword')
->where('type', '!=', Tokens::MAIN)
$tokens = Tokens::/*whereHas('dictionaryCampaignsEnabledForExternalUpdated.goalKeywordsForNeedUpdated.keyword')
->*/where('type', '!=', Tokens::MAIN)
->get();
foreach ($tokens as $token) {
$token->load([
/*$token->load([
'dictionaryCampaignsEnabledForExternalSynchronized' => function (HasManyThrough $query) {
return $query->has('goalKeywordsForNeedUpdated.keyword');
},
]);
]);*/
$factory = APIRequest::getInstance(API::YANDEX);
$factory->setToken($token);
$goalKeywords = DB::table('goal_keywords')
->join('keywords', 'goal_keywords.keyword_id', '=', 'keywords.id')
->whereNull('keywords.deleted_at')
->whereNull('keywords.reserve_update_at')
->whereNotNull('goal_advertisements.updated_need')
->whereNotNull('goal_keywords.goal_ad_group_external_id')
->whereNotNull('goal_keywords.dictionary_campaign_external_id')
->whereIn('goal_keywords.dictionary_campaign_id', $token->dictionaryCampaignsEnabledForExternalSynchronized->pluck('id'))
->select([
'goal_keywords.id as id',
'goal_keywords.dictionary_campaign_id as dictionary_campaign_id',
'keywords.keyword as keyword',
'keywords.user_param_1 as user_param_1',
'keywords.user_param_2 as user_param_2',
])
->get();
$goalKeywords = GoalKeyword::getForUpdate($token);
if ($goalKeywords->isEmpty()) continue;
foreach (array_chunk($goalKeywords->pluck('id')->toArray(), 1000) as $items){
GoalKeyword::whereIn('id', $items)
......
app/Models/AdGroup.php
......@@ -172,7 +172,7 @@ class AdGroup extends Model
*/
public function scopeForUpdatedSelf($query)
{
return $query->whereNotNull("{$query->getModel()->getTable()}.updated_self");
return $query->whereNotNull("updated_self");
}
/**
......
app/Models/Keyword.php
......@@ -143,4 +143,15 @@ class Keyword extends Model
return $this->belongsTo(Campaigns::class, 'campaign_id');
}
public function equals($data){
return $data['keyword'] == $this->keyword
&& $data['user_param_1'] == $this->user_param_1
&& $data['user_param_2'] == $this->user_param_2
&& $data['bid'] == $this->bid
&& $data['context_bid'] == $this->context_bid
&& $data['state'] == $this->state
&& $data['status'] == $this->status
&& $data['serving_status'] == $this->serving_status
;
}
}
app/Models/Pivots/GoalKeyword.php
......@@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Relations\Pivot;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
/**
* App\Models\Pivots\GoalKeyword
......@@ -183,4 +184,25 @@ class GoalKeyword extends Pivot
return $this->belongsTo(DictionaryCampaign::class, 'dictionary_campaign_id');
}
public static function getForUpdate($token){
return DB::table('goal_keywords')
->join('keywords', 'goal_keywords.keyword_id', '=', 'keywords.id')
->whereNull('keywords.deleted_at')
->whereNull('goal_keywords.reserve_update_at')
->whereNotNull('goal_keywords.updated_need')
->whereNotNull('goal_keywords.external_id')
->whereNotNull('goal_keywords.goal_ad_group_external_id')
->whereNotNull('goal_keywords.dictionary_campaign_external_id')
->whereIn('goal_keywords.dictionary_campaign_id', $token->dictionaryCampaignsEnabledForExternalSynchronized->pluck('id'))
->select([
'goal_keywords.id as id',
'goal_keywords.external_id',
'goal_keywords.dictionary_campaign_id as dictionary_campaign_id',
'keywords.keyword as keyword',
'keywords.user_param_1 as user_param_1',
'keywords.user_param_2 as user_param_2',
])
->get();
}
}
app/Service/Requests/Direct/GetKeywords.php
......@@ -56,6 +56,10 @@ class GetKeywords extends DirectRequest
function handle($response)
{
$connection = config('database.default');
$now = $connection=='sqlite'? DB::raw("strftime('%Y-%m-%d %H:%M:%S', datetime('now'))") : DB::raw('now()');
try {
$external_ids = [];
......@@ -90,6 +94,8 @@ class GetKeywords extends DirectRequest
// $campaign_ids_synced_need = [];
$insertData = [];
$cnt = 0;
$items = [];
foreach ($response['result']['Keywords'] as $keyword) {
$ad_group = $ad_groups->get((string)$keyword['AdGroupId']);
......@@ -100,11 +106,6 @@ class GetKeywords extends DirectRequest
$external_id = (string)$keyword['Id'];
//идентификатор фразы у нас уникален. Если такая фраза уже есть, нет смысла ее менять, т.к. данные в общем то неизменны.
//если будет изменена фраза, то на самом деле будет фраза с новым идентификатором
//поэтому будем всегда новые добавлять и игонрить те что уже есть
$data = [
'external_id' => $external_id,
'campaign_external_id' => $ad_group->campaign_external_id,
......@@ -120,77 +121,43 @@ class GetKeywords extends DirectRequest
'state' => $keyword['State'],
'status' => $keyword['Status'],
'serving_status' => $keyword['ServingStatus'],
'updated_at' => DB::raw('now()'),
'updated_at' => $now,
'deleted_at' => null//не забыть убрать признак удаления, если вдруг опять пришла удаленная ранее фраза
];
$insertData[] = $data;
// $keyword = Keyword::updateOrCreate([
// 'external_id' => $external_id
// ], $data);
// if ($keyword->wasRecentlyCreated) {
// $campaign_ids_synced_need[$keyword->campaign_id] = true;
// } elseif ($keyword->wasChanged(['campaign_id'])) {
// $campaign_ids_synced_need[$keyword->campaign_id] = true;
// } elseif ($keyword->wasChanged($keyword::getPropertiesWatch()->toArray())) {
// $keyword->goalKeywords()->has('dictionaryCampaign')->forExternal()->update([
// 'updated_need' => Carbon::now(),
// ]);
// }
$keyword_data = Keyword::where('external_id', $external_id)->get()->first();
if ($keyword_data){
if (!$keyword_data->equals($data)){
//если данные изменились, то надо обновить и в кеше и пометить на то что надо их выгрузить
$k_upd = Keyword::where('external_id', $external_id);
unset($data['external_id']);
$k_upd->update($data);
$k_upd->get()->first()->goalKeywords()->update(['updated_need' => $now]);
}else{
$items[] = $external_id;
}
} else{
//новый фразы
$insertData[$external_id] = $data;
}
}
Keyword::whereIn('external_id', $items)->update([
'updated_at' => $now
]);
foreach (array_chunk($insertData, 1000) as $data) {
$items = [];
foreach ($data as $item){
$items[] = $item['external_id'];
}
Keyword::insertOrIgnore($data);
Keyword::whereIn('external_id', $items)->update(['updated_at' => DB::raw('now()')]);
Keyword::whereIn('external_id', $items)->update([
'updated_at' => $now
]);
}
// if ($this->getToken()->isMain()) {
//
// if (isset($this->getParams()['SelectionCriteria']['AdGroupIds'])) {
// AdGroup::whereIn('external_id', $this->getParams()['SelectionCriteria']['AdGroupIds'])
// ->update([
// 'keywords_loaded_at' => Carbon::now(),
// ]);
// }
//
// if (count($campaign_ids_synced_need)) {
// Campaigns::findMany(array_keys($campaign_ids_synced_need))->each(function (Campaigns $campaign) {
// $campaign->dictionaryCampaigns()->update([
// 'synced_need' => Carbon::now(),
// ]);
// });
// }
//
// $keywordQuery = Keyword::query();
//
// if (isset($this->getParams()['SelectionCriteria']['AdGroupIds'])) {
// $keywordQuery->whereIn('ad_group_external_id', $this->getParams()['SelectionCriteria']['AdGroupIds']);
// } else {
// $keywordQuery->whereIn('ad_group_id', $ad_groups->pluck('id'));
// }
//
// } else {
// $keywordQuery = GoalKeyword::query()
// ->whereIn('goal_ad_group_id', $ad_groups->pluck('id'));
// }
//
// if (count($ids)) {
// $keywordQuery->whereNotIn('id', $ids);
// }
//
// $keywordQuery->get()->each(function ($goalKeyword) {
// /* @var $goalKeyword GoalKeyword|Keyword */
// $goalKeyword->delete();
// });
//удаление будет толко когда получаем изменения по группам, иначе это либо загрузка все фраз либо как то избранных
if ( !empty($this->getParams()['SelectionCriteria']['AdGroupIds']) &&
!isset($response['result']['LimitedBy']) ){
......@@ -200,17 +167,23 @@ class GetKeywords extends DirectRequest
//это означает что этой фразы не было в результатах и в БД она по этой причине не обновилась
//надо такие пометить на удаление
//при удалении для всех таких надо будет удалить фразы из целевых и потом удалить их сами
if ($connection=='sqlite'){
$sql = "UPDATE keywords
SET deleted_at = {$now->getValue()}
WHERE updated_at<=(SELECT ag.keywords_loaded_at FROM ad_groups ag WHERE keywords.ad_group_id=ag.id)
AND deleted_at is null
AND ad_group_external_id in (" . implode(", ", $ag_groups) . ")";
} else{
$sql = "UPDATE keywords k
INNER JOIN ad_groups ag ON k.ad_group_id=ag.id
SET k.deleted_at = now()
SET k.deleted_at = {$now->getValue()}
WHERE k.updated_at<=ag.keywords_loaded_at
AND k.deleted_at is null
AND ag.external_id in (" . implode(", ", $ag_groups) . ")";
DB::update($sql);
}
$sql = "UPDATE ad_groups SET keywords_loaded_at=now() WHERE external_id in (" . implode(", ", $ag_groups) . ")";
DB::update($sql);
AdGroup::whereIn('external_id', $ag_groups)->update(['keywords_loaded_at' => $now]);
}
} catch (\Exception $e) {
Log::debug($e);
......
app/Service/Requests/Direct/UpdateKeywords.php
......@@ -100,7 +100,6 @@ class UpdateKeywords extends DirectRequest
} else {
$list = $lists[$goalKeyword->dictionary_campaign_id];
}
$data = [
'Id' => $goalKeyword->external_id,
'Keyword' => StrReplaceByVariables::getInstance($goalKeyword->keyword, $list)->get(),
......
database/factories/KeywordsFactory.php 0 → 100644
<?php
use Faker\Generator as Faker;
$factory->define(App\Models\Keyword::class, function (Faker $faker) {
return [
'keyword' => $faker->name,
'external_id' => $faker->unique(true)->randomNumber(),
'bid' => $faker->randomNumber(),
'context_bid' => $faker->randomNumber(),
'strategy_priority' => $faker->randomElement(['LOW', 'NORMAL', 'HIGH']),
'state' => $faker->randomElement(['OFF', 'ON', 'SUSPENDED']),
'status' => $faker->randomElement(['ACCEPTED', 'DRAFT', 'REJECTED']),
'serving_status' => $faker->randomElement(['ELIGIBLE', 'RARELY_SERVED']),
];
});
tests/Unit/KeywordsTest.php 0 → 100644
<?php
namespace Tests\Unit;
use App\Jobs\ProcessCallLimitedAPI;
use App\Models\Account;
use App\Models\AdGroup;
use App\Models\Campaigns;
use App\Models\Dictionary;
use App\Models\Keyword;
use App\Models\Pivots\GoalKeyword;
use App\Models\Tokens;
use App\Models\User;
use App\Service\Contract\API;
use App\Service\Requests\APIRequest;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
use Illuminate\Foundation\Testing\RefreshDatabase;
class KeywordsTest extends TestCase
{
use RefreshDatabase;
private $request;
private $user;
private $token_main;
private $token;
private $campaign;
private $groups;
private $dictionary;
private $dicCampaign;
protected function setUp(): void
{
parent::setUp();
$account = Account::create(['name' => 'Acme Corporation']);
$this->user = factory(User::class)->create([
'account_id' => $account->id,
'first_name' => 'John',
'last_name' => 'Doe',
'email' => 'johndoe@example.com',
'owner' => true,
]);
$this->token = factory(Tokens::class)->create([
'created_by' => $this->user->getKey()
]);
$this->token_main = factory(Tokens::class)->create([
'type' => Tokens::MAIN,
'created_by' => $this->user->id,
]);
$this->campaign = factory(Campaigns::class)->create([
'external_id' => 1,
'manage' => true,
'token' => $this->token_main->getKey(),
]);
$this->dictionary = factory(Dictionary::class)->create([
'region_id' => 1,
'type' => Dictionary::CITY,
]);
$this->actingAs($this->user)
->post(route('token.city.store', [
$this->token->id,
$this->dictionary->getKey(),
]))
->assertStatus(302);
$this->dicCampaign = $this->campaign->dictionaryCampaigns()->first();
$this->dicCampaign->external_id = 12;
$this->dicCampaign->save();
//группы
$this->groups = factory(AdGroup::class)->create([
'external_id' => 1,
'campaign_id' => $this->campaign->getKey(),
'campaign_external_id' => $this->campaign->external_id,
]);
$connection = config('database.default');
$now = $connection=='sqlite'? DB::raw("strftime('%Y-%m-%d %H:%M:%S', datetime('now'))") : DB::raw('now()');
AdGroup::where('external_id', 1)->update(['keywords_loaded_at' => $now]);
//целевые группы
$this->goalGroups = $this->groups->goalGroups()->create([
'external_id' => 2,
'dictionary_campaign_external_id' => $this->dicCampaign->external_id,
'ad_group_id' => $this->groups->id,
'dictionary_campaign_id' => $this->dicCampaign->id,
'name' => $this->groups->name,
'negative_keywords' => $this->groups->negative_keywords
]);
//ключи
for($i=1; $i<11; $i++){
$this->groups->keywords()->create(
factory(Keyword::class)->make([
'campaign_id' => $this->campaign->getKey(),
'external_id' => $i,
'campaign_external_id' => $this->campaign->external_id,
'ad_group_external_id' => $this->groups->external_id,
])->toArray()
);
}
//целевые ключи
foreach ($this->groups->keywords as $keyword){
$keyword->goalKeywords()->create([
'external_id' => "101" . $keyword->external_id,
'dictionary_campaign_external_id' => $this->dicCampaign->external_id,
'goal_ad_group_external_id' => $this->goalGroups->external_id,
'dictionary_campaign_id' => $this->dicCampaign->id,
'goal_ad_group_id' => $this->goalGroups->id,
'keyword_id' => $keyword->id,
]);
}
}
public function testCallApi()
{
Queue::fake();
$request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token_main)
->getRequest('Keywords', 'get');
Queue::assertNothingPushed();
$ids_limit = [1,2,3];
$request->call(['AdGroupIds' => $ids_limit]);
Queue::assertPushed(ProcessCallLimitedAPI::class);
}
public function testHandleApi()
{
Queue::fake();
$request = APIRequest::getInstance(API::YANDEX)
->setToken($this->token_main)
->getRequest('Keywords', 'get');
$ids_limit = [1];
$request->call(['AdGroupIds' => $ids_limit]);
Queue::assertPushed(ProcessCallLimitedAPI::class);
sleep(2);
//генерим новые 10 - слов
$keywords = factory(Keyword::class, 10)->make([
'campaign_id' => $this->campaign->getKey(),
'campaign_external_id' => $this->campaign->external_id,
'ad_group_external_id' => $this->groups->external_id,
]);
$data = [
'result' => [
'Keywords' => [
]
]
];
//добавляем новые слова
foreach ($keywords as $k=>$keyword){
$data['result']['Keywords'][] = [
'Id' => $k+100,
'AdGroupId' => $keyword->ad_group_external_id,
'Keyword' => $keyword->keyword,
'UserParam1' => null,
'UserParam2' => null,
'Bid' => $keyword->bid,
'ContextBid' => $keyword->context_bid,
'StrategyPriority' => $keyword->strategy_priority,
'State' => $keyword->state,
'Status' => $keyword->status,
'ServingStatus' => $keyword->serving_status,
];
}
//добавляем существующие. Из них 3 оставим как есть, 3 изменим, а 4 удалим
foreach ($this->groups->keywords as $k=>$keyword){
if ($k<4){
continue;//удаляем
}
if ($k<7){//оставляем как есть
$data['result']['Keywords'][] = [
'Id' => $keyword->external_id,
'AdGroupId' => $keyword->ad_group_external_id,
'Keyword' => $keyword->keyword,
'UserParam1' => null,
'UserParam2' => null,
'Bid' => $keyword->bid,
'ContextBid' => $keyword->context_bid,
'StrategyPriority' => $keyword->strategy_priority,
'State' => $keyword->state,
'Status' => $keyword->status,
'ServingStatus' => $keyword->serving_status,
];
continue;
}
//остальные меняем
$data['result']['Keywords'][] = [
'Id' => $keyword->external_id,
'AdGroupId' => $keyword->ad_group_external_id,
'Keyword' => $keyword->keyword . ' change',
'UserParam1' => null,
'UserParam2' => null,
'Bid' => $keyword->bid,
'ContextBid' => $keyword->context_bid,
'StrategyPriority' => $keyword->strategy_priority,
'State' => $keyword->state,
'Status' => $keyword->status,
'ServingStatus' => $keyword->serving_status,
];
}
$request->handle($data);
$this->groups->refresh();
$this->goalGroups->refresh();
//в результате у нас должно стать 20 фраз
//из них 10 новых
$this->assertCount( 20, $this->groups->keywords()->withTrashed()->get());
//3 помечены на изменение
$this->assertCount( 3, $this->goalGroups->goalKeywords()->whereNotNull('updated_need')->get());
//20 - 4 на удаление = 16
$this->assertCount( 16, $this->groups->keywords);
$goalKeywords = GoalKeyword::getForUpdate($this->token);
$this->assertCount( 3, $goalKeywords);
}
}
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!