FRAC = 0.25

DIN_SESS_MAX_LEN = 50

DSIN_SESS_COUNT = 5

DSIN_SESS_MAX_LEN = 10

from collections import OrderedDict

from deepctr.input_embedding import get_inputs_list, create_singlefeat_inputdict, get_embedding_vec_list

from deepctr.layers.core import DNN, PredictionLayer

from deepctr.layers.sequence import AttentionSequencePoolingLayer, BiLSTM, Transformer, BiasEncoding

from deepctr.layers.utils import concat_fun, NoMask

from deepctr.utils import check_feature_config_dict

from tensorflow.python.keras.initializers import RandomNormal

from tensorflow.python.keras.layers import Input, Dense, Embedding, Concatenate, Flatten

from tensorflow.python.keras.models import Model

from tensorflow.python.keras.regularizers import l2

def DSIN(feature_dim_dict, sess_feature_list, embedding_size=8, sess_max_count=5, sess_len_max=10,

att_embedding_size=1, att_head_num=8, dnn_hidden_units=(200, 80), dnn_activation='sigmoid',

l2_reg_dnn=0, l2_reg_embedding=1e-6, task='binary', dnn_dropout=0, init_std=0.0001, seed=1024,

bias_encoding=False):

# 检查一下特征字典是否正确

check_feature_config_dict(feature_dim_dict)

sparse_input, dense_input, user_behavior_input_dict, _, user_sess_length = get_input(

feature_dim_dict, sess_feature_list, sess_max_count, sess_len_max)

sparse_embedding_dict = {feat.name: Embedding(feat.dimension, embedding_size,

embeddings_initializer=RandomNormal(

mean=0.0, stddev=init_std, seed=seed),

embeddings_regularizer=l2(

l2_reg_embedding),

name='sparse_emb_' + str(i) + '-' + feat.name,

mask_zero=(feat.name in sess_feature_list)) for i, feat in

enumerate(feature_dim_dict["sparse"])}

query_emb_list = get_embedding_vec_list(sparse_embedding_dict, sparse_input, feature_dim_dict["sparse"],

sess_feature_list, sess_feature_list)

query_emb = concat_fun(query_emb_list)

deep_input_emb_list = get_embedding_vec_list(sparse_embedding_dict, sparse_input, feature_dim_dict["sparse"],

mask_feat_list=sess_feature_list)

deep_input_emb = concat_fun(deep_input_emb_list)

deep_input_emb = Flatten()(NoMask()(deep_input_emb))

tr_input = sess_interest_division(sparse_embedding_dict, user_behavior_input_dict, feature_dim_dict['sparse'],

sess_feature_list, sess_max_count, bias_encoding=bias_encoding)

Self_Attention = Transformer(att_embedding_size, att_head_num, dropout_rate=0, use_layer_norm=False,

use_positional_encoding=(not bias_encoding), seed=seed, supports_masking=True,

blinding=True)

sess_fea = sess_interest_extractor(tr_input, sess_max_count, Self_Attention)

interest_attention_layer = AttentionSequencePoolingLayer(att_hidden_units=(64, 16), weight_normalization=True,

supports_masking=False)(

[query_emb, sess_fea, user_sess_length])

lstm_outputs = BiLSTM(len(sess_feature_list) * embedding_size, layers=2, res_layers=0, dropout_rate=0.2, )(sess_fea)

lstm_attention_layer = AttentionSequencePoolingLayer(att_hidden_units=(64, 16), weight_normalization=True)(

[query_emb, lstm_outputs, user_sess_length])

deep_input_emb = Concatenate()(

[deep_input_emb, Flatten()(interest_attention_layer), Flatten()(lstm_attention_layer)])

if len(dense_input) > 0:

deep_input_emb = Concatenate()([deep_input_emb] + list(dense_input.values()))

output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, False, seed)(deep_input_emb)

output = Dense(1, use_bias=False, activation=None)(output)

output = PredictionLayer(task)(output)

sess_input_list = []

# sess_input_length_list = []

for i in range(sess_max_count):

sess_name = "sess_" + str(i)

sess_input_list.extend(get_inputs_list([user_behavior_input_dict[sess_name]]))

# sess_input_length_list.append(user_behavior_length_dict[sess_name])

model_input_list = get_inputs_list([sparse_input, dense_input]) + sess_input_list + [

user_sess_length]

model = Model(inputs=model_input_list, outputs=output)

return model

def get_input(feature_dim_dict, seq_feature_list, sess_max_count, seq_max_len):

sparse_input, dense_input = create_singlefeat_inputdict(feature_dim_dict)

user_behavior_input = {}

for idx in range(sess_max_count):

sess_input = OrderedDict()

for i, feat in enumerate(seq_feature_list):

sess_input[feat] = Input(shape=(seq_max_len,), name='seq_' + str(idx) + str(i) + '-' + feat)

user_behavior_input["sess_" + str(idx)] = sess_input

user_behavior_length = {"sess_" + str(idx): Input(shape=(1,), name='seq_length' + str(idx)) for idx in

range(sess_max_count)}

user_sess_length = Input(shape=(1,), name='sess_length')

return sparse_input, dense_input, user_behavior_input, user_behavior_length, user_sess_length

def sess_interest_division(sparse_embedding_dict, user_behavior_input_dict, sparse_fg_list, sess_feture_list,

sess_max_count,

bias_encoding=True):

tr_input = []

for i in range(sess_max_count):

sess_name = "sess_" + str(i)

keys_emb_list = get_embedding_vec_list(sparse_embedding_dict, user_behavior_input_dict[sess_name],

sparse_fg_list, sess_feture_list, sess_feture_list)

# [sparse_embedding_dict[feat](user_behavior_input_dict[sess_name][feat]) for feat in

# sess_feture_list]

keys_emb = concat_fun(keys_emb_list)

tr_input.append(keys_emb)

if bias_encoding:

tr_input = BiasEncoding(sess_max_count)(tr_input)

return tr_input

def sess_interest_extractor(tr_input, sess_max_count, TR):

tr_out = []

for i in range(sess_max_count):

tr_out.append(TR(

[tr_input[i], tr_input[i]]))

sess_fea = concat_fun(tr_out, axis=1)

return sess_fea

import os

import numpy as np

import pandas as pd

from deepctr.utils import SingleFeat

from sklearn.preprocessing import LabelEncoder, StandardScaler

from tensorflow.python.keras.preprocessing.sequence import pad_sequences

from tqdm import tqdm

from config import DSIN_SESS_COUNT, DSIN_SESS_MAX_LEN, FRAC

FRAC = FRAC

SESS_COUNT = DSIN_SESS_COUNT

def gen_sess_feature_dsin(row):

sess_count = DSIN_SESS_COUNT

sess_max_len = DSIN_SESS_MAX_LEN

sess_input_dict = {}

sess_input_length_dict = {}

for i in range(sess_count):

sess_input_dict['sess_' + str(i)] = {'cate_id': [], 'brand': []}

sess_input_length_dict['sess_' + str(i)] = 0

sess_length = 0

user, time_stamp = row[1]['user'], row[1]['time_stamp']

# sample_time = pd.to_datetime(timestamp_datetime(time_stamp ))

if user not in user_hist_session:

for i in range(sess_count):

sess_input_dict['sess_' + str(i)]['cate_id'] = [0]

sess_input_dict['sess_' + str(i)]['brand'] = [0]

sess_input_length_dict['sess_' + str(i)] = 0

sess_length = 0

else:

valid_sess_count = 0

last_sess_idx = len(user_hist_session[user]) - 1

for i in reversed(range(len(user_hist_session[user]))):

cur_sess = user_hist_session[user][i]

if cur_sess[0][2] < time_stamp:

in_sess_count = 1

for j in range(1, len(cur_sess)):

if cur_sess[j][2] < time_stamp:

in_sess_count += 1

if in_sess_count > 2:

sess_input_dict['sess_0']['cate_id'] = [e[0] for e in cur_sess[max(0,

in_sess_count - sess_max_len):in_sess_count]]

sess_input_dict['sess_0']['brand'] = [e[1] for e in

cur_sess[max(0, in_sess_count - sess_max_len):in_sess_count]]

sess_input_length_dict['sess_0'] = min(

sess_max_len, in_sess_count)

last_sess_idx = i

valid_sess_count += 1

break

for i in range(1, sess_count):

if last_sess_idx - i >= 0:

cur_sess = user_hist_session[user][last_sess_idx - i]

sess_input_dict['sess_' + str(i)]['cate_id'] = [e[0]

for e in cur_sess[-sess_max_len:]]

sess_input_dict['sess_' + str(i)]['brand'] = [e[1]

for e in cur_sess[-sess_max_len:]]

sess_input_length_dict['sess_' +

str(i)] = min(sess_max_len, len(cur_sess))

valid_sess_count += 1

else:

sess_input_dict['sess_' + str(i)]['cate_id'] = [0]

sess_input_dict['sess_' + str(i)]['brand'] = [0]

sess_input_length_dict['sess_' + str(i)] = 0

sess_length = valid_sess_count

return sess_input_dict, sess_input_length_dict, sess_length

if __name__ == "__main__":

user_hist_session = {}

FILE_NUM = len(

list(filter(lambda x: x.startswith('user_hist_session_' + str(FRAC) + '_dsin_'),

os.listdir('../sampled_data/'))))

print('total', FILE_NUM, 'files')

for i in range(FILE_NUM):

user_hist_session_ = pd.read_pickle(

'../sampled_data/user_hist_session_' + str(FRAC) + '_dsin_' + str(i) + '.pkl') # 19,34

user_hist_session.update(user_hist_session_)

del user_hist_session_

sample_sub = pd.read_pickle(

'../sampled_data/raw_sample_' + str(FRAC) + '.pkl')

index_list = []

sess_input_dict = {}

sess_input_length_dict = {}

for i in range(SESS_COUNT):

sess_input_dict['sess_' + str(i)] = {'cate_id': [], 'brand': []}

sess_input_length_dict['sess_' + str(i)] = []

sess_length_list = []

for row in tqdm(sample_sub[['user', 'time_stamp']].iterrows()):

sess_input_dict_, sess_input_length_dict_, sess_length = gen_sess_feature_dsin(

row)

# index_list.append(index)

for i in range(SESS_COUNT):

sess_name = 'sess_' + str(i)

sess_input_dict[sess_name]['cate_id'].append(

sess_input_dict_[sess_name]['cate_id'])

sess_input_dict[sess_name]['brand'].append(

sess_input_dict_[sess_name]['brand'])

sess_input_length_dict[sess_name].append(

sess_input_length_dict_[sess_name])

sess_length_list.append(sess_length)

print('done')

user = pd.read_pickle('../sampled_data/user_profile_' + str(FRAC) + '.pkl')

ad = pd.read_pickle('../sampled_data/ad_feature_enc_' + str(FRAC) + '.pkl')

user = user.fillna(-1)

user.rename(

columns={'new_user_class_level ': 'new_user_class_level'}, inplace=True)

sample_sub = pd.read_pickle(

'../sampled_data/raw_sample_' + str(FRAC) + '.pkl')

sample_sub.rename(columns={'user': 'userid'}, inplace=True)

data = pd.merge(sample_sub, user, how='left', on='userid', )

data = pd.merge(data, ad, how='left', on='adgroup_id')

sparse_features = ['userid', 'adgroup_id', 'pid', 'cms_segid', 'cms_group_id', 'final_gender_code', 'age_level',

'pvalue_level', 'shopping_level', 'occupation', 'new_user_class_level', 'campaign_id',

'customer'] # sparse feature for user and ads

dense_features = ['price'] # dense feature for user and ads

for feat in tqdm(sparse_features):

lbe = LabelEncoder() # or Hash

data[feat] = lbe.fit_transform(data[feat]) # 将不同的取值转换为对应的编号

mms = StandardScaler()

data[dense_features] = mms.fit_transform(data[dense_features])

# class SingleFeat(namedtuple('SingleFeat', ['name', 'dimension', 'hash_flag', 'dtype'])):

sparse_feature_list = [SingleFeat(feat, data[feat].nunique(

) + 1) for feat in sparse_features + ['cate_id', 'brand']]

dense_feature_list = [SingleFeat(feat, 1) for feat in dense_features]

sess_feature = ['cate_id', 'brand'] # sess feature for ad

sess_input = []

sess_input_length = []

for i in tqdm(range(SESS_COUNT)):

sess_name = 'sess_' + str(i)

for feat in sess_feature:

sess_input.append(pad_sequences(

sess_input_dict[sess_name][feat], maxlen=SESS_COUNT, padding='post'))

sess_input_length.append(sess_input_length_dict[sess_name])

model_input = [data[feat.name].values for feat in sparse_feature_list] + \

[data[feat.name].values for feat in dense_feature_list]

sess_lists = sess_input + [np.array(sess_length_list)]

model_input += sess_lists

if not os.path.exists('../model_input/'):

os.mkdir('../model_input/')

pd.to_pickle(model_input, '../model_input/dsin_input_' +

str(FRAC) + '_' + str(SESS_COUNT) + '.pkl')

pd.to_pickle(data['clk'].values, '../model_input/dsin_label_' +

str(FRAC) + '_' + str(SESS_COUNT) + '.pkl')

pd.to_pickle({'sparse': sparse_feature_list, 'dense': dense_feature_list},

'../model_input/dsin_fd_' + str(FRAC) + '_' + str(SESS_COUNT) + '.pkl')

print("gen dsin input done")

import os

import numpy as np

import pandas as pd

from sklearn.preprocessing import LabelEncoder

from config import FRAC

if __name__ == "__main__":

user = pd.read_csv('../data/user_profile.csv')

sample = pd.read_csv('../data/raw_sample.csv')

if not os.path.exists('../sampled_data/'):

os.mkdir('../sampled_data/')

if os.path.exists('../sampled_data/user_profile_' + str(FRAC) + '_.pkl') and os.path.exists(

'../sampled_data/raw_sample_' + str(FRAC) + '_.pkl'):

user_sub = pd.read_pickle(

'../sampled_data/user_profile_' + str(FRAC) + '_.pkl')

sample_sub = pd.read_pickle(

'../sampled_data/raw_sample_' + str(FRAC) + '_.pkl')

else:

if FRAC < 1.0:

user_sub = user.sample(frac=FRAC, random_state=1024)

else:

user_sub = user

sample_sub = sample.loc[sample.user.isin(user_sub.userid.unique())]

pd.to_pickle(user_sub, '../sampled_data/user_profile_' +

str(FRAC) + '.pkl')

pd.to_pickle(sample_sub, '../sampled_data/raw_sample_' +

str(FRAC) + '.pkl')

if os.path.exists('../data/behavior_log_pv.pkl'):

log = pd.read_pickle('../data/behavior_log_pv.pkl')

else:

log = pd.read_csv('../data/behavior_log.csv')

log = log.loc[log['btag'] == 'pv']

pd.to_pickle(log, '../data/behavior_log_pv.pkl')

userset = user_sub.userid.unique()

log = log.loc[log.user.isin(userset)]

# pd.to_pickle(log, '../sampled_data/behavior_log_pv_user_filter_' + str(FRAC) + '_.pkl')

ad = pd.read_csv('../data/ad_feature.csv')

ad['brand'] = ad['brand'].fillna(-1)

lbe = LabelEncoder()

# unique_cate_id = ad['cate_id'].unique()

# log = log.loc[log.cate.isin(unique_cate_id)]

unique_cate_id = np.concatenate(

(ad['cate_id'].unique(), log['cate'].unique()))

lbe.fit(unique_cate_id)

ad['cate_id'] = lbe.transform(ad['cate_id']) + 1

log['cate'] = lbe.transform(log['cate']) + 1

lbe = LabelEncoder()

# unique_brand = np.ad['brand'].unique()

# log = log.loc[log.brand.isin(unique_brand)]

unique_brand = np.concatenate(

(ad['brand'].unique(), log['brand'].unique()))

lbe.fit(unique_brand)

ad['brand'] = lbe.transform(ad['brand']) + 1

log['brand'] = lbe.transform(log['brand']) + 1

log = log.loc[log.user.isin(sample_sub.user.unique())]

log.drop(columns=['btag'], inplace=True)

log = log.loc[log['time_stamp'] > 0]

pd.to_pickle(ad, '../sampled_data/ad_feature_enc_' + str(FRAC) + '.pkl')

pd.to_pickle(

log, '../sampled_data/behavior_log_pv_user_filter_enc_' + str(FRAC) + '.pkl')

print("0_gen_sampled_data done")