import os
import urllib.request
import zipfile
from pprint import pprint
import numpy as np
import tensorflow as tf
import keras as k
def set_session(device_count=None, seed=0):
gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)
if device_count is not None:
config = tf.compat.v1.ConfigProto(gpu_options=gpu_options, device_count=device_count)
else:
config = tf.compat.v1.ConfigProto(gpu_options=gpu_options)
sess = tf.compat.v1.Session(config=config)
tf.compat.v1.keras.backend.set_session(sess)
tf.compat.v1.set_random_seed(seed)
return sess
def transform(idxs):
return [idxs[:, i] for i in range(idxs.shape[1])]
def create_wsdream(shape, rank, nc):
inputs = [k.Input(shape=(1,), dtype="int32") for i in range(len(shape))]
embeds = [k.layers.Embedding(output_dim=rank, input_dim=shape[i])(inputs[i]) for i in range(len(shape))]
x = k.layers.Concatenate(axis=1)(embeds)
x = k.layers.Reshape(target_shape=(rank, len(shape), 1))(x)
x = k.layers.Conv2D(
nc,
kernel_size=(1, len(shape)),
activation='relu',
padding='valid'
)(x)
x = k.layers.Flatten()(x)
x = k.layers.Dense(nc, activation="relu")(x)
output_latency = k.layers.Dense(1, activation='relu', name='output_latency')(x)
output_throughput = k.layers.Dense(1, activation='relu', name='output_throughput')(x)
model = k.Model(inputs=inputs, outputs=[output_latency, output_throughput])
return model
# 计算均方绝对百分比误差
def mape_keras(y_true, y_pred, threshold=0.1):
v = k.backend.clip(k.backend.abs(y_true), threshold, None)
diff = k.backend.abs((y_true - y_pred) / v)
return 100.0 * k.backend.mean(diff, axis=-1)
# 计算均方绝对误差
def mae(y_true, y_pred):
return np.mean(np.abs(y_pred - y_true))
# 计算均方根误差
def rmse(y_true, y_pred):
return np.sqrt(np.mean(np.square(y_pred - y_true)))
# 计算平均绝对百分比误差
def mape(y_true, y_pred, threshold=0.1):
v = np.clip(np.abs(y_true), threshold, None)
diff = np.abs((y_true - y_pred) / v)
return 100.0 * np.mean(diff, axis=-1).mean()
def get_metrics(model, test_feature, truth_target, batch_size=1024):
pred_target_latency, pred_target_throughput = model.predict(test_feature, batch_size=batch_size, verbose=1)
pred_target_latency = pred_target_latency.ravel()
pred_target_throughput = pred_target_throughput.ravel()
return {
'均方根误差(rmse)': float(rmse(truth_target[0], pred_target_latency)),
# '平均绝对百分比误差(mape)': float(mape(y, yp)),
'均方绝对误差(mae)': float(mae(truth_target[1], pred_target_throughput))
}
def downloadset(url, folder):
# 下载数据
data_folder = folder
if not os.path.exists(data_folder):
url = url
filedata = urllib.request.urlopen(url)
with open('wsdream_dataset2.zip', 'wb') as f:
f.write(filedata.read())
with zipfile.ZipFile('wsdream_dataset2.zip', 'r') as f:
f.extractall('.')
return data_folder
if __name__ == '__main__':
# 下载数据集
url = 'https://zenodo.org/record/1133476/files/wsdream_dataset2.zip?download=1'
folder = 'dataset2'
data_folder = downloadset(url, folder)
# 加载数据
# 用户ID |服务ID |时间片ID |响应时间(秒)
# [142, 4500, 64, 20001]
rt_data = np.loadtxt(os.path.join(data_folder, "rtdata.txt"))
# 用户ID |服务ID |时间片ID |吞吐量(kbps)
# [142, 4500, 64, 20001]
tp_data = np.loadtxt(os.path.join(data_folder, "tpdata.txt"))
# 数据处理
pre_data = np.column_stack((rt_data, tp_data[:, 3]))
indices = np.arange(pre_data.shape[0])
np.random.shuffle(indices)
pre_data = pre_data[indices]
# 特征数据
feature_data = pre_data[:, :3]
# 目标标签
target_latency = pre_data[:, 3]
target_throughput = pre_data[:, 4]
# 划分数据集
split_ratio = 0.9
num_train_samples = int(pre_data.shape[0] * split_ratio)
# print(num_train_samples)
train_feature_data, test_feature_data = feature_data[:num_train_samples], feature_data[num_train_samples:]
# print(train_feature_data)
train_target_latency_data, test_target_latency_data = target_latency[:num_train_samples], target_latency[num_train_samples:]
# print(train_target_latency_data)
train_target_throughput_data, test_target_throughput_data = target_throughput[:num_train_samples], target_throughput[num_train_samples:]
# 设置参数
lr = 1e-5
# 模型嵌入维度
rank = 40
# 卷积层数量
nc = rank
epochs = 10
batch_size = 256
seed = 3
verbose = 1
set_session(device_count={"GPU": 0}, seed=seed)
optim = k.optimizers.legacy.Adam(learning_rate=lr)
shapes = [142, 4500, 64]
# column = transform(rt_data)
# for i in range(len(column)):
# unique_elements = np.unique(column[i])
# num_unique_elements = len(unique_elements)
# shapes.append(num_unique_elements)
# shapes = [len(np.unique(transform(rt_data)[i])) for i in range(len(transform(tp_data)))]
# print(shapes)
# print(np.delete(rt_data, 3, axis=1))
print('----------train-----------')
model = create_wsdream(shapes, rank, nc)
model.compile(optim, loss={'output_latency': 'mean_squared_error',
'output_throughput': 'mean_squared_error'},
metrics={'output_latency': 'mae', 'output_throughput': 'mae'})
hists_tp = model.fit(
x=transform(train_feature_data),
y={'output_latency': train_target_latency_data, 'output_throughput': train_target_throughput_data},
verbose=verbose,
epochs=epochs,
batch_size=batch_size,
validation_split=0.2,
shuffle=True,
validation_freq=1,
callbacks=[k.callbacks.EarlyStopping(
monitor='val_loss',
patience=10,
restore_best_weights=True
)]
)
tr_info = get_metrics(model, transform(train_feature_data),[train_target_latency_data, train_target_throughput_data])
te_info = get_metrics(model, transform(test_feature_data), [test_target_latency_data, test_target_throughput_data])
pprint({'tr_train': tr_info, 'te_train': te_info})
----------train-----------
2023-10-05 07:08:32.335982: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1639] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 13332 MB memory: -> device: 0, name: NVIDIA A16, pci bus id: 0000:14:00.0, compute capability: 8.6
Epoch 1/10
2023-10-05 07:08:38.269162: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:432] Loaded cuDNN version 8900
2023-10-05 07:08:40.594402: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:606] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
85184/85184 [==============================] - 371s 4ms/step - loss: 1834.1807 - output_latency_loss: 36.8099 - output_throughput_loss: 1797.3798 - output_latency_mae: 4.1002 - output_throughput_mae: 8.8414 - val_loss: 1417.3331 - val_output_latency_loss: 34.3290 - val_output_throughput_loss: 1383.0068 - val_output_latency_mae: 3.9667 - val_output_throughput_mae: 7.0288
Epoch 2/10
85184/85184 [==============================] - 362s 4ms/step - loss: 1310.9064 - output_latency_loss: 26.8001 - output_throughput_loss: 1284.1140 - output_latency_mae: 3.2583 - output_throughput_mae: 6.9711 - val_loss: 1327.2600 - val_output_latency_loss: 22.1980 - val_output_throughput_loss: 1305.0654 - val_output_latency_mae: 2.7652 - val_output_throughput_mae: 6.9545
Epoch 3/10
85184/85184 [==============================] - 364s 4ms/step - loss: 1275.5210 - output_latency_loss: 20.9113 - output_throughput_loss: 1254.6027 - output_latency_mae: 2.6034 - output_throughput_mae: 6.9322 - val_loss: 1315.2084 - val_output_latency_loss: 20.0041 - val_output_throughput_loss: 1295.2047 - val_output_latency_mae: 2.5084 - val_output_throughput_mae: 6.9197
Epoch 4/10
85184/85184 [==============================] - 364s 4ms/step - loss: 1268.3987 - output_latency_loss: 19.2282 - output_throughput_loss: 1249.1615 - output_latency_mae: 2.4463 - output_throughput_mae: 6.8742 - val_loss: 1308.3456 - val_output_latency_loss: 18.4484 - val_output_throughput_loss: 1289.8993 - val_output_latency_mae: 2.3776 - val_output_throughput_mae: 6.8397
Epoch 5/10
85184/85184 [==============================] - 364s 4ms/step - loss: 1263.1598 - output_latency_loss: 17.8324 - output_throughput_loss: 1245.3314 - output_latency_mae: 2.3079 - output_throughput_mae: 6.8044 - val_loss: 1305.6429 - val_output_latency_loss: 17.3143 - val_output_throughput_loss: 1288.3311 - val_output_latency_mae: 2.2478 - val_output_throughput_mae: 6.8194
Epoch 6/10
85184/85184 [==============================] - 365s 4ms/step - loss: 1261.4520 - output_latency_loss: 16.9506 - output_throughput_loss: 1244.4976 - output_latency_mae: 2.1852 - output_throughput_mae: 6.8004 - val_loss: 1304.4865 - val_output_latency_loss: 16.6423 - val_output_throughput_loss: 1287.8406 - val_output_latency_mae: 2.1503 - val_output_throughput_mae: 6.8022
Epoch 7/10
85184/85184 [==============================] - 363s 4ms/step - loss: 1260.5026 - output_latency_loss: 16.4530 - output_throughput_loss: 1244.0607 - output_latency_mae: 2.1177 - output_throughput_mae: 6.8005 - val_loss: 1303.6454 - val_output_latency_loss: 16.2684 - val_output_throughput_loss: 1287.3763 - val_output_latency_mae: 2.0922 - val_output_throughput_mae: 6.8159
Epoch 8/10
85184/85184 [==============================] - 364s 4ms/step - loss: 1260.1501 - output_latency_loss: 16.1473 - output_throughput_loss: 1244.0028 - output_latency_mae: 2.0822 - output_throughput_mae: 6.7979 - val_loss: 1303.8026 - val_output_latency_loss: 16.0234 - val_output_throughput_loss: 1287.7805 - val_output_latency_mae: 2.0579 - val_output_throughput_mae: 6.8028
Epoch 9/10
85184/85184 [==============================] - 365s 4ms/step - loss: 1259.7853 - output_latency_loss: 15.9269 - output_throughput_loss: 1243.8567 - output_latency_mae: 2.0630 - output_throughput_mae: 6.7995 - val_loss: 1302.9790 - val_output_latency_loss: 15.8171 - val_output_throughput_loss: 1287.1680 - val_output_latency_mae: 2.0541 - val_output_throughput_mae: 6.8110
Epoch 10/10
85184/85184 [==============================] - 365s 4ms/step - loss: 1259.6364 - output_latency_loss: 15.7070 - output_throughput_loss: 1243.9320 - output_latency_mae: 2.0483 - output_throughput_mae: 6.7933 - val_loss: 1302.3773 - val_output_latency_loss: 15.5923 - val_output_throughput_loss: 1286.7860 - val_output_latency_mae: 2.0361 - val_output_throughput_mae: 6.8297
26620/26620 [==============================] - 45s 2ms/step
{'te_train': {'均方根误差(rmse)': 3.955129854560439,
'均方绝对误差(mae)': 6.8050018924910285},
'tr_train': {'均方根误差(rmse)': 3.9485392931711623,
'均方绝对误差(mae)': 6.813898810378947}}