电影TensorFlow推荐系统

import tensorflow as tf2 tf = tf2.compat.v1 tf.disable_v2_behavior() import tf_slim import pandas as pd from sklearn.model_selection import train_test_split import numpy as np from collections import Counter import os import pickle import re from tensorflow.python.ops import math_ops from urllib.request import urlretrieve from os.path import isfile, isdir from tqdm import tqdm import zipfile import hashlib import os import shutil from tensorflow import keras tf.enable_eager_execution() os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' def _unzip(save_path, _, database_name, data_path): """ Unzip wrapper with the same interface as _ungzip :param save_path: The path of the gzip files :param database_name: Name of database :param data_path: Path to extract to :param _: HACK - Used to have to same interface as _ungzip """ print('Extracting {}...'.format(database_name)) with zipfile.ZipFile(save_path) as zf: zf.extractall(data_path) def download_extract(database_name, data_path): """ Download and extract database :param database_name: Database name """ DATASET_ML1M = 'ml-1m' if database_name == DATASET_ML1M: url = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip' hash_code = 'c4d9eecfca2ab87c1945afe126590906' extract_path = os.path.join(data_path, 'ml-1m') save_path = os.path.join(data_path, 'ml-1m.zip') extract_fn = _unzip if os.path.exists(extract_path): print('Found {} Data'.format(database_name)) return if not os.path.exists(data_path): os.makedirs(data_path) if not os.path.exists(save_path): with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Downloading {}'.format(database_name)) as pbar: urlretrieve( url, save_path, pbar.hook) assert hashlib.md5(open(save_path, 'rb').read()).hexdigest() == hash_code, \ '{} file is corrupted. Remove the file and try again.'.format(save_path) os.makedirs(extract_path) try: extract_fn(save_path, extract_path, database_name, data_path) except Exception as err: shutil.rmtree(extract_path) # Remove extraction folder if there is an error raise err print('Done.') # Remove compressed data # os.remove(save_path) class DLProgress(tqdm): """ Handle Progress Bar while Downloading """ last_block = 0 def hook(self, block_num=1, block_size=1, total_size=None): """ A hook function that will be called once on establishment of the network connection and once after each block read thereafter. :param block_num: A count of blocks transferred so far :param block_size: Block size in bytes :param total_size: The total size of the file. This may be -1 on older FTP servers which do not return a file size in response to a retrieval request. """ self.total = total_size self.update((block_num - self.last_block) * block_size) self.last_block = block_num def load_data(): """ Load Dataset from File """ #读取User数据 users_title = ['UserID', 'Gender', 'Age', 'JobID', 'Zip-code'] users = pd.read_table('./ml-1m/users.dat', sep='::', header=None, names=users_title, engine = 'python') users = users.filter(regex='UserID|Gender|Age|JobID') users_orig = users.values #改变User数据中性别和年龄 gender_map = {'F':0, 'M':1} users['Gender'] = users['Gender'].map(gender_map) age_map = {val:ii for ii,val in enumerate(set(users['Age']))} users['Age'] = users['Age'].map(age_map) #读取Movie数据集 movies_title = ['MovieID', 'Title', 'Genres'] movies = pd.read_table('./ml-1m/movies.dat', sep='::', header=None, names=movies_title, engine = 'python') movies_orig = movies.values #将Title中的年份去掉 pattern = re.compile(r'^(.*)\((\d+)\)$') title_map = {val:pattern.match(val).group(1) for ii,val in enumerate(set(movies['Title']))} movies['Title'] = movies['Title'].map(title_map) #电影类型转数字字典 genres_set = set() for val in movies['Genres'].str.split('|'): genres_set.update(val) genres_set.add('<PAD>') genres2int = {val:ii for ii, val in enumerate(genres_set)} #将电影类型转成等长数字列表，长度是18 genres_map = {val:[genres2int[row] for row in val.split('|')] for ii,val in enumerate(set(movies['Genres']))} for key in genres_map: for cnt in range(max(genres2int.values()) - len(genres_map[key])): genres_map[key].insert(len(genres_map[key]) + cnt,genres2int['<PAD>']) movies['Genres'] = movies['Genres'].map(genres_map) #电影Title转数字字典 title_set = set() for val in movies['Title'].str.split(): title_set.update(val) title_set.add('<PAD>') title2int = {val:ii for ii, val in enumerate(title_set)} #将电影Title转成等长数字列表，长度是15 title_count = 15 title_map = {val:[title2int[row] for row in val.split()] for ii,val in enumerate(set(movies['Title']))} for key in title_map: for cnt in range(title_count - len(title_map[key])): title_map[key].insert(len(title_map[key]) + cnt,title2int['<PAD>']) movies['Title'] = movies['Title'].map(title_map) #读取评分数据集 ratings_title = ['UserID','MovieID', 'ratings', 'timestamps'] ratings = pd.read_table('./ml-1m/ratings.dat', sep='::', header=None, names=ratings_title, engine = 'python') ratings = ratings.filter(regex='UserID|MovieID|ratings') #合并三个表 data = pd.merge(pd.merge(ratings, users), movies) #将数据分成X和y两张表 target_fields = ['ratings'] features_pd, targets_pd = data.drop(target_fields, axis=1), data[target_fields] features = features_pd.values targets_values = targets_pd.values return title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig ratings = pd.read_table('./ml-1m/ratings.dat', sep='::', header=None, names=ratings_title, engine = 'python') ratings = ratings.filter(regex='UserID|MovieID|ratings') #合并三个表 data = pd.merge(pd.merge(ratings, users), movies) #将数据分成X和y两张表 target_fields = ['ratings'] features_pd, targets_pd = data.drop(target_fields, axis=1), data[target_fields] features = features_pd.values targets_values = targets_pd.values return title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig import os import pickle def save_params(params): """ Save parameters to file """ pickle.dump(params, open('params.p', 'wb')) def load_params(): """ Load parameters from file """ return pickle.load(open('params.p', mode='rb')) def get_inputs(): uid = tf.compat.v1.placeholder(tf.int32, [None, 1], name="uid") user_gender = tf.compat.v1.placeholder(tf.int32, [None, 1], name="user_gender") user_age = tf.compat.v1.placeholder(tf.int32, [None, 1], name="user_age") user_job = tf.compat.v1.placeholder(tf.int32, [None, 1], name="user_job") movie_id = tf.compat.v1.placeholder(tf.int32, [None, 1], name="movie_id") movie_categories = tf.compat.v1.placeholder(tf.int32, [None, 18], name="movie_categories") movie_titles = tf.compat.v1.placeholder(tf.int32, [None, 15], name="movie_titles") targets = tf.compat.v1.placeholder(tf.int32, [None, 1], name="targets") LearningRate = tf.compat.v1.placeholder(tf.float32, name = "Learn