## Please consider to cite our paper those results helps you, thanks:
```
================================================
FILE: preprocessing/0rating2inter.ipynb
================================================
{
"cells": [
{
"cell_type": "markdown",
"source": [
"# 从ratings_Sports_and_Outdoors.csv文件中提取U-I交互图, 5-core后重新编号\n",
"- Extracting U-I interactions and performing 5-core, re-indexing\n",
"- dataset located at: http://jmcauley.ucsd.edu/data/amazon/links.html, rating only file in \"Small\" subsets for experimentation"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os, csv\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"os.chdir('/home/enoche/MMRec/Sports14')\n",
"os.getcwd()"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n",
"is_executing": true
}
}
},
{
"cell_type": "markdown",
"source": [
"## 先5-core过滤\n",
"## 5-core filtering"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 3,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (3268695, 4)\n"
]
},
{
"data": {
"text/plain": " userID itemID rating timestamp\n0 A3PMSRCL80KSA1 0000031852 4.0 1388275200\n1 A1SNLWGLFXD70K 0000031852 4.0 1392940800\n2 A1KJ4CVG87QW09 0000031852 4.0 1389657600\n3 AA9ITO6ZLZW6 0000031852 5.0 1399507200\n4 APJ5ULJ1RMZ4 0000031852 1.0 1398556800",
"text/html": "\n\n\n \n
\n
"
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = pd.read_csv('ratings_Sports_and_Outdoors.csv', names=['userID', 'itemID', 'rating', 'timestamp'], header=None)\n",
"print(f'shape: {df.shape}')\n",
"df[:5]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 4,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"After dropped: (3268695, 4)\n"
]
},
{
"data": {
"text/plain": " userID itemID rating timestamp\n0 A3PMSRCL80KSA1 0000031852 4.0 1388275200\n1 A1SNLWGLFXD70K 0000031852 4.0 1392940800\n2 A1KJ4CVG87QW09 0000031852 4.0 1389657600",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \n
|---|---|---|---|---|
| 0 | \nA3PMSRCL80KSA1 | \n0000031852 | \n4.0 | \n1388275200 | \n
| 1 | \nA1SNLWGLFXD70K | \n0000031852 | \n4.0 | \n1392940800 | \n
| 2 | \nA1KJ4CVG87QW09 | \n0000031852 | \n4.0 | \n1389657600 | \n
| 3 | \nAA9ITO6ZLZW6 | \n0000031852 | \n5.0 | \n1399507200 | \n
| 4 | \nAPJ5ULJ1RMZ4 | \n0000031852 | \n1.0 | \n1398556800 | \n
\n\n\n \n
\n
"
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"k_core = 5\n",
"learner_id, course_id, tmstmp_str = 'userID', 'itemID', 'timestamp'\n",
"\n",
"df.dropna(subset=[learner_id, course_id, tmstmp_str], inplace=True)\n",
"df.drop_duplicates(subset=[learner_id, course_id, tmstmp_str], inplace=True)\n",
"print(f'After dropped: {df.shape}')\n",
"df[:3]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 5,
"outputs": [],
"source": [
"from collections import Counter\n",
"import numpy as np\n",
"\n",
"min_u_num, min_i_num = 5, 5\n",
"\n",
"def get_illegal_ids_by_inter_num(df, field, max_num=None, min_num=None):\n",
" if field is None:\n",
" return set()\n",
" if max_num is None and min_num is None:\n",
" return set()\n",
"\n",
" max_num = max_num or np.inf\n",
" min_num = min_num or -1\n",
"\n",
" ids = df[field].values\n",
" inter_num = Counter(ids)\n",
" ids = {id_ for id_ in inter_num if inter_num[id_] < min_num or inter_num[id_] > max_num}\n",
" print(f'{len(ids)} illegal_ids_by_inter_num, field={field}')\n",
"\n",
" return ids\n",
"\n",
"\n",
"def filter_by_k_core(df):\n",
" while True:\n",
" ban_users = get_illegal_ids_by_inter_num(df, field=learner_id, max_num=None, min_num=min_u_num)\n",
" ban_items = get_illegal_ids_by_inter_num(df, field=course_id, max_num=None, min_num=min_i_num)\n",
" if len(ban_users) == 0 and len(ban_items) == 0:\n",
" return\n",
"\n",
" dropped_inter = pd.Series(False, index=df.index)\n",
" if learner_id:\n",
" dropped_inter |= df[learner_id].isin(ban_users)\n",
" if course_id:\n",
" dropped_inter |= df[course_id].isin(ban_items)\n",
" print(f'{len(dropped_inter)} dropped interactions')\n",
" df.drop(df.index[dropped_inter], inplace=True)\n",
"\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## k-core"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 6,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1906153 illegal_ids_by_inter_num, field=userID\n",
"376127 illegal_ids_by_inter_num, field=itemID\n",
"3268695 dropped interactions\n",
"22213 illegal_ids_by_inter_num, field=userID\n",
"54919 illegal_ids_by_inter_num, field=itemID\n",
"589029 dropped interactions\n",
"18323 illegal_ids_by_inter_num, field=userID\n",
"3743 illegal_ids_by_inter_num, field=itemID\n",
"422478 dropped interactions\n",
"2298 illegal_ids_by_inter_num, field=userID\n",
"4388 illegal_ids_by_inter_num, field=itemID\n",
"349749 dropped interactions\n",
"3331 illegal_ids_by_inter_num, field=userID\n",
"639 illegal_ids_by_inter_num, field=itemID\n",
"326238 dropped interactions\n",
"579 illegal_ids_by_inter_num, field=userID\n",
"1012 illegal_ids_by_inter_num, field=itemID\n",
"311188 dropped interactions\n",
"897 illegal_ids_by_inter_num, field=userID\n",
"169 illegal_ids_by_inter_num, field=itemID\n",
"305054 dropped interactions\n",
"155 illegal_ids_by_inter_num, field=userID\n",
"308 illegal_ids_by_inter_num, field=itemID\n",
"300866 dropped interactions\n",
"301 illegal_ids_by_inter_num, field=userID\n",
"47 illegal_ids_by_inter_num, field=itemID\n",
"299031 dropped interactions\n",
"50 illegal_ids_by_inter_num, field=userID\n",
"79 illegal_ids_by_inter_num, field=itemID\n",
"297646 dropped interactions\n",
"87 illegal_ids_by_inter_num, field=userID\n",
"11 illegal_ids_by_inter_num, field=itemID\n",
"297132 dropped interactions\n",
"16 illegal_ids_by_inter_num, field=userID\n",
"24 illegal_ids_by_inter_num, field=itemID\n",
"296741 dropped interactions\n",
"24 illegal_ids_by_inter_num, field=userID\n",
"1 illegal_ids_by_inter_num, field=itemID\n",
"296581 dropped interactions\n",
"1 illegal_ids_by_inter_num, field=userID\n",
"8 illegal_ids_by_inter_num, field=itemID\n",
"296481 dropped interactions\n",
"8 illegal_ids_by_inter_num, field=userID\n",
"0 illegal_ids_by_inter_num, field=itemID\n",
"296445 dropped interactions\n",
"0 illegal_ids_by_inter_num, field=userID\n",
"5 illegal_ids_by_inter_num, field=itemID\n",
"296413 dropped interactions\n",
"5 illegal_ids_by_inter_num, field=userID\n",
"0 illegal_ids_by_inter_num, field=itemID\n",
"296393 dropped interactions\n",
"0 illegal_ids_by_inter_num, field=userID\n",
"3 illegal_ids_by_inter_num, field=itemID\n",
"296373 dropped interactions\n",
"4 illegal_ids_by_inter_num, field=userID\n",
"0 illegal_ids_by_inter_num, field=itemID\n",
"296361 dropped interactions\n",
"0 illegal_ids_by_inter_num, field=userID\n",
"1 illegal_ids_by_inter_num, field=itemID\n",
"296345 dropped interactions\n",
"1 illegal_ids_by_inter_num, field=userID\n",
"0 illegal_ids_by_inter_num, field=itemID\n",
"296341 dropped interactions\n",
"0 illegal_ids_by_inter_num, field=userID\n",
"0 illegal_ids_by_inter_num, field=itemID\n",
"k-core shape: (296337, 4)\n",
"shape after k-core: (296337, 4)\n"
]
},
{
"data": {
"text/plain": " userID itemID rating timestamp\n564 AIXZKN4ACSKI 1881509818 5.0 1390694400\n565 A1L5P841VIO02V 1881509818 5.0 1328140800",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \n
|---|---|---|---|---|
| 0 | \nA3PMSRCL80KSA1 | \n0000031852 | \n4.0 | \n1388275200 | \n
| 1 | \nA1SNLWGLFXD70K | \n0000031852 | \n4.0 | \n1392940800 | \n
| 2 | \nA1KJ4CVG87QW09 | \n0000031852 | \n4.0 | \n1389657600 | \n
\n\n\n \n
\n
"
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"filter_by_k_core(df)\n",
"print(f'k-core shape: {df.shape}')\n",
"print(f'shape after k-core: {df.shape}')\n",
"df[:2]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## Re-index"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 7,
"outputs": [],
"source": [
"df.reset_index(drop=True, inplace=True)"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 8,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mapping dumped...\n"
]
}
],
"source": [
"\n",
"i_mapping_file = 'i_id_mapping.csv'\n",
"u_mapping_file = 'u_id_mapping.csv'\n",
"\n",
"splitting = [0.8, 0.1, 0.1]\n",
"uid_field, iid_field = learner_id, course_id\n",
"\n",
"uni_users = pd.unique(df[uid_field])\n",
"uni_items = pd.unique(df[iid_field])\n",
"\n",
"# start from 0\n",
"u_id_map = {k: i for i, k in enumerate(uni_users)}\n",
"i_id_map = {k: i for i, k in enumerate(uni_items)}\n",
"\n",
"df[uid_field] = df[uid_field].map(u_id_map)\n",
"df[iid_field] = df[iid_field].map(i_id_map)\n",
"df[uid_field] = df[uid_field].astype(int)\n",
"df[iid_field] = df[iid_field].astype(int)\n",
"\n",
"# dump\n",
"rslt_dir = './'\n",
"u_df = pd.DataFrame(list(u_id_map.items()), columns=['user_id', 'userID'])\n",
"i_df = pd.DataFrame(list(i_id_map.items()), columns=['asin', 'itemID'])\n",
"\n",
"u_df.to_csv(os.path.join(rslt_dir, u_mapping_file), sep='\\t', index=False)\n",
"i_df.to_csv(os.path.join(rslt_dir, i_mapping_file), sep='\\t', index=False)\n",
"print(f'mapping dumped...')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"\n",
"# =========2. splitting\n",
"print(f'splitting ...')\n",
"tot_ratio = sum(splitting)\n",
"# remove 0.0 in ratios\n",
"ratios = [i for i in splitting if i > .0]\n",
"ratios = [_ / tot_ratio for _ in ratios]\n",
"split_ratios = np.cumsum(ratios)[:-1]\n",
"\n",
"#df[tmstmp_str] = df[tmstmp_str].map(lambda x: datetime.strptime(x, \"%Y-%m-%dT%H:%M:%SZ\"))\n",
"split_ratios"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n",
"is_executing": true
}
}
},
{
"cell_type": "code",
"execution_count": 10,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"columns: Index(['userID', 'itemID', 'rating', 'timestamp', 'x_label'], dtype='object')\n"
]
},
{
"data": {
"text/plain": " userID itemID rating timestamp x_label\n1 1 0 5.0 1328140800 0\n2 2 0 4.0 1330387200 0\n3 3 0 4.0 1328400000 0\n4 4 0 4.0 1366675200 0\n5 5 0 5.0 1351814400 0",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \n
|---|---|---|---|---|
| 564 | \nAIXZKN4ACSKI | \n1881509818 | \n5.0 | \n1390694400 | \n
| 565 | \nA1L5P841VIO02V | \n1881509818 | \n5.0 | \n1328140800 | \n
\n\n\n \n
\n
"
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ts_id = 'timestamp'\n",
"\n",
"split_timestamps = list(np.quantile(df[ts_id], split_ratios))\n",
"# get df training dataset unique users/items\n",
"df_train = df.loc[df[ts_id] < split_timestamps[0]].copy()\n",
"df_val = df.loc[(split_timestamps[0] <= df[ts_id]) & (df[ts_id] < split_timestamps[1])].copy()\n",
"df_test = df.loc[(split_timestamps[1] <= df[ts_id])].copy()\n",
"\n",
"x_label, rslt_file = 'x_label', 'sports14-indexed.inter'\n",
"df_train[x_label] = 0\n",
"df_val[x_label] = 1\n",
"df_test[x_label] = 2\n",
"temp_df = pd.concat([df_train, df_val, df_test])\n",
"temp_df = temp_df[[learner_id, course_id, 'rating', ts_id, x_label]]\n",
"print(f'columns: {temp_df.columns}')\n",
"\n",
"temp_df.columns = [learner_id, course_id, 'rating', ts_id, x_label]\n",
"\n",
"temp_df.to_csv(os.path.join(rslt_dir, rslt_file), sep='\\t', index=False)\n",
"temp_df[:5]\n",
"#print('done!')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "markdown",
"source": [],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "markdown",
"source": [
"## Reload"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 11,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (296337, 5)\n"
]
},
{
"data": {
"text/plain": " userID itemID rating timestamp x_label\n0 1 0 5.0 1328140800 0\n1 2 0 4.0 1330387200 0\n2 3 0 4.0 1328400000 0\n3 4 0 4.0 1366675200 0",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \nx_label | \n
|---|---|---|---|---|---|
| 1 | \n1 | \n0 | \n5.0 | \n1328140800 | \n0 | \n
| 2 | \n2 | \n0 | \n4.0 | \n1330387200 | \n0 | \n
| 3 | \n3 | \n0 | \n4.0 | \n1328400000 | \n0 | \n
| 4 | \n4 | \n0 | \n4.0 | \n1366675200 | \n0 | \n
| 5 | \n5 | \n0 | \n5.0 | \n1351814400 | \n0 | \n
\n\n\n \n
\n
"
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"indexed_df = pd.read_csv(rslt_file, sep='\\t')\n",
"print(f'shape: {indexed_df.shape}')\n",
"indexed_df[:4]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 12,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# of unique learners: 35598\n",
"# of unique courses: 18357\n",
"min/max of unique learners: 0/35597\n",
"min/max of unique courses: 0/18356\n"
]
}
],
"source": [
"u_uni = indexed_df[learner_id].unique()\n",
"c_uni = indexed_df[course_id].unique()\n",
"\n",
"print(f'# of unique learners: {len(u_uni)}')\n",
"print(f'# of unique courses: {len(c_uni)}')\n",
"\n",
"print('min/max of unique learners: {0}/{1}'.format(min(u_uni), max(u_uni)))\n",
"print('min/max of unique courses: {0}/{1}'.format(min(c_uni), max(c_uni)))\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
================================================
FILE: preprocessing/1splitting.ipynb
================================================
{
"cells": [
{
"cell_type": "markdown",
"source": [
"# 基于rating2inter.ipynb生成的5-core交互图，Train/Validation/Test data splitting\n",
"- Based on generated interactions, perform data splitting\n"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os, csv\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"os.chdir('/home/enoche/MMRec/Sports14')\n",
"os.getcwd()"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n",
"is_executing": true
}
}
},
{
"cell_type": "markdown",
"source": [
"## 直接加载现成的, Load interactions"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 3,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (296337, 5)\n"
]
},
{
"data": {
"text/plain": " userID itemID rating timestamp x_label\n0 1 0 5.0 1328140800 0\n1 2 0 4.0 1330387200 0\n2 3 0 4.0 1328400000 0\n3 4 0 4.0 1366675200 0",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \nx_label | \n
|---|---|---|---|---|---|
| 0 | \n1 | \n0 | \n5.0 | \n1328140800 | \n0 | \n
| 1 | \n2 | \n0 | \n4.0 | \n1330387200 | \n0 | \n
| 2 | \n3 | \n0 | \n4.0 | \n1328400000 | \n0 | \n
| 3 | \n4 | \n0 | \n4.0 | \n1366675200 | \n0 | \n
\n\n\n \n
\n
"
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"rslt_file = 'sports14-indexed.inter'\n",
"df = pd.read_csv(rslt_file, sep='\\t')\n",
"print(f'shape: {df.shape}')\n",
"df[:4]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 4,
"outputs": [],
"source": [
"import random\n",
"import numpy as np"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 4,
"outputs": [
{
"data": {
"text/plain": " userID itemID rating timestamp x_label\n154667 0 11981 2.0 1390694400 1\n295557 0 15852 5.0 1390694400 1\n189316 0 17787 3.0 1391990400 2\n151302 0 0 5.0 1390694400 1\n1820 0 3369 5.0 1405123200 2\n60040 0 13372 5.0 1391990400 2\n199192 0 5458 5.0 1405123200 2\n163234 0 3327 3.0 1391990400 2\n60837 1 2322 5.0 1337212800 0\n233786 1 4123 5.0 1354838400 0\n163460 1 14212 5.0 1368230400 0\n206628 1 1542 4.0 1302220800 0\n261633 1 8802 4.0 1368230400 0\n99658 1 9198 5.0 1318377600 0\n268935 1 7215 5.0 1285372800 0\n77956 1 13468 5.0 1328140800 0\n105444 1 2374 5.0 1391558400 1\n237889 1 7169 5.0 1302220800 0\n173295 1 6677 5.0 1318377600 0\n50074 1 15278 5.0 1344902400 0",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \nx_label | \n
|---|---|---|---|---|---|
| 0 | \n1 | \n0 | \n5.0 | \n1328140800 | \n0 | \n
| 1 | \n2 | \n0 | \n4.0 | \n1330387200 | \n0 | \n
| 2 | \n3 | \n0 | \n4.0 | \n1328400000 | \n0 | \n
| 3 | \n4 | \n0 | \n4.0 | \n1366675200 | \n0 | \n
\n\n\n \n
\n
"
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"df = df.sample(frac=1).reset_index(drop=True)\n",
"\n",
"df.sort_values(by=['userID'], inplace=True)\n",
"df[:20]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"uid_field, iid_field = 'userID', 'itemID'\n",
"\n",
"uid_freq = df.groupby(uid_field)[iid_field]\n",
"u_i_dict = {}\n",
"for u, u_ls in uid_freq:\n",
" u_i_dict[u] = list(u_ls)\n",
"u_i_dict"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n",
"is_executing": true
}
}
},
{
"cell_type": "code",
"execution_count": 6,
"outputs": [
{
"data": {
"text/plain": "[0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 1,\n 2,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 1,\n 2,\n 2,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 1,\n 1,\n 2,\n 2,\n 0,\n 0,\n 0,\n 1,\n 2,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 1,\n 2,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0]"
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_label = []\n",
"u_ids_sorted = sorted(u_i_dict.keys())\n",
"\n",
"for u in u_ids_sorted:\n",
" items = u_i_dict[u]\n",
" n_items = len(items)\n",
" if n_items < 10:\n",
" tmp_ls = [0] * (n_items - 2) + [1] + [2]\n",
" else:\n",
" val_test_len = int(n_items * 0.2)\n",
" train_len = n_items - val_test_len\n",
" val_len = val_test_len // 2\n",
" test_len = val_test_len - val_len\n",
" tmp_ls = [0] * train_len + [1] * val_len + [2] * test_len\n",
" new_label.extend(tmp_ls)\n",
"\n",
"new_label[:100]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 7,
"outputs": [
{
"data": {
"text/plain": " userID itemID rating timestamp x_label\n154667 0 11981 2.0 1390694400 0\n295557 0 15852 5.0 1390694400 0\n189316 0 17787 3.0 1391990400 0\n151302 0 0 5.0 1390694400 0\n1820 0 3369 5.0 1405123200 0\n60040 0 13372 5.0 1391990400 0\n199192 0 5458 5.0 1405123200 1\n163234 0 3327 3.0 1391990400 2\n60837 1 2322 5.0 1337212800 0\n233786 1 4123 5.0 1354838400 0\n163460 1 14212 5.0 1368230400 0\n206628 1 1542 4.0 1302220800 0\n261633 1 8802 4.0 1368230400 0\n99658 1 9198 5.0 1318377600 0\n268935 1 7215 5.0 1285372800 0\n77956 1 13468 5.0 1328140800 0\n105444 1 2374 5.0 1391558400 0\n237889 1 7169 5.0 1302220800 0\n173295 1 6677 5.0 1318377600 0\n50074 1 15278 5.0 1344902400 0",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \nx_label | \n
|---|---|---|---|---|---|
| 154667 | \n0 | \n11981 | \n2.0 | \n1390694400 | \n1 | \n
| 295557 | \n0 | \n15852 | \n5.0 | \n1390694400 | \n1 | \n
| 189316 | \n0 | \n17787 | \n3.0 | \n1391990400 | \n2 | \n
| 151302 | \n0 | \n0 | \n5.0 | \n1390694400 | \n1 | \n
| 1820 | \n0 | \n3369 | \n5.0 | \n1405123200 | \n2 | \n
| 60040 | \n0 | \n13372 | \n5.0 | \n1391990400 | \n2 | \n
| 199192 | \n0 | \n5458 | \n5.0 | \n1405123200 | \n2 | \n
| 163234 | \n0 | \n3327 | \n3.0 | \n1391990400 | \n2 | \n
| 60837 | \n1 | \n2322 | \n5.0 | \n1337212800 | \n0 | \n
| 233786 | \n1 | \n4123 | \n5.0 | \n1354838400 | \n0 | \n
| 163460 | \n1 | \n14212 | \n5.0 | \n1368230400 | \n0 | \n
| 206628 | \n1 | \n1542 | \n4.0 | \n1302220800 | \n0 | \n
| 261633 | \n1 | \n8802 | \n4.0 | \n1368230400 | \n0 | \n
| 99658 | \n1 | \n9198 | \n5.0 | \n1318377600 | \n0 | \n
| 268935 | \n1 | \n7215 | \n5.0 | \n1285372800 | \n0 | \n
| 77956 | \n1 | \n13468 | \n5.0 | \n1328140800 | \n0 | \n
| 105444 | \n1 | \n2374 | \n5.0 | \n1391558400 | \n1 | \n
| 237889 | \n1 | \n7169 | \n5.0 | \n1302220800 | \n0 | \n
| 173295 | \n1 | \n6677 | \n5.0 | \n1318377600 | \n0 | \n
| 50074 | \n1 | \n15278 | \n5.0 | \n1344902400 | \n0 | \n
\n\n\n \n
\n
"
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df['x_label'] = new_label\n",
"df[:20]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 12,
"outputs": [
{
"data": {
"text/plain": "'beauty14-indexed'"
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"rslt_file[:-6]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 8,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"done!!!\n"
]
}
],
"source": [
"new_labeled_file = rslt_file[:-6] + '-v4.inter'\n",
"df.to_csv(os.path.join('./', new_labeled_file), sep='\\t', index=False)\n",
"print('done!!!')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "markdown",
"source": [],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "markdown",
"source": [
"## Reload"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 9,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (296337, 5)\n"
]
},
{
"data": {
"text/plain": " userID itemID rating timestamp x_label\n0 0 11981 2.0 1390694400 0\n1 0 15852 5.0 1390694400 0\n2 0 17787 3.0 1391990400 0\n3 0 0 5.0 1390694400 0\n4 0 3369 5.0 1405123200 0\n5 0 13372 5.0 1391990400 0\n6 0 5458 5.0 1405123200 1\n7 0 3327 3.0 1391990400 2\n8 1 2322 5.0 1337212800 0\n9 1 4123 5.0 1354838400 0\n10 1 14212 5.0 1368230400 0\n11 1 1542 4.0 1302220800 0\n12 1 8802 4.0 1368230400 0\n13 1 9198 5.0 1318377600 0\n14 1 7215 5.0 1285372800 0\n15 1 13468 5.0 1328140800 0\n16 1 2374 5.0 1391558400 0\n17 1 7169 5.0 1302220800 0\n18 1 6677 5.0 1318377600 0\n19 1 15278 5.0 1344902400 0",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \nx_label | \n
|---|---|---|---|---|---|
| 154667 | \n0 | \n11981 | \n2.0 | \n1390694400 | \n0 | \n
| 295557 | \n0 | \n15852 | \n5.0 | \n1390694400 | \n0 | \n
| 189316 | \n0 | \n17787 | \n3.0 | \n1391990400 | \n0 | \n
| 151302 | \n0 | \n0 | \n5.0 | \n1390694400 | \n0 | \n
| 1820 | \n0 | \n3369 | \n5.0 | \n1405123200 | \n0 | \n
| 60040 | \n0 | \n13372 | \n5.0 | \n1391990400 | \n0 | \n
| 199192 | \n0 | \n5458 | \n5.0 | \n1405123200 | \n1 | \n
| 163234 | \n0 | \n3327 | \n3.0 | \n1391990400 | \n2 | \n
| 60837 | \n1 | \n2322 | \n5.0 | \n1337212800 | \n0 | \n
| 233786 | \n1 | \n4123 | \n5.0 | \n1354838400 | \n0 | \n
| 163460 | \n1 | \n14212 | \n5.0 | \n1368230400 | \n0 | \n
| 206628 | \n1 | \n1542 | \n4.0 | \n1302220800 | \n0 | \n
| 261633 | \n1 | \n8802 | \n4.0 | \n1368230400 | \n0 | \n
| 99658 | \n1 | \n9198 | \n5.0 | \n1318377600 | \n0 | \n
| 268935 | \n1 | \n7215 | \n5.0 | \n1285372800 | \n0 | \n
| 77956 | \n1 | \n13468 | \n5.0 | \n1328140800 | \n0 | \n
| 105444 | \n1 | \n2374 | \n5.0 | \n1391558400 | \n0 | \n
| 237889 | \n1 | \n7169 | \n5.0 | \n1302220800 | \n0 | \n
| 173295 | \n1 | \n6677 | \n5.0 | \n1318377600 | \n0 | \n
| 50074 | \n1 | \n15278 | \n5.0 | \n1344902400 | \n0 | \n
\n\n\n \n
\n
"
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"indexed_df = pd.read_csv(new_labeled_file, sep='\\t')\n",
"print(f'shape: {indexed_df.shape}')\n",
"indexed_df[:20]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 10,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# of unique learners: 35598\n",
"# of unique courses: 18357\n",
"min/max of unique learners: 0/35597\n",
"min/max of unique courses: 0/18356\n"
]
}
],
"source": [
"u_id_str, i_id_str = 'userID', 'itemID'\n",
"u_uni = indexed_df[u_id_str].unique()\n",
"c_uni = indexed_df[i_id_str].unique()\n",
"\n",
"print(f'# of unique learners: {len(u_uni)}')\n",
"print(f'# of unique courses: {len(c_uni)}')\n",
"\n",
"print('min/max of unique learners: {0}/{1}'.format(min(u_uni), max(u_uni)))\n",
"print('min/max of unique courses: {0}/{1}'.format(min(c_uni), max(c_uni)))\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
================================================
FILE: preprocessing/2reindex-feat.ipynb
================================================
{
"cells": [
{
"cell_type": "markdown",
"source": [
"# 利用rating2inter.ipynb中U/I的index对features进行一一对应(meta-text)\n",
"- Reindex item feature ID with IDs generated in 0rating2inter.ipynb"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [
{
"data": {
"text/plain": "'/home/xin/XMMRec/Sports14'"
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"os.chdir('/home/xin/MMRec/Sports14')\n",
"os.getcwd()"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 3,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (18357, 2)\n"
]
},
{
"data": {
"text/plain": " asin itemID\n0 1881509818 0\n1 2094869245 1\n2 7245456259 2\n3 7245456313 3",
"text/html": "| \n | userID | \nitemID | \nrating | \ntimestamp | \nx_label | \n
|---|---|---|---|---|---|
| 0 | \n0 | \n11981 | \n2.0 | \n1390694400 | \n0 | \n
| 1 | \n0 | \n15852 | \n5.0 | \n1390694400 | \n0 | \n
| 2 | \n0 | \n17787 | \n3.0 | \n1391990400 | \n0 | \n
| 3 | \n0 | \n0 | \n5.0 | \n1390694400 | \n0 | \n
| 4 | \n0 | \n3369 | \n5.0 | \n1405123200 | \n0 | \n
| 5 | \n0 | \n13372 | \n5.0 | \n1391990400 | \n0 | \n
| 6 | \n0 | \n5458 | \n5.0 | \n1405123200 | \n1 | \n
| 7 | \n0 | \n3327 | \n3.0 | \n1391990400 | \n2 | \n
| 8 | \n1 | \n2322 | \n5.0 | \n1337212800 | \n0 | \n
| 9 | \n1 | \n4123 | \n5.0 | \n1354838400 | \n0 | \n
| 10 | \n1 | \n14212 | \n5.0 | \n1368230400 | \n0 | \n
| 11 | \n1 | \n1542 | \n4.0 | \n1302220800 | \n0 | \n
| 12 | \n1 | \n8802 | \n4.0 | \n1368230400 | \n0 | \n
| 13 | \n1 | \n9198 | \n5.0 | \n1318377600 | \n0 | \n
| 14 | \n1 | \n7215 | \n5.0 | \n1285372800 | \n0 | \n
| 15 | \n1 | \n13468 | \n5.0 | \n1328140800 | \n0 | \n
| 16 | \n1 | \n2374 | \n5.0 | \n1391558400 | \n0 | \n
| 17 | \n1 | \n7169 | \n5.0 | \n1302220800 | \n0 | \n
| 18 | \n1 | \n6677 | \n5.0 | \n1318377600 | \n0 | \n
| 19 | \n1 | \n15278 | \n5.0 | \n1344902400 | \n0 | \n
\n\n\n \n
\n
"
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# load item mapping\n",
"i_id_mapping = 'i_id_mapping.csv'\n",
"df = pd.read_csv(i_id_mapping, sep='\\t')\n",
"print(f'shape: {df.shape}')\n",
"df[:4]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 4,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 Extracting U-I interactions.\n",
"Total records: (532197, 9)\n"
]
},
{
"data": {
"text/plain": " asin title price \\\n0 0000032069 Adult Ballet Tutu Cheetah Pink 7.89 \n1 0000031909 Girls Ballet Tutu Neon Pink 7.00 \n2 0000032034 Adult Ballet Tutu Yellow 7.87 \n\n imUrl \\\n0 http://ecx.images-amazon.com/images/I/51EzU6qu... \n1 http://ecx.images-amazon.com/images/I/41xBoP0F... \n2 http://ecx.images-amazon.com/images/I/21GNUNIa... \n\n related brand \\\n0 {'also_bought': ['0000032050', 'B00D0DJAEG', '... BubuBibi \n1 {'also_bought': ['B002BZX8Z6', 'B00JHONN1S', '... Unknown \n2 {'also_bought': ['B00D2JSRFQ', '0000032042', '... BubuBibi \n\n categories \\\n0 [[Sports & Outdoors, Other Sports, Dance, Clot... \n1 [[Sports & Outdoors, Other Sports, Dance]] \n2 [[Sports & Outdoors, Other Sports, Dance, Clot... \n\n salesRank description \n0 NaN NaN \n1 {'Toys & Games': 201847} High quality 3 layer ballet tutu. 12 inches in... \n2 NaN NaN ",
"text/html": "| \n | asin | \nitemID | \n
|---|---|---|
| 0 | \n1881509818 | \n0 | \n
| 1 | \n2094869245 | \n1 | \n
| 2 | \n7245456259 | \n2 | \n
| 3 | \n7245456313 | \n3 | \n
\n\n\n \n
\n
"
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"import gzip, json\n",
"meta_file = 'meta_Sports_and_Outdoors.json.gz'\n",
"\n",
"print('0 Extracting U-I interactions.')\n",
"\n",
"def parse(path):\n",
" g = gzip.open(path, 'rb')\n",
" for l in g:\n",
" yield eval(l)\n",
"\n",
"def getDF(path):\n",
" i = 0\n",
" df = {}\n",
" for d in parse(path):\n",
" df[i] = d\n",
" i += 1\n",
" return pd.DataFrame.from_dict(df, orient='index')\n",
"\n",
"meta_df = getDF(meta_file)\n",
"\n",
"print(f'Total records: {meta_df.shape}')\n",
"meta_df[:3]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 5,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (18357, 10)\n"
]
},
{
"data": {
"text/plain": " asin title price \\\n132 1881509818 Ghost Inc Glock Armorers Tool 3/32 Punch 9.99 \n155 2094869245 5 LED Bicycle Rear Tail Red Bike Torch Laser B... 8.26 \n\n imUrl \\\n132 http://ecx.images-amazon.com/images/I/21iMxsyD... \n155 http://ecx.images-amazon.com/images/I/51RtwnJw... \n\n related brand \\\n132 {'also_bought': ['B000U3YWEM', 'B000U401J6', '... Ghost \n155 {'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... \n\n categories \\\n132 [[Sports & Outdoors, Hunting & Fishing, Huntin... \n155 [[Sports & Outdoors, Cycling, Lights & Reflect... \n\n salesRank \\\n132 {'Sports & Outdoors': 172909} \n155 {'Sports & Outdoors': 14293} \n\n description itemID \n132 Ghost Armorer Tool (1). The GAT is made with a... 0 \n155 This newly-designed Laser tail light can emit ... 1 ",
"text/html": "| \n | asin | \ntitle | \nprice | \nimUrl | \nrelated | \nbrand | \ncategories | \nsalesRank | \ndescription | \n
|---|---|---|---|---|---|---|---|---|---|
| 0 | \n0000032069 | \nAdult Ballet Tutu Cheetah Pink | \n7.89 | \nhttp://ecx.images-amazon.com/images/I/51EzU6qu... | \n{'also_bought': ['0000032050', 'B00D0DJAEG', '... | \nBubuBibi | \n[[Sports & Outdoors, Other Sports, Dance, Clot... | \nNaN | \nNaN | \n
| 1 | \n0000031909 | \nGirls Ballet Tutu Neon Pink | \n7.00 | \nhttp://ecx.images-amazon.com/images/I/41xBoP0F... | \n{'also_bought': ['B002BZX8Z6', 'B00JHONN1S', '... | \nUnknown | \n[[Sports & Outdoors, Other Sports, Dance]] | \n{'Toys & Games': 201847} | \nHigh quality 3 layer ballet tutu. 12 inches in... | \n
| 2 | \n0000032034 | \nAdult Ballet Tutu Yellow | \n7.87 | \nhttp://ecx.images-amazon.com/images/I/21GNUNIa... | \n{'also_bought': ['B00D2JSRFQ', '0000032042', '... | \nBubuBibi | \n[[Sports & Outdoors, Other Sports, Dance, Clot... | \nNaN | \nNaN | \n
\n\n\n \n
\n
"
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# remapping\n",
"map_dict = dict(zip(df['asin'], df['itemID']))\n",
"\n",
"meta_df['itemID'] = meta_df['asin'].map(map_dict)\n",
"meta_df.dropna(subset=['itemID'], inplace=True)\n",
"meta_df['itemID'] = meta_df['itemID'].astype('int64')\n",
"#meta_df['description'] = meta_df['description'].fillna(\" \")\n",
"meta_df.sort_values(by=['itemID'], inplace=True)\n",
"\n",
"print(f'shape: {meta_df.shape}')\n",
"meta_df[:2]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 6,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"new column names: ['itemID', 'asin', 'title', 'price', 'imUrl', 'related', 'brand', 'categories', 'salesRank', 'description']\n"
]
}
],
"source": [
"ori_cols = meta_df.columns.tolist()\n",
"\n",
"ret_cols = [ori_cols[-1]] + ori_cols[:-1]\n",
"print(f'new column names: {ret_cols}')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 7,
"outputs": [
{
"data": {
"text/plain": " asin title price \\\n132 1881509818 Ghost Inc Glock Armorers Tool 3/32 Punch 9.99 \n155 2094869245 5 LED Bicycle Rear Tail Red Bike Torch Laser B... 8.26 \n201 7245456259 Black Mountain Products Single Resistance Band... 10.49 \n\n imUrl \\\n132 http://ecx.images-amazon.com/images/I/21iMxsyD... \n155 http://ecx.images-amazon.com/images/I/51RtwnJw... \n201 http://ecx.images-amazon.com/images/I/411Ikpf1... \n\n related brand \\\n132 {'also_bought': ['B000U3YWEM', 'B000U401J6', '... Ghost \n155 {'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... \n201 {'also_bought': ['B00DDBS2JE', 'B00H1KNHE8', '... Black Mountain \n\n categories \\\n132 [[Sports & Outdoors, Hunting & Fishing, Huntin... \n155 [[Sports & Outdoors, Cycling, Lights & Reflect... \n201 [[Sports & Outdoors, Exercise & Fitness, Acces... \n\n salesRank \\\n132 {'Sports & Outdoors': 172909} \n155 {'Sports & Outdoors': 14293} \n201 {'Sports & Outdoors': 1010} \n\n description itemID \n132 Ghost Armorer Tool (1). The GAT is made with a... 0 \n155 This newly-designed Laser tail light can emit ... 1 \n201 Black Mountain Products single resistance band... 2 ",
"text/html": "| \n | asin | \ntitle | \nprice | \nimUrl | \nrelated | \nbrand | \ncategories | \nsalesRank | \ndescription | \nitemID | \n
|---|---|---|---|---|---|---|---|---|---|---|
| 132 | \n1881509818 | \nGhost Inc Glock Armorers Tool 3/32 Punch | \n9.99 | \nhttp://ecx.images-amazon.com/images/I/21iMxsyD... | \n{'also_bought': ['B000U3YWEM', 'B000U401J6', '... | \nGhost | \n[[Sports & Outdoors, Hunting & Fishing, Huntin... | \n{'Sports & Outdoors': 172909} | \nGhost Armorer Tool (1). The GAT is made with a... | \n0 | \n
| 155 | \n2094869245 | \n5 LED Bicycle Rear Tail Red Bike Torch Laser B... | \n8.26 | \nhttp://ecx.images-amazon.com/images/I/51RtwnJw... | \n{'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... | \n\n | [[Sports & Outdoors, Cycling, Lights & Reflect... | \n{'Sports & Outdoors': 14293} | \nThis newly-designed Laser tail light can emit ... | \n1 | \n
\n\n\n \n
\n
"
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"meta_df[:3]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 8,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"done!\n"
]
}
],
"source": [
"ret_df = meta_df[ret_cols]\n",
"# dump\n",
"ret_df.to_csv(os.path.join('./', 'meta-sports14.csv'), index=False)\n",
"print('done!')"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "markdown",
"source": [
"## Reload"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": 9,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (18357, 10)\n"
]
},
{
"data": {
"text/plain": " itemID asin title \\\n0 0 1881509818 Ghost Inc Glock Armorers Tool 3/32 Punch \n1 1 2094869245 5 LED Bicycle Rear Tail Red Bike Torch Laser B... \n2 2 7245456259 Black Mountain Products Single Resistance Band... \n3 3 7245456313 Black Mountain Products Resistance Band Set wi... \n\n price imUrl \\\n0 9.99 http://ecx.images-amazon.com/images/I/21iMxsyD... \n1 8.26 http://ecx.images-amazon.com/images/I/51RtwnJw... \n2 10.49 http://ecx.images-amazon.com/images/I/411Ikpf1... \n3 32.99 http://ecx.images-amazon.com/images/I/51FdHlZS... \n\n related brand \\\n0 {'also_bought': ['B000U3YWEM', 'B000U401J6', '... Ghost \n1 {'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... NaN \n2 {'also_bought': ['B00DDBS2JE', 'B00H1KNHE8', '... Black Mountain \n3 {'also_bought': ['1612431712', 'B00GSBMW2Y', '... Black Mountain \n\n categories \\\n0 [['Sports & Outdoors', 'Hunting & Fishing', 'H... \n1 [['Sports & Outdoors', 'Cycling', 'Lights & Re... \n2 [['Sports & Outdoors', 'Exercise & Fitness', '... \n3 [['Sports & Outdoors', 'Exercise & Fitness', '... \n\n salesRank \\\n0 {'Sports & Outdoors': 172909} \n1 {'Sports & Outdoors': 14293} \n2 {'Sports & Outdoors': 1010} \n3 {'Sports & Outdoors': 15} \n\n description \n0 Ghost Armorer Tool (1). The GAT is made with a... \n1 This newly-designed Laser tail light can emit ... \n2 Black Mountain Products single resistance band... \n3 [if gte mso 9]>| \n | asin | \ntitle | \nprice | \nimUrl | \nrelated | \nbrand | \ncategories | \nsalesRank | \ndescription | \nitemID | \n
|---|---|---|---|---|---|---|---|---|---|---|
| 132 | \n1881509818 | \nGhost Inc Glock Armorers Tool 3/32 Punch | \n9.99 | \nhttp://ecx.images-amazon.com/images/I/21iMxsyD... | \n{'also_bought': ['B000U3YWEM', 'B000U401J6', '... | \nGhost | \n[[Sports & Outdoors, Hunting & Fishing, Huntin... | \n{'Sports & Outdoors': 172909} | \nGhost Armorer Tool (1). The GAT is made with a... | \n0 | \n
| 155 | \n2094869245 | \n5 LED Bicycle Rear Tail Red Bike Torch Laser B... | \n8.26 | \nhttp://ecx.images-amazon.com/images/I/51RtwnJw... | \n{'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... | \n\n | [[Sports & Outdoors, Cycling, Lights & Reflect... | \n{'Sports & Outdoors': 14293} | \nThis newly-designed Laser tail light can emit ... | \n1 | \n
| 201 | \n7245456259 | \nBlack Mountain Products Single Resistance Band... | \n10.49 | \nhttp://ecx.images-amazon.com/images/I/411Ikpf1... | \n{'also_bought': ['B00DDBS2JE', 'B00H1KNHE8', '... | \nBlack Mountain | \n[[Sports & Outdoors, Exercise & Fitness, Acces... | \n{'Sports & Outdoors': 1010} | \nBlack Mountain Products single resistance band... | \n2 | \n
| \n | itemID | \nasin | \ntitle | \nprice | \nimUrl | \nrelated | \nbrand | \ncategories | \nsalesRank | \ndescription | \n
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n0 | \n1881509818 | \nGhost Inc Glock Armorers Tool 3/32 Punch | \n9.99 | \nhttp://ecx.images-amazon.com/images/I/21iMxsyD... | \n{'also_bought': ['B000U3YWEM', 'B000U401J6', '... | \nGhost | \n[['Sports & Outdoors', 'Hunting & Fishing', 'H... | \n{'Sports & Outdoors': 172909} | \nGhost Armorer Tool (1). The GAT is made with a... | \n
| 1 | \n1 | \n2094869245 | \n5 LED Bicycle Rear Tail Red Bike Torch Laser B... | \n8.26 | \nhttp://ecx.images-amazon.com/images/I/51RtwnJw... | \n{'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... | \nNaN | \n[['Sports & Outdoors', 'Cycling', 'Lights & Re... | \n{'Sports & Outdoors': 14293} | \nThis newly-designed Laser tail light can emit ... | \n
| 2 | \n2 | \n7245456259 | \nBlack Mountain Products Single Resistance Band... | \n10.49 | \nhttp://ecx.images-amazon.com/images/I/411Ikpf1... | \n{'also_bought': ['B00DDBS2JE', 'B00H1KNHE8', '... | \nBlack Mountain | \n[['Sports & Outdoors', 'Exercise & Fitness', '... | \n{'Sports & Outdoors': 1010} | \nBlack Mountain Products single resistance band... | \n
| 3 | \n3 | \n7245456313 | \nBlack Mountain Products Resistance Band Set wi... | \n32.99 | \nhttp://ecx.images-amazon.com/images/I/51FdHlZS... | \n{'also_bought': ['1612431712', 'B00GSBMW2Y', '... | \nBlack Mountain | \n[['Sports & Outdoors', 'Exercise & Fitness', '... | \n{'Sports & Outdoors': 15} | \n[if gte mso 9]><xml> <o:OfficeDocumentSettings... | \n
\n\n\n \n
\n
"
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i_id, desc_str = 'itemID', 'description'\n",
"\n",
"file_path = './'\n",
"file_name = 'meta-sports14.csv'\n",
"\n",
"meta_file = os.path.join(file_path, file_name)\n",
"\n",
"df = pd.read_csv(meta_file)\n",
"df.sort_values(by=[i_id], inplace=True)\n",
"\n",
"print('data loaded!')\n",
"print(f'shape: {df.shape}')\n",
"\n",
"df[:3]"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 4,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(91, 10)\n",
"(2659, 10)\n",
"(40, 10)\n",
"(40, 10)\n",
"(0, 10)\n"
]
}
],
"source": [
"\n",
"# sentences: title + brand + category + description | All have title + description\n",
"\n",
"title_na_df = df[df['title'].isnull()]\n",
"print(title_na_df.shape)\n",
"\n",
"desc_na_df = df[df['description'].isnull()]\n",
"print(desc_na_df.shape)\n",
"\n",
"na_df = df[df['description'].isnull() & df['title'].isnull()]\n",
"print(na_df.shape)\n",
"\n",
"na3_df = df[df['description'].isnull() & df['title'].isnull() & df['brand'].isnull()]\n",
"print(na3_df.shape)\n",
"\n",
"na4_df = df[df['description'].isnull() & df['title'].isnull() & df['brand'].isnull() & df['categories'].isnull()]\n",
"print(na4_df.shape)"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 5,
"outputs": [],
"source": [
"\n",
"df[desc_str] = df[desc_str].fillna(\" \")\n",
"df['title'] = df['title'].fillna(\" \")\n",
"df['brand'] = df['brand'].fillna(\" \")\n",
"df['categories'] = df['categories'].fillna(\" \")\n"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 6,
"outputs": [
{
"data": {
"text/plain": "['Ghost Inc Glock Armorers Tool 3/32 Punch Ghost Sports & Outdoors Hunting & Fishing Hunting Gun Maintenance Gunsmithing Tools Ghost Armorer Tool (1). The GAT is made with a spring steel punch. The diameter is 3/32 of an inch or 2.5mm, this is the same as the OEM tool size. The difference is you will be able to press harder without bending the shaft of this punch. Just a better tool to work on your Glock with.',\n '5 LED Bicycle Rear Tail Red Bike Torch Laser Beam Lamp Light Sports & Outdoors Cycling Lights & Reflectors Taillights This newly-designed Laser tail light can emit two parallel lines, to form a virtual lane together with the moving of bicycle on the road. LED flash light and two lines not only enhance the waring effect strongly and greatly but also improve the safety of night riding.',\n 'Black Mountain Products Single Resistance Band - Door Anchor and Starter Guide Included Black Mountain Sports & Outdoors Exercise & Fitness Accessories Exercise Bands Black Mountain Products single resistance bands are made out of the highest quality rubber to ensure maximum life and are 99 percent latex free! These bands are ideal for physical therapy, exercise, weight loss, Pilates, muscle toning, muscle strengthening, stretching, rehabilitation, and general health and fitness. B.M.P. resistance bands are great for home use, gym use, offices, and are ideal for travel. B.M.P. single resistance bands are a great alternative to conventional weights and exercise equipment. All Black Mountain Products Resistance bands come with a manufactures warranty.',\n 'Black Mountain Products Resistance Band Set with Door Anchor, Ankle Strap, Exercise Chart, and Resistance Band Carrying Case Black Mountain Sports & Outdoors Exercise & Fitness Accessories Exercise Bands [if gte mso 9]>| \n | itemID | \nasin | \ntitle | \nprice | \nimUrl | \nrelated | \nbrand | \ncategories | \nsalesRank | \ndescription | \n
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n0 | \n1881509818 | \nGhost Inc Glock Armorers Tool 3/32 Punch | \n9.99 | \nhttp://ecx.images-amazon.com/images/I/21iMxsyD... | \n{'also_bought': ['B000U3YWEM', 'B000U401J6', '... | \nGhost | \n[['Sports & Outdoors', 'Hunting & Fishing', 'H... | \n{'Sports & Outdoors': 172909} | \nGhost Armorer Tool (1). The GAT is made with a... | \n
| 1 | \n1 | \n2094869245 | \n5 LED Bicycle Rear Tail Red Bike Torch Laser B... | \n8.26 | \nhttp://ecx.images-amazon.com/images/I/51RtwnJw... | \n{'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... | \nNaN | \n[['Sports & Outdoors', 'Cycling', 'Lights & Re... | \n{'Sports & Outdoors': 14293} | \nThis newly-designed Laser tail light can emit ... | \n
| 2 | \n2 | \n7245456259 | \nBlack Mountain Products Single Resistance Band... | \n10.49 | \nhttp://ecx.images-amazon.com/images/I/411Ikpf1... | \n{'also_bought': ['B00DDBS2JE', 'B00H1KNHE8', '... | \nBlack Mountain | \n[['Sports & Outdoors', 'Exercise & Fitness', '... | \n{'Sports & Outdoors': 1010} | \nBlack Mountain Products single resistance band... | \n
| \n | itemID | \nasin | \ntitle | \nprice | \nimUrl | \nrelated | \nbrand | \ncategories | \nsalesRank | \ndescription | \n
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n0 | \n1881509818 | \nGhost Inc Glock Armorers Tool 3/32 Punch | \n9.99 | \nhttp://ecx.images-amazon.com/images/I/21iMxsyD... | \n{'also_bought': ['B000U3YWEM', 'B000U401J6', '... | \nGhost | \n[['Sports & Outdoors', 'Hunting & Fishing', 'H... | \n{'Sports & Outdoors': 172909} | \nGhost Armorer Tool (1). The GAT is made with a... | \n
| 1 | \n1 | \n2094869245 | \n5 LED Bicycle Rear Tail Red Bike Torch Laser B... | \n8.26 | \nhttp://ecx.images-amazon.com/images/I/51RtwnJw... | \n{'also_bought': ['B0081O93N2', 'B00EYTCHJA', '... | \n\n | [['Sports & Outdoors', 'Cycling', 'Lights & Re... | \n{'Sports & Outdoors': 14293} | \nThis newly-designed Laser tail light can emit ... | \n
| 2 | \n2 | \n7245456259 | \nBlack Mountain Products Single Resistance Band... | \n10.49 | \nhttp://ecx.images-amazon.com/images/I/411Ikpf1... | \n{'also_bought': ['B00DDBS2JE', 'B00H1KNHE8', '... | \nBlack Mountain | \n[['Sports & Outdoors', 'Exercise & Fitness', '... | \n{'Sports & Outdoors': 1010} | \nBlack Mountain Products single resistance band... | \n
| 3 | \n3 | \n7245456313 | \nBlack Mountain Products Resistance Band Set wi... | \n32.99 | \nhttp://ecx.images-amazon.com/images/I/51FdHlZS... | \n{'also_bought': ['1612431712', 'B00GSBMW2Y', '... | \nBlack Mountain | \n[['Sports & Outdoors', 'Exercise & Fitness', '... | \n{'Sports & Outdoors': 15} | \n[if gte mso 9]><xml> <o:OfficeDocumentSettings... | \n
| 4 | \n4 | \nB000002NUS | \nOuters Universal 32-Piece Blow Molded Gun Clea... | \n21.99 | \nhttp://ecx.images-amazon.com/images/I/510GjWgd... | \n{'also_bought': ['B000PW64JY', 'B0010KHNEU', '... | \nOuters | \n[['Sports & Outdoors', 'Hunting & Fishing', 'H... | \n{'Sports & Outdoors': 26738} | \nOuters now offers this rigid and durable hard ... | \n