7.1处理缺失数据
Nan（not a number ）在pandas表示缺失值
import pandas as pd
import numpy as np
string_data=pd.Series(['aardvark','artichoke',np.nan,'avocado'])
string_data
0 aardvark
1 artichoke
2 NaN
3 avocado
dtype: object
string_data.isnull()
0 False
1 False
2 True
3 False
dtype: bool
string_data[0]=None
string_data.isnull()
0 True
1 False
2 True
3 False
dtype: bool
- dropna:删除缺失数据
- fillna:插值方法填充缺失数据
- isnull: 返回布尔值，表明哪些是缺失值
- notnull :isnull的反面
滤除缺失数据
from numpy import nan as NA
data=pd.Series([1,NA,3.5,NA,7])
data.dropna()
0 1.0
2 3.5
4 7.0
dtype: float64
data[data.notnull()]
0 1.0
2 3.5
4 7.0
dtype: float64
data = pd.DataFrame([[1., 6.5, 3.], [1., NA, NA],
[NA, NA, NA], [NA, 6.5, 3.]])
data
01201.06.53.011.0NaNNaN2NaNNaNNaN3NaN6.53.0
cleaned=data.dropna()
cleaned
01201.06.53.0
data.dropna(how='all')
01201.06.53.011.0NaNNaN3NaN6.53.0
data[4]=NA
data
012401.06.53.0NaN11.0NaNNaNNaN2NaNNaNNaNNaN3NaN6.53.0NaN
data.dropna(axis=1,how='all')
01201.06.53.011.0NaNNaN2NaNNaNNaN3NaN6.53.0
df=pd.DataFrame(np.random.randn(7,3))
df.iloc[:4,1]=NA
df.iloc[:2,2]=NA
df
01201.230124NaNNaN1-0.671868NaNNaN2-0.596658NaN0.0024183-1.061044NaN-0.2460414-0.677290-1.394329-1.8705105-0.3134590.133874-1.1722826-0.495465-0.9541270.150156
df.dropna()
0124-0.677290-1.394329-1.8705105-0.3134590.133874-1.1722826-0.495465-0.9541270.150156
df.dropna(thresh=2)
0122-0.596658NaN0.0024183-1.061044NaN-0.2460414-0.677290-1.394329-1.8705105-0.3134590.133874-1.1722826-0.495465-0.9541270.150156
填充缺失数据
df.fillna(0)
01201.2301240.0000000.0000001-0.6718680.0000000.0000002-0.5966580.0000000.0024183-1.0610440.000000-0.2460414-0.677290-1.394329-1.8705105-0.3134590.133874-1.1722826-0.495465-0.9541270.150156
df.fillna({1:0.5,2:0})
01201.2301240.5000000.0000001-0.6718680.5000000.0000002-0.5966580.5000000.0024183-1.0610440.500000-0.2460414-0.677290-1.394329-1.8705105-0.3134590.133874-1.1722826-0.495465-0.9541270.150156
_=df.fillna(0,inplace=True)
df
01201.2301240.0000000.0000001-0.6718680.0000000.0000002-0.5966580.0000000.0024183-1.0610440.000000-0.2460414-0.677290-1.394329-1.8705105-0.3134590.133874-1.1722826-0.495465-0.9541270.150156
df=pd.DataFrame(np.random.randn(6,3))
df.iloc[2:,1]=NA
df.iloc[4:,2]=NA
df
01200.536292-0.231305-0.9441161-0.2165951.8084021.0860822-0.457510NaN-0.6170133-0.163709NaN0.45009940.969959NaNNaN51.136978NaNNaN
df.fillna(method='ffill')
01200.536292-0.231305-0.9441161-0.2165951.8084021.0860822-0.4575101.808402-0.6170133-0.1637091.8084020.45009940.9699591.8084020.45009951.1369781.8084020.450099
df.fillna(method='ffill',limit=2)
01200.536292-0.231305-0.9441161-0.2165951.8084021.0860822-0.4575101.808402-0.6170133-0.1637091.8084020.45009940.969959NaN0.45009951.136978NaN0.450099
data=pd.Series([1., NA, 3.5, NA, 7])
data
0 1.0
1 NaN
2 3.5
3 NaN
4 7.0
dtype: float64
data.fillna(data.mean())
0 1.000000
1 3.833333
2 3.500000
3 3.833333
4 7.000000
dtype: float64
fillna参数：
- value:用于填充缺失值的标量值或字典对象
- method：插值方式，未指定方式为ffill
- axis：默认为axis=0
- inplace：如果为true，对原件更改
- limit：向前/后可以连续填充最大数目
7.2数据转换
移除重复数据
data = pd.DataFrame({'k1': ['one', 'two'] * 3 + ['two'],
'k2': [1, 1, 2, 3, 3, 4, 4]})
data
k1k20one11two12one23two34one35two46two4
data.duplicated()
0 False
1 False
2 False
3 False
4 False
5 False
6 True
dtype: bool
data.drop_duplicates()
k1k20one11two12one23two34one35two4
data['v1']=range(7)
data
k1k2v10one101two112one223two334one345two456two46
data.drop_duplicates(['k1'])
k1k2v10one101two11
data.drop_duplicates(['k1','k2'],keep='last')
k1k2v10one101two112one223two334one346two46
利用函数或映射进行数据转换
data = pd.DataFrame({'food': ['bacon', 'pulled pork', 'bacon',
'Pastrami', 'corned beef', 'Bacon',
'pastrami', 'honey ham', 'nova lox'],
'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})
data
foodounces0bacon4.01pulled pork3.02bacon12.03Pastrami6.04corned beef7.55Bacon8.06pastrami3.07honey ham5.08nova lox6.0
meat_to_animal = {
'bacon': 'pig',
'pulled pork': 'pig',
'pastrami': 'cow',
'corned beef': 'cow',
'honey ham': 'pig',
'nova lox': 'salmon'
}
meat_to_animal
{'bacon': 'pig',
'pulled pork': 'pig',
'pastrami': 'cow',
'corned beef': 'cow',
'honey ham': 'pig',
'nova lox': 'salmon'}
lowercased=data['food'].str.lower()
lowercased
0 bacon
1 pulled pork
2 bacon
3 pastrami
4 corned beef
5 bacon
6 pastrami
7 honey ham
8 nova lox
Name: food, dtype: object
data['animal']=lowercased.map(meat_to_animal)
data
foodouncesanimal0bacon4.0pig1pulled pork3.0pig2bacon12.0pig3Pastrami6.0cow4corned beef7.5cow5Bacon8.0pig6pastrami3.0cow7honey ham5.0pig8nova lox6.0salmon
data['food']
0 bacon
1 pulled pork
2 bacon
3 Pastrami
4 corned beef
5 Bacon
6 pastrami
7 honey ham
8 nova lox
Name: food, dtype: object
data['food'].str.lower().map(meat_to_animal)
0 pig
1 pig
2 pig
3 cow
4 cow
5 pig
6 cow
7 pig
8 salmon
Name: food, dtype: object
data['food'].map(lambda x:meat_to_animal[x.lower()])
0 pig
1 pig
2 pig
3 cow
4 cow
5 pig
6 cow
7 pig
8 salmon
Name: food, dtype: object
替换值
data=pd.Series([1,-999,2,-999,-1000,3])
data
0 1
1 -999
2 2
3 -999
4 -1000
5 3
dtype: int64
data.replace(-999,np.nan)
0 1.0
1 NaN
2 2.0
3 NaN
4 -1000.0
5 3.0
dtype: float64
data.replace([-999,-1000],np.nan)
0 1.0
1 NaN
2 2.0
3 NaN
4 NaN
5 3.0
dtype: float64
data.replace([-999,-1000],[np.nan,0])
0 1.0
1 NaN
2 2.0
3 NaN
4 0.0
5 3.0
dtype: float64
data.replace({-999:np.nan,-1000:0})
0 1.0
1 NaN
2 2.0
3 NaN
4 0.0
5 3.0
dtype: float64
重新命名轴索引
data = pd.DataFrame(np.arange(12).reshape((3, 4)),
index=['Ohio', 'Colorado', 'New York'],
columns=['one', 'two', 'three', 'four'])
data
onetwothreefourOhio0123Colorado4567New York891011
data.index
Index(['Ohio', 'Colorado', 'New York'], dtype='object')
transform=lambda x: x[:4].upper()
data.index.map(transform)
Index(['OHIO', 'COLO', 'NEW '], dtype='object')
data.rename(index=str.title,columns=str.upper)
ONETWOTHREEFOUROhio0123Colorado4567New York891011
data.rename(index={'Ohio':'INDIANA'},
columns={'three':'peekaboo'})
onetwopeekaboofourINDIANA0123Colorado4567New York891011
data.rename(index={'Ohio':'INDIANA'},inplace=True)
data
onetwothreefourINDIANA0123Colorado4567New York891011
离散化和面元划分
假设有一组数据被分成不同的年龄段，怎么做？
[En]
Suppose there is a set of data divided into different age groups, how to do it?
ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]
bins=[18,25,35,60,100]
cats=pd.cut(ages,bins)
cats
[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]
Length: 12
Categories (4, interval[int64]): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]
cats.codes
array([0, 0, 0, 1, 0, 0, 2, 1, 3, 2, 2, 1], dtype=int8)
cats.categories
IntervalIndex([(18, 25], (25, 35], (35, 60], (60, 100]],
closed='right',
dtype='interval[int64]')
pd.value_counts(cats)
(18, 25] 5
(35, 60] 3
(25, 35] 3
(60, 100] 1
dtype: int64
pd.cut(ages,[18,26,36,61,100],right=False)
[[18, 26), [18, 26), [18, 26), [26, 36), [18, 26), ..., [26, 36), [61, 100), [36, 61), [36, 61), [26, 36)]
Length: 12
Categories (4, interval[int64]): [[18, 26) < [26, 36) < [36, 61) < [61, 100)]
group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']
pd.cut(ages,bins,labels=group_names)
['Youth', 'Youth', 'Youth', 'YoungAdult', 'Youth', ..., 'YoungAdult', 'Senior', 'MiddleAged', 'MiddleAged', 'YoungAdult']
Length: 12
Categories (4, object): ['Youth' < 'YoungAdult' < 'MiddleAged' < 'Senior']
data=np.random.rand(20)
pd.cut(data,4,precision=2)
[(0.7, 0.92], (0.7, 0.92], (0.065, 0.28], (0.7, 0.92], (0.065, 0.28], ..., (0.7, 0.92], (0.49, 0.7], (0.28, 0.49], (0.7, 0.92], (0.065, 0.28]]
Length: 20
Categories (4, interval[float64]): [(0.065, 0.28] < (0.28, 0.49] < (0.49, 0.7] < (0.7, 0.92]]
data=np.random.randn(1000)
cats=pd.qcut(data,4)
cats
[(-3.5669999999999997, -0.673], (-3.5669999999999997, -0.673], (-0.673, -0.039], (-3.5669999999999997, -0.673], (-0.039, 0.631], ..., (-3.5669999999999997, -0.673], (-3.5669999999999997, -0.673], (-0.039, 0.631], (-3.5669999999999997, -0.673], (-3.5669999999999997, -0.673]]
Length: 1000
Categories (4, interval[float64]): [(-3.5669999999999997, -0.673] < (-0.673, -0.039] < (-0.039, 0.631] < (0.631, 3.121]]
pd.value_counts(cats)
(0.631, 3.121] 250
(-0.039, 0.631] 250
(-0.673, -0.039] 250
(-3.5669999999999997, -0.673] 250
dtype: int64
pd.qcut(data,[0,0.1,0.5,0.9,1])
[(-1.294, -0.039], (-1.294, -0.039], (-1.294, -0.039], (-3.5669999999999997, -1.294], (-0.039, 1.256], ..., (-1.294, -0.039], (-1.294, -0.039], (-0.039, 1.256], (-1.294, -0.039], (-1.294, -0.039]]
Length: 1000
Categories (4, interval[float64]): [(-3.5669999999999997, -1.294] < (-1.294, -0.039] < (-0.039, 1.256] < (1.256, 3.121]]
检测和过滤异常值
import pandas as pd
import numpy as np
data=pd.DataFrame(np.random.randn(1000,4))
data.describe()
0123count1000.0000001000.0000001000.0000001000.000000mean0.007752-0.032028-0.037349-0.036083std0.9767700.9744560.9834291.015825min-3.357288-3.298192-2.813273-3.23562925%-0.663744-0.662803-0.703169-0.75849650%0.011468-0.026073-0.087028-0.06326275%0.6622350.5812480.6392370.642911max3.3202222.8337083.5361392.816898
col=data[2]
col[np.abs(col)>3]
992 3.536139
Name: 2, dtype: float64
data[(np.abs(data)>3).any(1)]
0123610.502113-3.298192-1.4454270.7287766430.430629-3.0607440.731826-1.0391446780.1154040.0179180.058429-3.2356297183.3202220.4862550.6868230.966785750-3.206397-1.8368571.102002-0.180903813-3.357288-0.662363-1.293561-1.962479824-0.9503422.208761-0.203996-3.0597869920.920673-0.6881963.5361390.528149
data[np.abs(data)>3]=np.sign(data)*3
data.describe()
0123count1000.0000001000.0000001000.0000001000.000000mean0.007996-0.031669-0.037885-0.035788std0.9739070.9733130.9816231.014933min-3.000000-3.000000-2.813273-3.00000025%-0.663744-0.662803-0.703169-0.75849650%0.011468-0.026073-0.087028-0.06326275%0.6622350.5812480.6392370.642911max3.0000002.8337083.0000002.816898
np.sign(data).head()
012301.01.01.0-1.01-1.0-1.0-1.0-1.021.0-1.0-1.0-1.031.01.0-1.01.041.01.0-1.01.0
排列和随机采样
df=pd.DataFrame(np.arange(5*4).reshape(5,4))
df
012300123145672891011312131415416171819
sampler=np.random.permutation(5)
sampler
array([2, 1, 3, 0, 4])
df.take(sampler)
012328910111456731213141500123416171819
df.sample(n=3)
012328910110012314567
choices = pd.Series([5, 7, -1, 6, 4])
draws=choices.sample(n=10,replace=True)
draws
0 5
1 7
4 4
1 7
0 5
2 -1
3 6
0 5
2 -1
4 4
dtype: int64
计算指标/哑变量
将分类变量转化为”哑变量”/指标矩阵
df = pd.DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'],
'data1': range(6)})
df
keydata10b01b12a23c34a45b5
pd.get_dummies(df['key'])
abc001010102100300141005010
pd.get_dummies:pandas 实现one hot encode的方式
one-hot的基本思想：将离散型特征的每一种取值都看成一种状态，若你的这一特征中有N个不相同的取值，那么我们就可以将该特征抽象成N种不同的状态，one-hot编码保证了每一个取值只会使得一种状态处于”激活态”，也就是说这N种状态中只有一个状态位值为1，其他状态位都是0。
引用：https://zhuanlan.zhihu.com/p/139144355
dummies = pd.get_dummies(df['key'], prefix='key')
dummies
key_akey_bkey_c001010102100300141005010
如果输入字符串’data1′,得到结果位series
df['data1']
0 0
1 1
2 2
3 3
4 4
5 5
Name: data1, dtype: int64
如果输入列表[‘data1’],则返回DataFrame
df[['data1']]
data1001122334455
df_with_dummy=df[['data1']].join(dummies)
df_with_dummy
data1key_akey_bkey_c000101101022100330014410055010
mnames = ['movie_id', 'title', 'genres']
movies = pd.read_table('pydata-book/datasets/movielens/movies.dat', sep='::',
header=None, names=mnames)
movies
E:\Anaconda\lib\site-packages\pandas\io\parsers.py:765: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.
return read_csv(**locals())
movie_idtitlegenres01Toy Story (1995)Animation|Children’s|Comedy12Jumanji (1995)Adventure|Children’s|Fantasy23Grumpier Old Men (1995)Comedy|Romance34Waiting to Exhale (1995)Comedy|Drama45Father of the Bride Part II (1995)Comedy…………38783948Meet the Parents (2000)Comedy38793949Requiem for a Dream (2000)Drama38803950Tigerland (2000)Drama38813951Two Family House (2000)Drama38823952Contender, The (2000)Drama|Thriller
3883 rows × 3 columns
movies.shape
(3883, 3)
all_genres=[]
for x in movies.genres:
all_genres.extend(x.split('|'))
all_genres
['Animation',
"Children's",
'Comedy',
'Adventure',
"Children's",
'Fantasy',
'Comedy',
'Romance',
'Comedy',
'Drama',
'Comedy',
'Action',
'Crime',
'Thriller',
'Comedy',
'Romance',
'Adventure',
"Children's",
'Action',
'Action',
'Adventure',
'Thriller',
'Comedy',
'Drama',
'Romance',
'Comedy',
'Horror',
'Animation',
"Children's",
'Drama',
'Action',
'Adventure',
'Romance',
'Drama',
'Thriller',
'Drama',
'Romance',
'Thriller',
'Comedy',
'Action',
'Action',
'Comedy',
'Drama',
'Crime',
'Drama',
'Thriller',
'Thriller',
'Drama',
'Sci-Fi',
'Drama',
'Romance',
'Drama',
'Drama',
'Romance',
'Adventure',
'Sci-Fi',
'Drama',
'Drama',
'Drama',
'Sci-Fi',
'Adventure',
'Romance',
"Children's",
'Comedy',
'Drama',
'Drama',
'Romance',
'Drama',
'Documentary',
'Comedy',
'Comedy',
'Romance',
'Drama',
'Drama',
'War',
'Action',
'Crime',
'Drama',
'Drama',
'Action',
'Adventure',
'Comedy',
'Drama',
'Drama',
'Romance',
'Crime',
'Thriller',
'Animation',
"Children's",
'Musical',
'Romance',
'Drama',
'Romance',
'Crime',
'Thriller',
'Action',
'Drama',
'Thriller',
'Comedy',
'Drama',
"Children's",
'Comedy',
'Drama',
'Adventure',
"Children's",
'Fantasy',
'Drama',
'Drama',
'Romance',
'Drama',
'Mystery',
'Adventure',
"Children's",
'Fantasy',
'Drama',
'Thriller',
'Drama',
'Comedy',
'Comedy',
'Romance',
'Comedy',
'Sci-Fi',
'Thriller',
'Drama',
'Comedy',
'Romance',
'Comedy',
'Action',
'Comedy',
'Crime',
'Horror',
'Thriller',
'Action',
'Comedy',
'Drama',
'Drama',
'Musical',
'Drama',
'Romance',
'Comedy',
'Drama',
'Sci-Fi',
'Thriller',
'Documentary',
'Drama',
'Drama',
'Thriller',
'Drama',
'Crime',
'Drama',
'Romance',
'Drama',
'Drama',
'Comedy',
'Drama',
'Drama',
'Romance',
'Adventure',
'Drama',
"Children's",
'Comedy',
'Comedy',
'Action',
'Thriller',
'Drama',
'Drama',
'Thriller',
'Comedy',
'Romance',
'Drama',
'Action',
'Thriller',
'Comedy',
'Drama',
'Action',
'Thriller',
'Documentary',
'Drama',
'Thriller',
'Comedy',
'Comedy',
'Thriller',
'Comedy',
'Drama',
'Romance',
'Comedy',
'Drama',
'Adventure',
"Children's",
'Comedy',
'Musical',
'Documentary',
'Comedy',
'Action',
'Drama',
'War',
'Drama',
'Thriller',
'Action',
'Adventure',
'Crime',
'Drama',
'Mystery',
'Drama',
'Comedy',
'Documentary',
'Crime',
'Comedy',
'Romance',
'Comedy',
'Drama',
'Drama',
'Comedy',
'Romance',
'Drama',
'Mystery',
'Romance',
'Drama',
'Comedy',
'Adventure',
"Children's",
'Fantasy',
'Drama',
'Documentary',
'Comedy',
'Romance',
'Drama',
'Drama',
'Romance',
'Thriller',
'Comedy',
'Drama',
'Documentary',
'Comedy',
'Documentary',
'Documentary',
'Drama',
'Action',
'Drama',
'Drama',
'Romance',
'Comedy',
'Drama',
'Drama',
'Comedy',
'Action',
'Adventure',
"Children's",
'Drama',
'Drama',
'Crime',
'Drama',
'Thriller',
'Drama',
'Drama',
'Romance',
'War',
'Horror',
'Action',
'Adventure',
'Comedy',
'Crime',
'Drama',
'Drama',
'War',
'Comedy',
'Comedy',
'War',
'Adventure',
"Children's",
'Drama',
'Action',
'Adventure',
'Mystery',
'Sci-Fi',
'Drama',
'Thriller',
'War',
'Documentary',
'Action',
'Romance',
'Thriller',
'Crime',
'Film-Noir',
'Mystery',
'Thriller',
'Action',
'Thriller',
'Comedy',
'Drama',
'Drama',
'Action',
'Adventure',
'Drama',
'Romance',
'Adventure',
"Children's",
'Drama',
'Action',
'Crime',
'Thriller',
'Comedy',
'Action',
'Sci-Fi',
'Thriller',
'Action',
'Adventure',
'Sci-Fi',
'Comedy',
'Drama',
'Comedy',
'Horror',
'Comedy',
'Drama',
'Romance',
'Comedy',
'Action',
"Children's",
'Drama',
'Romance',
'Thriller',
'Drama',
'Sci-Fi',
'Thriller',
'Comedy',
'Comedy',
'Horror',
'Comedy',
'Thriller',
'Drama',
'Documentary',
'Drama',
'Drama',
'Comedy',
'Drama',
'Romance',
'Horror',
'Sci-Fi',
'Drama',
'Action',
'Crime',
'Sci-Fi',
'Drama',
'Musical',
'Thriller',
'Drama',
'Drama',
'Romance',
'Comedy',
'Action',
'Comedy',
'Drama',
'Documentary',
'Drama',
'Romance',
'Action',
'Adventure',
'Drama',
'Western',
'Drama',
'Comedy',
'Drama',
'Drama',
'Drama',
'Romance',
'Comedy',
'Drama',
'Thriller',
'Comedy',
'Drama',
'Drama',
'Horror',
'Drama',
'Romance',
'Comedy',
'Comedy',
'Drama',
'Romance',
'Drama',
'Thriller',
'Thriller',
'Action',
'Comedy',
'Drama',
'Thriller',
'Drama',
'Thriller',
'Comedy',
'Comedy',
'Drama',
'Drama',
'Comedy',
'Comedy',
'Drama',
'Comedy',
'Romance',
'Comedy',
'Romance',
'Adventure',
"Children's",
'Animation',
"Children's",
'Comedy',
'Romance',
'Thriller',
"Children's",
'Drama',
'Drama',
'Musical',
'Comedy',
'Animation',
"Children's",
'Crime',
'Drama',
'Documentary',
'Drama',
'Fantasy',
'Romance',
'Thriller',
'Comedy',
'Drama',
'Romance',
"Children's",
'Comedy',
'Action',
'Comedy',
'Romance',
'Drama',
'Horror',
'Drama',
'Comedy',
'Comedy',
'Sci-Fi',
'Mystery',
'Thriller',
'Adventure',
"Children's",
'Comedy',
'Fantasy',
'Romance',
'Crime',
'Drama',
'Thriller',
'Action',
'Adventure',
'Fantasy',
'Sci-Fi',
'Drama',
"Children's",
'Drama',
'Drama',
'Drama',
'Drama',
'Romance',
'Drama',
'Romance',
'War',
'Western',
'Comedy',
'Drama',
'Drama',
'Drama',
'Romance',
'Drama',
'Drama',
'Drama',
'Horror',
'Comedy',
'Comedy',
'Comedy',
'Romance',
'Drama',
'Comedy',
'Drama',
'Drama',
'Thriller',
'Drama',
'Drama',
'Crime',
'Drama',
'Action',
'Crime',
'Drama',
'Horror',
'Action',
'Sci-Fi',
'Thriller',
'Comedy',
'Romance',
'Action',
'Thriller',
'Comedy',
'Romance',
'Crime',
'Drama',
'Thriller',
'Action',
'Drama',
'Thriller',
'Crime',
'Drama',
'Romance',
'Thriller',
'Comedy',
'Romance',
'Comedy',
'Romance',
'Crime',
'Drama',
'Drama',
'Comedy',
'Drama',
'Drama',
'Drama',
'Romance',
'Drama',
'Romance',
'Action',
'Adventure',
'Western',
'Comedy',
'Drama',
'Comedy',
'Drama',
'Drama',
'Drama',
'Drama',
'Comedy',
'Horror',
'Thriller',
'Comedy',
'Animation',
"Children's",
'Drama',
'Action',
'Action',
'Adventure',
'Sci-Fi',
"Children's",
'Comedy',
'Fantasy',
'Drama',
'Thriller',
'Film-Noir',
'Thriller',
'Drama',
'Comedy',
'Drama',
'Comedy',
'Comedy',
'Drama',
'Action',
'Comedy',
'Musical',
'Sci-Fi',
'Horror',
'Action',
'Adventure',
'Sci-Fi',
'Comedy',
'Horror',
'Drama',
'Horror',
'Sci-Fi',
'Comedy',
'Drama',
'Mystery',
'Thriller',
'Drama',
'War',
'Drama',
'Sci-Fi',
'Thriller',
'Comedy',
'Romance',
'Adventure',
'Drama',
'Drama',
'Comedy',
'Romance',
"Children's",
'Comedy',
'Comedy',
'Drama',
'Drama',
'Musical',
'Drama',
'Comedy',
'Action',
'Adventure',
'Thriller',
'Drama',
'Mystery',
'Thriller',
'Comedy',
'Drama',
'Romance',
'Comedy',
'Action',
'Romance',
'Thriller',
'Drama',
"Children's",
'Comedy',
'Comedy',
'Romance',
'War',
'Comedy',
'Romance',
'Drama',
'Comedy',
'Drama',
'Romance',
'Action',
'Comedy',
'Drama',
'Romance',
'Adventure',
"Children's",
'Romance',
'Documentary',
'Animation',
"Children's",
'Musical',
'Drama',
'Horror',
'Comedy',
'Crime',
'Fantasy',
'Action',
'Comedy',
'Western',
'Drama',
'Comedy',
'Comedy',
'Drama',
'Comedy',
'Drama',
'Thriller',
"Children's",
'Comedy',
'Drama',
'Action',
'Thriller',
'Action',
'Romance',
'Thriller',
'Comedy',
'Romance',
'Action',
'Sci-Fi',
'Action',
'Adventure',
'Comedy',
'Romance',
'Drama',
'Drama',
'Horror',
'Western',
'Action',
'Drama',
'Drama',
'Action',
'Comedy',
'Drama',
'Drama',
'Romance',
'War',
'Action',
'Comedy',
'Drama',
'Crime',
'Drama',
'Adventure',
"Children's",
'Action',
'Action',
'Drama',
'Drama',
'Horror',
'Documentary',
'Drama',
'Drama',
'Action',
'Thriller',
'Comedy',
'Comedy',
'Crime',
'Drama',
'Documentary',
'Action',
'Sci-Fi',
'Drama',
'Horror',
'Thriller',
'Drama',
'Drama',
'Comedy',
'Comedy',
'Drama',
'Comedy',
'Comedy',
'Comedy',
'Thriller',
'Western',
'Comedy',
'Romance',
'Drama',
'Comedy',
'Action',
'Comedy',
'Adventure',
"Children's",
'Thriller',
'Action',
'Thriller',
'Drama',
'Drama',
'Romance',
'Horror',
'Sci-Fi',
'Thriller',
'Mystery',
'Romance',
'Thriller',
'Drama',
'Comedy',
'Drama',
'Crime',
'Drama',
'Comedy',
'Western',
'Comedy',
'Action',
'Adventure',
'Crime',
'Comedy',
'Sci-Fi',
'Drama',
'Thriller',
'Comedy',
'Action',
'Comedy',
'Drama',
'Comedy',
'Romance',
'Comedy',
'Action',
'Sci-Fi',
'Documentary',
'Comedy',
'Romance',
'Comedy',
'Drama',
'Romance',
'Comedy',
'Romance',
'Drama',
'Comedy',
'Comedy',
'Drama',
'Drama',
'Mystery',
'Romance',
'Drama',
'Comedy',
'Drama',
'Thriller',
'Adventure',
"Children's",
'Drama',
'Drama',
'Action',
'Thriller',
'Drama',
'Western',
'Action',
'Comedy',
'Drama',
'Romance',
'Action',
'Adventure',
'Crime',
'Drama',
'Thriller',
'Action',
'Adventure',
'Crime',
'Thriller',
'Action',
'Drama',
'War',
'Action',
'Comedy',
'War',
'Comedy',
'Comedy',
'Romance',
'Drama',
'Romance',
'Comedy',
'Comedy',
'Romance',
'Comedy',
'Drama',
'Comedy',
'War',
'Action',
'Thriller',
'Drama',
'Comedy',
'Drama',
'Drama',
'Comedy',
'Action',
'Action',
'Adventure',
'Sci-Fi',
'Drama',
'Thriller',
'Thriller',
'Drama',
'Adventure',
"Children's",
'Action',
'Comedy',
'Comedy',
'Comedy',
'Western',
'Drama',
'Comedy',
'Thriller',
'Drama',
'Comedy',
'Mystery',
'Action',
'Crime',
'Drama',
'Action',
'Thriller',
'Drama',
'Comedy',
'Drama',
'Romance',
'Comedy',
'Romance',
'Drama',
'Romance',
'Comedy',
'Romance',
'Comedy',
'Drama',
'Action',
"Children's",
'Drama',
'Action',
'Sci-Fi',
'Comedy',
'Drama',
'Action',
'Drama',
'Drama',
'Drama',
'Romance',
'Drama',
'Action',
'Drama',
'Horror',
'Sci-Fi',
'Comedy',
'Mystery',
'Romance',
'Comedy',
'Drama',
'Comedy',
'Drama',
'War',
'Action',
'Drama',
'Mystery',
'Comedy',
'Sci-Fi',
'Thriller',
'Comedy',
'Crime',
'Thriller',
'Action',
'Drama',
'Drama',
'Drama',
'Drama',
'Drama',
'Drama',
'War',
'Drama',
'Drama',
'Drama',
"Children's",
'Drama',
'Comedy',
'Crime',
'Horror',
'Action',
'Drama',
'Romance',
'Drama',
'Drama',
'Comedy',
'Drama',
'Drama',
'Comedy',
'Romance',
'Thriller',
'Film-Noir',
'Sci-Fi',
'Comedy',
'Comedy',
'Romance',
'Thriller',
'Action',
'Drama',
'Action',
'Adventure',
"Children's",
'Sci-Fi',
'Action',
'Adventure',
'Thriller',
'Action',
'Documentary',
'Comedy',
'Romance',
"Children's",
'Comedy',
'Musical',
'Action',
'Adventure',
'Comedy',
'Western',
'Thriller',
'Action',
'Crime',
'Romance',
'Documentary',
'Drama',
'Action',
'Adventure',
'Animation',
"Children's",
'Fantasy',
'Comedy',
'Drama',
'Thriller',
'Comedy',
'Drama',
'Drama',
'Comedy',
'Horror',
'Comedy',
'Romance',
'Drama',
'Comedy',
'Drama',
"Children's",
'Comedy',
'Comedy',
'Drama',
'Drama',
'Drama',
'Drama',
'Comedy',
'Drama',
"Children's",
'Comedy',
'Comedy',
'Adventure',
"Children's",
'Drama',
'Mystery',
'Thriller',
'Drama',
'Documentary',
'Comedy',
'Comedy',
'Drama',
'Drama',
'Comedy',
"Children's",
'Comedy',
'Comedy',
'Romance',
'Thriller',
'Animation',
"Children's",
'Comedy',
'Musical',
'Action',
'Sci-Fi',
'Thriller',
'Adventure',
...]
extend函数：向列表尾部追加一个列表，将列表中的每个元素都追加进来，在原有列表上增加。
split函数：通过指定分隔符对字符串进行切片
genres=pd.unique(all_genres)
genres
array(['Animation', "Children's", 'Comedy', 'Adventure', 'Fantasy',
'Romance', 'Drama', 'Action', 'Crime', 'Thriller', 'Horror',
'Sci-Fi', 'Documentary', 'War', 'Musical', 'Mystery', 'Film-Noir',
'Western'], dtype=object)
zero_matrix=np.zeros((len(movies),len(genres)))
dummies=pd.DataFrame(zero_matrix,columns=genres)
dummies
AnimationChildren’sComedyAdventureFantasyRomanceDramaActionCrimeThrillerHorrorSci-FiDocumentaryWarMusicalMysteryFilm-NoirWestern00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.010.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.020.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.030.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.040.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.0…………………………………………………38780.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.038790.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.038800.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.038810.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.038820.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.0
3883 rows × 18 columns
gen = movies.genres[0]
gen
"Animation|Children's|Comedy"
gen.split('|')
['Animation', "Children's", 'Comedy']
dummies.columns.get_indexer(gen.split('|'))
array([0, 1, 2], dtype=int64)
pandas.index.get_indexer(target, method=None, limit=None, tolerance=None)
- 作用：确定target的值在给的pandas的index的位置
- target:输入的索引
- method:选的方法，包括ffill，bfill
- returns:从0到n – 1的整数表示这些位置处的索引与相应的target值匹配。
for i ,gen in enumerate(movies.genres):
indices=dummies.columns.get_indexer(gen.split('|'))
dummies.iloc[i,indices]=1
dummies
AnimationChildren’sComedyAdventureFantasyRomanceDramaActionCrimeThrillerHorrorSci-FiDocumentaryWarMusicalMysteryFilm-NoirWestern01.01.01.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.010.01.00.01.01.00.00.00.00.00.00.00.00.00.00.00.00.00.020.00.01.00.00.01.00.00.00.00.00.00.00.00.00.00.00.00.030.00.01.00.00.00.01.00.00.00.00.00.00.00.00.00.00.00.040.00.01.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.0…………………………………………………38780.00.01.00.00.00.00.00.00.00.00.00.00.00.00.00.00.00.038790.00.00.00.00.00.01.00.00.00.00.00.00.00.00.00.00.00.038800.00.00.00.00.00.01.00.00.00.00.00.00.00.00.00.00.00.038810.00.00.00.00.00.01.00.00.00.00.00.00.00.00.00.00.00.038820.00.00.00.00.00.01.00.00.01.00.00.00.00.00.00.00.00.0
3883 rows × 18 columns
movies_windic=movies.join(dummies.add_prefix('Genre_'))
movies_windic
movie_idtitlegenresGenre_AnimationGenre_Children’sGenre_ComedyGenre_AdventureGenre_FantasyGenre_RomanceGenre_Drama…Genre_CrimeGenre_ThrillerGenre_HorrorGenre_Sci-FiGenre_DocumentaryGenre_WarGenre_MusicalGenre_MysteryGenre_Film-NoirGenre_Western01Toy Story (1995)Animation|Children’s|Comedy1.01.01.00.00.00.00.0…0.00.00.00.00.00.00.00.00.00.012Jumanji (1995)Adventure|Children’s|Fantasy0.01.00.01.01.00.00.0…0.00.00.00.00.00.00.00.00.00.023Grumpier Old Men (1995)Comedy|Romance0.00.01.00.00.01.00.0…0.00.00.00.00.00.00.00.00.00.034Waiting to Exhale (1995)Comedy|Drama0.00.01.00.00.00.01.0…0.00.00.00.00.00.00.00.00.00.045Father of the Bride Part II (1995)Comedy0.00.01.00.00.00.00.0…0.00.00.00.00.00.00.00.00.00.0…………………………………………………………38783948Meet the Parents (2000)Comedy0.00.01.00.00.00.00.0…0.00.00.00.00.00.00.00.00.00.038793949Requiem for a Dream (2000)Drama0.00.00.00.00.00.01.0…0.00.00.00.00.00.00.00.00.00.038803950Tigerland (2000)Drama0.00.00.00.00.00.01.0…0.00.00.00.00.00.00.00.00.00.038813951Two Family House (2000)Drama0.00.00.00.00.00.01.0…0.00.00.00.00.00.00.00.00.00.038823952Contender, The (2000)Drama|Thriller0.00.00.00.00.00.01.0…0.01.00.00.00.00.00.00.00.00.0
3883 rows × 21 columns
movies_windic.iloc[0]
movie_id 1
title Toy Story (1995)
genres Animation|Children's|Comedy
Genre_Animation 1
Genre_Children's 1
Genre_Comedy 1
Genre_Adventure 0
Genre_Fantasy 0
Genre_Romance 0
Genre_Drama 0
Genre_Action 0
Genre_Crime 0
Genre_Thriller 0
Genre_Horror 0
Genre_Sci-Fi 0
Genre_Documentary 0
Genre_War 0
Genre_Musical 0
Genre_Mystery 0
Genre_Film-Noir 0
Genre_Western 0
Name: 0, dtype: object
7.3字符和操作
字符串对象方法
val = 'a,b, guido'
val.split(',')
['a', 'b', ' guido']
val= 'a,b, guido '
pieces=[x.strip() for x in val.split(',')]
pieces
['a', 'b', 'guido']
first,second,third=pieces
first+'::'+second+'::'+third
'a::b::guido'
语法： ‘sep’.join(seq)
参数说明
sep：分隔符。可以为空
seq：要连接的元素序列、字符串、元组、字典
上面的语法即：以sep作为分隔符，将seq所有的元素合并成一个新的字符串
返回值：返回一个以分隔符sep连接各个元素后生成的字符串
'::'.join(pieces)
'a::b::guido'
'guido' in val
True
index() 方法查找指定值的首次出现。
如果找不到该值，index() 方法将引发异常。
index() 方法与 find() 方法几乎相同，唯一的区别是，如果找不到该值，则 find() 方法将返回 -1。
val.index(',')
1
val.find(':')
-1
val.count(',')
2
val.replace(',','::')
'a::b:: guido '
正则式表达式
import re
text = "foo bar\t baz \tqux"
re.split('\s+',text)
['foo', 'bar', 'baz', 'qux']
regex=re.compile('\s+')
regex.split(text)
['foo', 'bar', 'baz', 'qux']
regex.findall(text)
[' ', '\t ', ' \t']
pandas的矢量化字符串函数
data = {'Dave': 'dave@google.com', 'Steve': 'steve@gmail.com',
'Rob': 'rob@gmail.com', 'Wes': np.nan}
data=pd.Series(data)
data
Dave dave@google.com
Steve steve@gmail.com
Rob rob@gmail.com
Wes NaN
dtype: object
data.isnull()
Dave False
Steve False
Rob False
Wes True
dtype: bool
data.str.contains('gmail')
Dave False
Steve True
Rob True
Wes NaN
dtype: object
Original: https://blog.csdn.net/m0_45055763/article/details/124202695
Author: 热爱学习的小鲁同学
Title: 第七章 数据清洗和处理
相关阅读
Title: python机器学习 train_test_split()函数用法解析及示例 划分训练集和测试集 以鸢尾数据为例 入门级讲解
文章目录
train_test_split()用法
python机器学习中常用 train_test_split()函数划分训练集和测试集，其用法语法如下：
- X_train, X_test, y_train, y_test = train_test_split(train_data, train_target, test_size, random_state, shuffle)
变量描述X_train划分的训练集数据X_test划分的测试集数据y_train划分的训练集标签y_test划分的测试集标签 参数描述train_data还未划分的数据集train_target还未划分的标签test_size分割比例，默认为
0.25
，即测试集占完整数据集的比例random_state随机数种子，应用于分割前对数据的洗牌。可以是int，RandomState实例或None，默认值=None。设成定值意味着，对于同一个数据集，只有第一次运行是随机的，随后多次分割只要rondom_state相同，则划分结果也相同。shuffle是否在分割前对完整数据进行洗牌（打乱），默认为True，打乱
以sklearn库内置的iris数据集（鸢尾数据集）为例，首先获取数据：
获取数据
from sklearn.model_selection import train_test_split
dataset = load_iris()
这里的dataset数据是 sklearn.utils.Bunch类型的数据，比较像字典
将其打印出~
print(dataset)
如下所示
从中取出其data属性和target属性， X是特征数组（也称数据集），y表示类别数组（也称标签）
X = dataset.data
y = dataset.target
此例中，有四个特征（即data的4列表示4个特征），分别是鸢尾植物的萼片的长，萼片的宽，花瓣的长，花瓣的宽。
X中共150行，即150个样本，类别数据总共有150个数据(对应150个样本的类别)。
print(y)
y的150个数据如上图，其中，有0,1,2三个取值，表示三种花：
012Iris Setosa(山鸢尾)Iris Versicolour(变色鸢尾)Iris Virginica(维吉尼亚鸢尾)
使用最简单的离散化算法，以均值为阈值，使大于阈值的特征值为1，小于阈值的特征值为0.
attribute_means = X.mean(axis=0)
X_d = np.array(X >= attribute_means, dtype='int')
print(X_d)
运行结果（成功将X的数据转换为bool类型）：
划分训练集和测试集
然后就是使用train_test_split()函数将数据划分训练集和测试集了。
random_state = 10
X_train, X_test, y_train, y_test = train_test_split(X_d, y, random_state=random_state)
print("There are {} training samples".format(y_train.shape[0]))
print("There are {} testing samples".format(y_test.shape[0]))
如图得到的数据中112/38接近3:1。分割成功！
完整代码脚手架
（将上述分步的代码写在一块儿方便复制使用）：
from sklearn.model_selection import train_test_split
dataset = load_iris()
X = dataset.data
y = dataset.target
attribute_means = X.mean(axis=0)
X_d = np.array(X >= attribute_means, dtype='int')
random_state = 10
X_train, X_test, y_train, y_test = train_test_split(X_d, y, random_state=random_state)
Original: https://blog.csdn.net/weixin_48964486/article/details/122866347
Author: 侯小啾
Title: python机器学习 train_test_split()函数用法解析及示例 划分训练集和测试集 以鸢尾数据为例 入门级讲解
原创文章受到原创版权保护。转载请注明出处：https://www.johngo689.com/312126/
转载文章受原作者版权保护。转载请注明原作者出处！
