用几行代码显著加快pandas速度的6种方法¶

import pandas as pd
import numpy as np
import numba

# create a table of 100,000 rows and 4 columns filled with random numbers from 0 to 100
df = pd.DataFrame(np.random.randint(0,100,size=(100000, 4)),columns=['a', 'b', 'c', 'd'])

# function for creating new col
def multiply(x):
  return x * 5

# optimized version of this function
@numba.vectorize
def multiply_numba(x):
  return x * 5

# our function
In [1]: %timeit df['new_col'] = df['a'].apply(multiply)
23.9 ms ± 1.93 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

# pandas implementation
In [2]: %timeit df['new_col'] = df['a'] * 5
545 µs ± 21.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

# out function, optimized by numba
# we use vector of values so that numba itself performs the loop optimization
In [3]: %timeit df['new_col'] = multiply_numba(df['a'].to_numpy())
329 µs ± 2.37 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

# square the values and take their mean
def square_mean(row):
  row = np.power(row, 2)
  return np.mean(row)
# usage:
# df['new_col'] = df.apply(square_mean, axis=1)

# numba does not know how to work with pandas primitives (Dataframe, Series, etc.)
# so that we use a two-dimensional numpy array
@numba.njit
def square_mean_numba(arr):
  res = np.empty(arr.shape[0])
  arr = np.power(arr, 2)
  for i in range(arr.shape[0]):
    res[i] = np.mean(arr[i])
  return res
# usage:
# df['new_col'] = square_mean_numba(df.to_numpy())

df = pd.read_csv('abcnews-date-text.csv', header=0)
# increase the dataset 10 times, adding copies to the end
df = pd.concat([df] * 10)
df.head()

# calculate the average word length in the title
def mean_word_len(line):
  # this cycle just complicates the task
  for i in range(6):
    words = [len(i) for i in line.split()]
    res = sum(words) / len(words)
    return res

def compute_avg_word(df):
  return df['headline_text'].apply(mean_word_len)

from multiprocessing import Pool
# I have 4 cores
n_cores = 4
pool = Pool(n_cores)

def apply_parallel(df, func):
  # split dataframe
  df_split = np.array_split(df, n_cores)
  # calculate metrics for each and concatenate
  df = pd.concat(pool.map(func, df_split))
  return df

# df['new_col'] = apply_parallel(df, compute_avg_word)

from pandarallel import pandarallel
# pandarallel will determine how many cores you have, but you can specify it yourself
pandarallel.initialize()

df['headline_text'].parallel_apply(mean_word_len)

def multiply(x):
    return x * 5

# df['publish_date'].apply(multiply)
# df['publish_date'].swifter.apply(multiply)
# df['publish_date'].parallel_apply(multiply)
# multiply(df['publish_date'])

# calculate the average word length in the title
def mean_word_len(line):
  # this cycle just complicates the task
  for i in range(6):
    words = [len(i) for i in line.split()]
    res = sum(words) / len(words)
    return res

# to work with strings, we initiate a special function allow_dask_on_strings ()
df['headline_text'].swifter.allow_dask_on_strings().apply(mean_word_len)

# The Dask engine is currently considered experimental, so I use ray
%env MODIN_ENGINE=ray
import modin.pandas as mpd

df = mpd.read_csv('abcnews-date-text.csv', header=0)
df = mpd.concat([df] * 15)
df.to_csv('big_csv.csv')

In [1]: %timeit mpd.read_csv('big_csv.csv', header=0)
8.61 s ± 176 ms per loop (mean ± std. dev. of 5 runs, 1 loop each)

In [2]: %timeit pd.read_csv('big_csv.csv', header=0)
22.9 s ± 1.95 s per loop (mean ± std. dev. of 5 runs, 1 loop each)

# This time I used numerical data
df = pd.DataFrame(np.random.randint(0, 100, size=(10**7, 6)), columns=list('abcdef'))

from distributed import Client
# create local cluster
client = Client(n_workers=8

import dask.dataframe as dd

In [1]: %timeit dd.read_csv('big_csv.csv', header=0)
6.79 s ± 798 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)In [2]: %timeit pd.read_csv('big_csv.csv', header=0)
19.8 s ± 2.75 s per loop (mean ± std. dev. of 7 runs, 1 loop each)

# compute () is needed because all calculations in dask are lazy
# dd.from_pandas - method for converting pandas dataframe to dask version
dd.from_pandas(df, npartitions=8).apply(mean_word_len, meta=(float)).compute()

# same numerical data as for modin
df = pd.DataFrame(np.random.randint(0, 100, size=(10**7, 6)), columns=list('abcdef'))