import numpy as np import pandas as pd import pyarrow as pa type_ = np.dtype('float64') DATA_SIZE = (1 << 30) NCOLS = 100 NROWS = DATA_SIZE / NCOLS / np.dtype(type_).itemsize data = { 'c' + str(i): np.random.randn(NROWS) for i in range(NCOLS) }