HadISD Tutorial Six - Processing with Parquet#
NOTE Before beginning this tutorial, you should first read HadISD Tutorial One - Introduction to Station Data and ensure you complete the tutorials in order
This notebook groups the stations similarly to notebook 3, but produces outputs in the Parquet format rather than NetCDF files. This is much more efficient later when retrieving data, and the conversion is also faster.
[2]:
import tarfile
import gzip
import shutil
from pathlib import Path
import numpy as np
from datetime import datetime
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
import os
import xarray as xr
[3]:
UNPACKED_DIR = Path('/scratch/nf33/data/') / 'hadisd' / 'unpacked' # We need a place on disk to unpack the archives
PROCESSING_DIR = Path('/scratch/nf33/data/') / 'hadisd' / 'processing' # We need to cache some data on disk during reprocessing
PARQUET_DIR_JOB = Path('/scratch/nf33/data/') / 'hadisd' / 'parquet'
[4]:
files = list(UNPACKED_DIR.glob('*.nc'))
len(files)
[4]:
10393
[5]:
def simplify(ds):
'''
Here we move the latitude, longitude and elevation from a coordinate to a data variable
This fits the structure of the data more efficiently, allowing simpler concatenation.
'''
lats = xr.DataArray(data=np.full(len(ds.time), ds.latitude.values[0]), coords={'time': ds.time.values})
lons = xr.DataArray(data=np.full(len(ds.time), ds.longitude.values[0]), coords={'time': ds.time.values})
elev = xr.DataArray(data=np.full(len(ds.time), ds.elevation.values[0]), coords={'time': ds.time.values})
year = xr.DataArray(data=np.full(len(ds.time), ds.time.dt.year), coords={'time': ds.time.values})
ds = ds.reset_coords(names=('latitude', 'longitude', 'elevation'), drop=True)
ds['lat'] = lats
ds['lon'] = lons
ds['elev'] = elev
ds['year'] = year
ds = ds.drop_attrs()
ds = ds.drop_vars([
"input_station_id",
"quality_control_flags",
"flagged_obs",
"reporting_stats"
], errors="ignore")
return ds
[ ]:
[11]:
files = list(Path(UNPACKED_DIR).glob('*.nc'))
filegroups = [files[i:i + 100] for i in range(0, len(files), 100)]
print(len(filegroups)) # We come up with 134 such file groupings from the test data or 1040 for the full dataset
104
[12]:
_decades = [('1800', '1930'), # Just in case there is undocumented early data
('1930', '1940'), ('1940', '1950'), # Dataset begins in 1930, start by decade here
('1950', '1960'), ('1960', '1970'), ('1970', '1980'),
('1980', '1990'), ('1990', '2000'), ('2000', '2010'), # 1980 is a common time to start from
('2010', '2020'), ('2020', '2030')
]
[13]:
# This takes around 20-60 seconds per grouping. If you just want to get the hang of it, limit it to three groupings
# Otherwise, the test set have 67 groupings, so will take around half an hour to run
# The full set of stations will take several hours!
# For testing, just try three file groups
decades = map(lambda x: _decades[x], [2,4,6])
import time
import pyarrow as pa
from tqdm import tqdm
import pandas as pd
import pyarrow.parquet as pq
import pyarrow.dataset as pa_ds
from multiprocessing import Pool
def iter_batch(_files):
for f in tqdm(_files):
with xr.open_dataset(f, engine='netcdf4', chunks=None) as ds:
ds = simplify(ds)
yield pa.table(ds.to_dataframe())
def run_single_report(pool_arg):
ts = time.time()
report_id, fg = pool_arg
ts = time.time()
record_dir = f"{PARQUET_DIR_JOB}/report_{report_id}"
os.makedirs(record_dir, exist_ok=True)
pa_ds.write_dataset(
pa.concat_tables(iter_batch(fg)).sort_by("time"),
base_dir=record_dir,
partitioning=["year"],
format="parquet",
existing_data_behavior="overwrite_or_ignore"
)
te = time.time()
print(f"[parquet-group={report_id}] time taken total: {te - ts}s")
def parquet_processing():
n_workers = 4
with Pool(n_workers) as p:
p.map(run_single_report, enumerate(filegroups))
[14]:
%%time
# This uses up to 65GB of RAM for grouping sizes of 100, you may be able
# to run this on smaller machines by controlling the file group size and/or the
# number of workers
parquet_processing()
100%|██████████| 100/100 [00:31<00:00, 3.15it/s]
100%|██████████| 100/100 [00:37<00:00, 2.68it/s]
100%|██████████| 100/100 [00:37<00:00, 2.66it/s]
100%|██████████| 100/100 [00:40<00:00, 2.49it/s]
[parquet-group=7] time taken total: 43.72324585914612s
20%|██ | 20/100 [00:06<00:23, 3.44it/s]
[parquet-group=0] time taken total: 50.971986055374146s
1%| | 1/100 [00:00<00:44, 2.22it/s]]
[parquet-group=14] time taken total: 51.86360049247742s
11%|█ | 11/100 [00:03<00:35, 2.48it/s]
[parquet-group=21] time taken total: 55.37885141372681s
100%|██████████| 100/100 [00:33<00:00, 3.00it/s]
100%|██████████| 100/100 [00:32<00:00, 3.10it/s]
100%|██████████| 100/100 [00:34<00:00, 2.94it/s]
100%|██████████| 100/100 [00:31<00:00, 3.18it/s]
[parquet-group=8] time taken total: 45.8794207572937s
4%|▍ | 4/100 [00:01<00:27, 3.53it/s]
[parquet-group=15] time taken total: 43.57876634597778s
9%|▉ | 9/100 [00:07<00:55, 1.64it/s]
[parquet-group=1] time taken total: 46.094953298568726s
9%|▉ | 9/100 [00:02<00:37, 2.43it/s]]
[parquet-group=22] time taken total: 42.98993515968323s
100%|██████████| 100/100 [00:31<00:00, 3.19it/s]
100%|██████████| 100/100 [00:33<00:00, 2.97it/s]
100%|██████████| 100/100 [00:42<00:00, 2.34it/s]
100%|██████████| 100/100 [00:36<00:00, 2.74it/s]
[parquet-group=2] time taken total: 43.67014837265015s
1%| | 1/100 [00:00<00:32, 3.04it/s]
[parquet-group=16] time taken total: 46.08212184906006s
15%|█▌ | 15/100 [00:05<00:25, 3.36it/s]
[parquet-group=9] time taken total: 57.058125734329224s
3%|▎ | 3/100 [00:00<00:33, 2.93it/s]]
[parquet-group=23] time taken total: 49.32966756820679s
100%|██████████| 100/100 [00:31<00:00, 3.13it/s]
100%|██████████| 100/100 [00:34<00:00, 2.86it/s]
100%|██████████| 100/100 [00:34<00:00, 2.92it/s]
100%|██████████| 100/100 [00:34<00:00, 2.91it/s]
[parquet-group=17] time taken total: 44.06975245475769s
18%|█▊ | 18/100 [00:07<00:37, 2.19it/s]
[parquet-group=10] time taken total: 46.62184810638428s
20%|██ | 20/100 [00:08<00:31, 2.51it/s]
[parquet-group=3] time taken total: 53.31804633140564s
23%|██▎ | 23/100 [00:09<00:24, 3.14it/s]
[parquet-group=24] time taken total: 47.157567262649536s
100%|██████████| 100/100 [00:32<00:00, 3.07it/s]
100%|██████████| 100/100 [00:41<00:00, 2.39it/s]
100%|██████████| 100/100 [00:33<00:00, 2.97it/s]
100%|██████████| 100/100 [00:39<00:00, 2.52it/s]
[parquet-group=11] time taken total: 45.00732898712158s
6%|▌ | 6/100 [00:02<00:38, 2.46it/s]
[parquet-group=25] time taken total: 46.26852226257324s
9%|▉ | 9/100 [00:02<00:29, 3.09it/s]]
[parquet-group=18] time taken total: 58.442479848861694s
21%|██ | 21/100 [00:08<00:31, 2.53it/s]
[parquet-group=4] time taken total: 52.757867097854614s
100%|██████████| 100/100 [00:34<00:00, 2.88it/s]
100%|██████████| 100/100 [00:34<00:00, 2.91it/s]
100%|██████████| 100/100 [00:34<00:00, 2.92it/s]
100%|██████████| 100/100 [00:33<00:00, 2.96it/s]
[parquet-group=12] time taken total: 47.830859661102295s
4%|▍ | 4/100 [00:01<00:34, 2.78it/s]
[parquet-group=26] time taken total: 46.970720529556274s
17%|█▋ | 17/100 [00:05<00:18, 4.57it/s]
[parquet-group=19] time taken total: 47.439521074295044s
3%|▎ | 3/100 [00:01<00:44, 2.18it/s]]
[parquet-group=5] time taken total: 46.352010011672974s
100%|██████████| 100/100 [00:33<00:00, 2.96it/s]
100%|██████████| 100/100 [00:34<00:00, 2.93it/s]
100%|██████████| 100/100 [00:34<00:00, 2.91it/s]
100%|██████████| 100/100 [00:33<00:00, 2.99it/s]
[parquet-group=13] time taken total: 47.053099155426025s
1%| | 1/100 [00:00<00:12, 7.72it/s]
[parquet-group=27] time taken total: 45.73852300643921s
18%|█▊ | 18/100 [00:05<00:24, 3.35it/s]
[parquet-group=6] time taken total: 45.59616708755493s
14%|█▍ | 14/100 [00:05<00:25, 3.33it/s]
[parquet-group=20] time taken total: 47.71624302864075s
100%|██████████| 100/100 [00:30<00:00, 3.28it/s]
100%|██████████| 100/100 [00:30<00:00, 3.23it/s]
100%|██████████| 100/100 [00:38<00:00, 2.56it/s]
100%|██████████| 100/100 [00:34<00:00, 2.89it/s]
[parquet-group=35] time taken total: 41.261908769607544s
15%|█▌ | 15/100 [00:05<00:19, 4.28it/s]
[parquet-group=42] time taken total: 42.10287833213806s
12%|█▏ | 12/100 [00:04<00:29, 2.97it/s]
[parquet-group=28] time taken total: 52.4815673828125s
37%|███▋ | 37/100 [00:12<00:22, 2.80it/s]
[parquet-group=49] time taken total: 48.30430769920349s
100%|██████████| 100/100 [00:33<00:00, 3.02it/s]
100%|██████████| 100/100 [00:34<00:00, 2.87it/s]
100%|██████████| 100/100 [00:34<00:00, 2.90it/s]
98%|█████████▊| 98/100 [00:32<00:00, 3.83it/s]
[parquet-group=36] time taken total: 45.35342621803284s
100%|██████████| 100/100 [00:33<00:00, 3.00it/s]
16%|█▌ | 16/100 [00:06<00:43, 1.91it/s]
[parquet-group=43] time taken total: 46.54223012924194s
32%|███▏ | 32/100 [00:11<00:23, 2.89it/s]
[parquet-group=50] time taken total: 44.97547698020935s
33%|███▎ | 33/100 [00:12<00:21, 3.12it/s]
[parquet-group=29] time taken total: 47.01907467842102s
100%|██████████| 100/100 [00:32<00:00, 3.09it/s]
89%|████████▉ | 89/100 [00:30<00:04, 2.43it/s]
[parquet-group=37] time taken total: 43.09721517562866s
100%|██████████| 100/100 [00:36<00:00, 2.76it/s]
100%|██████████| 100/100 [00:34<00:00, 2.90it/s]
100%|██████████| 100/100 [00:35<00:00, 2.78it/s]
46%|████▌ | 46/100 [00:15<00:22, 2.40it/s]
[parquet-group=30] time taken total: 46.78835964202881s
47%|████▋ | 47/100 [00:16<00:24, 2.16it/s]
[parquet-group=44] time taken total: 52.71414923667908s
48%|████▊ | 48/100 [00:16<00:23, 2.17it/s]
[parquet-group=51] time taken total: 48.063724756240845s
100%|██████████| 100/100 [00:34<00:00, 2.93it/s]
86%|████████▌ | 86/100 [00:28<00:04, 2.87it/s]
[parquet-group=38] time taken total: 45.497915744781494s
100%|██████████| 100/100 [00:33<00:00, 3.02it/s]
100%|██████████| 100/100 [00:32<00:00, 3.05it/s]
100%|██████████| 100/100 [00:34<00:00, 2.91it/s]
46%|████▌ | 46/100 [00:14<00:24, 2.16it/s]
[parquet-group=31] time taken total: 44.51861834526062s
51%|█████ | 51/100 [00:16<00:19, 2.46it/s]
[parquet-group=52] time taken total: 45.26561188697815s
15%|█▌ | 15/100 [00:04<00:27, 3.15it/s]
[parquet-group=45] time taken total: 48.628894567489624s
100%|██████████| 100/100 [00:30<00:00, 3.24it/s]
80%|████████ | 80/100 [00:24<00:05, 3.44it/s]
[parquet-group=39] time taken total: 41.629621744155884s
100%|██████████| 100/100 [00:32<00:00, 3.06it/s]
100%|██████████| 100/100 [00:31<00:00, 3.22it/s]
100%|██████████| 100/100 [00:30<00:00, 3.26it/s]
56%|█████▌ | 56/100 [00:17<00:10, 4.01it/s]
[parquet-group=32] time taken total: 44.13243317604065s
1%| | 1/100 [00:00<00:34, 2.89it/s]]
[parquet-group=53] time taken total: 42.75717234611511s
5%|▌ | 5/100 [00:01<00:34, 2.78it/s]]
[parquet-group=46] time taken total: 41.18223595619202s
100%|██████████| 100/100 [00:32<00:00, 3.05it/s]
83%|████████▎ | 83/100 [00:27<00:04, 4.25it/s]
[parquet-group=40] time taken total: 45.0415096282959s
100%|██████████| 100/100 [00:31<00:00, 3.18it/s]
100%|██████████| 100/100 [00:33<00:00, 3.01it/s]
100%|██████████| 100/100 [00:38<00:00, 2.61it/s]
50%|█████ | 50/100 [00:15<00:19, 2.56it/s]
[parquet-group=54] time taken total: 43.489718437194824s
1%| | 1/100 [00:00<00:56, 1.76it/s]]
[parquet-group=47] time taken total: 43.045570850372314s
26%|██▌ | 26/100 [00:08<00:20, 3.56it/s]
[parquet-group=33] time taken total: 53.607056856155396s
100%|██████████| 100/100 [00:33<00:00, 3.02it/s]
60%|██████ | 60/100 [00:18<00:14, 2.83it/s]
[parquet-group=41] time taken total: 44.92406463623047s
100%|██████████| 100/100 [00:33<00:00, 2.97it/s]
100%|██████████| 100/100 [00:37<00:00, 2.68it/s]
100%|██████████| 100/100 [00:33<00:00, 2.99it/s]
49%|████▉ | 49/100 [00:17<00:18, 2.69it/s]
[parquet-group=48] time taken total: 45.06527805328369s
12%|█▏ | 12/100 [00:04<00:24, 3.66it/s]
[parquet-group=55] time taken total: 50.94333338737488s
73%|███████▎ | 73/100 [00:26<00:10, 2.57it/s]
[parquet-group=34] time taken total: 44.916912317276s
100%|██████████| 100/100 [00:34<00:00, 2.89it/s]
57%|█████▋ | 57/100 [00:21<00:14, 2.98it/s]
[parquet-group=56] time taken total: 47.93559980392456s
100%|██████████| 100/100 [00:34<00:00, 2.89it/s]
100%|██████████| 100/100 [00:32<00:00, 3.09it/s]
53%|█████▎ | 53/100 [00:17<00:11, 3.95it/s]
[parquet-group=70] time taken total: 42.82279562950134s
100%|██████████| 100/100 [00:40<00:00, 2.48it/s]
5%|▌ | 5/100 [00:01<00:30, 3.07it/s]]
[parquet-group=63] time taken total: 49.7618465423584s
31%|███ | 31/100 [00:12<00:24, 2.86it/s]
[parquet-group=77] time taken total: 53.309619665145874s
100%|██████████| 100/100 [00:31<00:00, 3.18it/s]
35%|███▌ | 35/100 [00:10<00:24, 2.68it/s]
[parquet-group=57] time taken total: 42.09168767929077s
100%|██████████| 100/100 [00:32<00:00, 3.04it/s]
100%|██████████| 100/100 [00:35<00:00, 2.83it/s]
88%|████████▊ | 88/100 [00:30<00:03, 3.25it/s]
[parquet-group=71] time taken total: 44.41039538383484s
13%|█▎ | 13/100 [00:04<00:20, 4.20it/s]
[parquet-group=64] time taken total: 46.73083019256592s
100%|██████████| 100/100 [00:34<00:00, 2.89it/s]
100%|██████████| 100/100 [00:33<00:00, 2.95it/s]
53%|█████▎ | 53/100 [00:16<00:12, 3.80it/s]
[parquet-group=78] time taken total: 47.11274743080139s
34%|███▍ | 34/100 [00:09<00:20, 3.21it/s]
[parquet-group=58] time taken total: 45.931764125823975s
100%|██████████| 100/100 [00:34<00:00, 2.92it/s]
100%|██████████| 100/100 [00:33<00:00, 3.02it/s]
100%|██████████| 100/100 [00:30<00:00, 3.24it/s]
73%|███████▎ | 73/100 [00:23<00:09, 2.83it/s]
[parquet-group=72] time taken total: 49.94980549812317s
8%|▊ | 8/100 [00:01<00:20, 4.60it/s]]
[parquet-group=65] time taken total: 47.58602261543274s
29%|██▉ | 29/100 [00:08<00:20, 3.45it/s]
[parquet-group=79] time taken total: 41.38049554824829s
100%|██████████| 100/100 [00:33<00:00, 2.95it/s]
63%|██████▎ | 63/100 [00:21<00:12, 3.03it/s]
[parquet-group=59] time taken total: 45.28192353248596s
100%|██████████| 100/100 [00:32<00:00, 3.08it/s]
100%|██████████| 100/100 [00:32<00:00, 3.07it/s]
100%|██████████| 100/100 [00:32<00:00, 3.12it/s]
63%|██████▎ | 63/100 [00:21<00:15, 2.34it/s]
[parquet-group=73] time taken total: 43.42348861694336s
13%|█▎ | 13/100 [00:03<00:20, 4.16it/s]
[parquet-group=66] time taken total: 45.5210964679718s
10%|█ | 10/100 [00:03<00:24, 3.72it/s]
[parquet-group=80] time taken total: 42.479992389678955s
100%|██████████| 100/100 [00:33<00:00, 2.96it/s]
90%|█████████ | 90/100 [00:30<00:04, 2.47it/s]
[parquet-group=60] time taken total: 52.62767720222473s
100%|██████████| 100/100 [00:34<00:00, 2.88it/s]
100%|██████████| 100/100 [00:32<00:00, 3.11it/s]
100%|██████████| 100/100 [00:32<00:00, 3.11it/s]
42%|████▏ | 42/100 [00:15<00:18, 3.07it/s]
[parquet-group=74] time taken total: 46.452868938446045s
6%|▌ | 6/100 [00:01<00:29, 3.20it/s]]
[parquet-group=67] time taken total: 44.24177885055542s
13%|█▎ | 13/100 [00:03<00:27, 3.14it/s]
[parquet-group=81] time taken total: 43.21087026596069s
100%|██████████| 100/100 [00:36<00:00, 2.76it/s]
100%|██████████| 100/100 [00:31<00:00, 3.16it/s]
80%|████████ | 80/100 [00:28<00:08, 2.35it/s]
[parquet-group=61] time taken total: 48.2315411567688s
100%|██████████| 100/100 [00:33<00:00, 2.98it/s]
100%|██████████| 100/100 [00:36<00:00, 2.77it/s]
31%|███ | 31/100 [00:10<00:20, 3.30it/s]
[parquet-group=75] time taken total: 43.57420611381531s
52%|█████▏ | 52/100 [00:17<00:21, 2.26it/s]
[parquet-group=82] time taken total: 46.137006759643555s
0%| | 0/100 [00:00<?, ?it/s]
[parquet-group=68] time taken total: 48.57669949531555s
100%|██████████| 100/100 [00:36<00:00, 2.74it/s]
100%|██████████| 100/100 [00:32<00:00, 3.07it/s]
87%|████████▋ | 87/100 [00:31<00:06, 1.95it/s]
[parquet-group=62] time taken total: 48.89878511428833s
100%|██████████| 100/100 [00:35<00:00, 2.82it/s]
100%|██████████| 100/100 [00:37<00:00, 2.68it/s]
[parquet-group=76] time taken total: 43.9840829372406s
27%|██▋ | 27/100 [00:09<00:40, 1.79it/s]
[parquet-group=69] time taken total: 47.15520095825195s
47%|████▋ | 47/100 [00:18<00:22, 2.40it/s]
[parquet-group=83] time taken total: 49.68277645111084s
100%|██████████| 100/100 [00:36<00:00, 2.75it/s]
100%|██████████| 100/100 [00:33<00:00, 2.95it/s]
98%|█████████▊| 98/100 [00:34<00:00, 2.96it/s]
[parquet-group=84] time taken total: 50.5812349319458s
2%|▏ | 2/100 [00:00<00:37, 2.61it/s]]
[parquet-group=91] time taken total: 45.43888211250305s
100%|██████████| 100/100 [00:35<00:00, 2.82it/s]
39%|███▉ | 39/100 [00:13<00:21, 2.87it/s]
[parquet-group=98] time taken total: 48.93868064880371s
100%|██████████| 100/100 [00:33<00:00, 3.01it/s]
100%|██████████| 100/100 [00:33<00:00, 2.96it/s]
89%|████████▉ | 89/100 [00:30<00:03, 2.76it/s]
[parquet-group=85] time taken total: 44.86662673950195s
4%|▍ | 4/100 [00:01<00:30, 3.10it/s]]
[parquet-group=92] time taken total: 45.50661063194275s
100%|██████████| 100/100 [00:33<00:00, 2.96it/s]
34%|███▍ | 34/100 [00:12<00:28, 2.31it/s]
[parquet-group=99] time taken total: 44.34385824203491s
100%|██████████| 100/100 [00:36<00:00, 2.72it/s]
100%|██████████| 100/100 [00:36<00:00, 2.74it/s]
86%|████████▌ | 86/100 [00:35<00:04, 2.91it/s]
[parquet-group=93] time taken total: 48.015738010406494s
8%|▊ | 8/100 [00:02<00:27, 3.40it/s]]
[parquet-group=86] time taken total: 51.90695858001709s
100%|██████████| 100/100 [00:40<00:00, 2.45it/s]
57%|█████▋ | 57/100 [00:18<00:09, 4.48it/s]
[parquet-group=100] time taken total: 54.474796772003174s
100%|██████████| 100/100 [00:31<00:00, 3.22it/s]
100%|██████████| 100/100 [00:33<00:00, 3.00it/s]
65%|██████▌ | 65/100 [00:23<00:18, 1.93it/s]
[parquet-group=94] time taken total: 42.38950562477112s
12%|█▏ | 12/100 [00:04<00:27, 3.20it/s]
[parquet-group=87] time taken total: 44.22594451904297s
100%|██████████| 100/100 [00:35<00:00, 2.78it/s]
56%|█████▌ | 56/100 [00:19<00:11, 3.95it/s]
[parquet-group=101] time taken total: 47.65267515182495s
100%|██████████| 100/100 [00:31<00:00, 3.17it/s]
100%|██████████| 100/100 [00:33<00:00, 2.95it/s]
64%|██████▍ | 64/100 [00:21<00:09, 3.69it/s]
[parquet-group=95] time taken total: 45.43871712684631s
11%|█ | 11/100 [00:04<00:32, 2.75it/s]
[parquet-group=88] time taken total: 45.98599314689636s
100%|██████████| 100/100 [00:34<00:00, 2.92it/s]
58%|█████▊ | 58/100 [00:23<00:14, 2.87it/s]
[parquet-group=102] time taken total: 44.91448926925659s
100%|██████████| 100/100 [00:37<00:00, 2.66it/s]
100%|██████████| 100/100 [00:36<00:00, 2.74it/s]
83%|████████▎ | 77/93 [00:28<00:05, 3.04it/s]
[parquet-group=96] time taken total: 52.859530210494995s
5%|▌ | 5/100 [00:01<00:23, 4.05it/s]
[parquet-group=89] time taken total: 49.49197030067444s
100%|██████████| 93/93 [00:33<00:00, 2.76it/s]]
35%|███▌ | 35/100 [00:12<00:23, 2.72it/s]
[parquet-group=103] time taken total: 43.45616912841797s
100%|██████████| 100/100 [00:31<00:00, 3.19it/s]
100%|██████████| 100/100 [00:39<00:00, 2.52it/s]
[parquet-group=97] time taken total: 45.09441018104553s
[parquet-group=90] time taken total: 56.166526317596436s
CPU times: user 7.55 s, sys: 3.4 s, total: 10.9 s
Wall time: 21min 56s
[10]:
print('done')
done