{ "cells": [ { "cell_type": "markdown", "id": "0a55c57e", "metadata": {}, "source": [ "# Create chips from a temporal set of Sentinel 2 snapshots\n", "\n", "- create chip bounds augmented by sliding\n", "- sample S2\n", " - loop over tiles\n", " - get bounds by sjoin\n", " - loop over times\n", " - loop over bounds (bag)\n", " - check cloud probability/cloud mask\n", " - sample a chip\n", " - check for missing values\n", " - save a chip" ] }, { "cell_type": "code", "execution_count": 1, "id": "52ab33a3", "metadata": {}, "outputs": [], "source": [ "# this cell does not work in the gds_env:7.0 and requires newer versions of GDAL and pyogrio. Works with GDAL 3.4.1, pyogrio 0.3.0.\n", "# import pyogrio\n", "\n", "# tiles_of_interest = [\n", "# '29UQR', '30UUA', '29UQS', '30UWA', '30UVA', '30UXB', '30UWB',\n", "# '30UVB', '30UXC', '30UWC', '30UVC', '30UXD', '30UWD', '30UUB',\n", "# '29UQT', '30UUC', '29UQU', '30UUD', '30UVD', '30UXE', '30UWE',\n", "# '30UVE', '30UXF', '30UWF', '30UVF', '30UXG', '30UWG', '30UVG',\n", "# '29UQV', '30UUE', '30UUF', '30UUG', '30VWH', '30VVH', '29VPC',\n", "# '30VUH', '30UYB', '31UCS', '30UYC', '31UCT', '31UDT', '30UYD',\n", "# '31UCU', '31UDU', '30UYE', '31UCV'\n", "# ]\n", "\n", "# tile_geometry = pyogrio.read_dataframe(\"https://sentinels.copernicus.eu/documents/247904/1955685/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml\")\n", "# tile_geometry = tile_geometry[tile_geometry.Name.isin(tiles_of_interest)].explode(index_parts=False)\n", "# tile_geometry[tile_geometry.geom_type == \"Polygon\"].to_file(\"../chips_gb/sentinel_tiles.gpkg\")" ] }, { "cell_type": "code", "execution_count": 1, "id": "a508b469", "metadata": {}, "outputs": [], "source": [ "import os\n", "import pyogrio\n", "import geopandas\n", "import glob\n", "import math\n", "import rasterio\n", "import rioxarray\n", "import matplotlib.pyplot as plt\n", "import pandas\n", "from pathlib import Path\n", "from dask.distributed import Client, LocalCluster, as_completed" ] }, { "cell_type": "code", "execution_count": 2, "id": "0ec9b1a9", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "
\n", "
\n", "

Client

\n", "

Client-42df0a96-90dd-11ec-b8ad-33d6c79f778e

\n", " \n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "
Connection method: Cluster objectCluster type: distributed.LocalCluster
\n", " Dashboard: http://127.0.0.1:8787/status\n", "
\n", "\n", " \n", "
\n", "

Cluster Info

\n", "
\n", "
\n", "
\n", "
\n", "

LocalCluster

\n", "

367344c5

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", " \n", "
\n", " Dashboard: http://127.0.0.1:8787/status\n", " \n", " Workers: 16\n", "
\n", " Total threads: 16\n", " \n", " Total memory: 125.54 GiB\n", "
Status: runningUsing processes: True
\n", "\n", "
\n", " \n", "

Scheduler Info

\n", " \n", "\n", "
\n", "
\n", "
\n", "
\n", "

Scheduler

\n", "

Scheduler-552ece40-15b3-4532-8c0e-81ff9a02c8af

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
\n", " Comm: tcp://127.0.0.1:45485\n", " \n", " Workers: 16\n", "
\n", " Dashboard: http://127.0.0.1:8787/status\n", " \n", " Total threads: 16\n", "
\n", " Started: Just now\n", " \n", " Total memory: 125.54 GiB\n", "
\n", "
\n", "
\n", "\n", "
\n", " \n", "

Workers

\n", " \n", "\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 0

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:37517\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:45131/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:33627\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ti72jamp\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 1

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:33861\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:37831/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:34055\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-li1ckzid\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 2

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:40359\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:34507/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:37495\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ugjw3vp0\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 3

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:32769\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:40385/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:44175\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-_of1varo\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 4

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:42397\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:38323/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:36105\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-deugq5ut\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 5

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:45319\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:33163/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:43973\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-w4fp16zz\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 6

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:43325\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:38091/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:33521\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-dh_x9s5w\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 7

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:44189\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:35107/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:42629\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-p13o2j4n\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 8

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:44887\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:45457/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:39577\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-6d01bipq\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 9

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:39487\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:38543/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:34711\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-2uyqfsay\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 10

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:34551\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:41183/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:39319\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-jvypepw4\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 11

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:42383\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:45585/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:45477\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-qxm_twpf\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 12

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:37287\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:43957/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:37429\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-1w8shg4v\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 13

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:43141\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:42381/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:41919\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-s0uasck0\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 14

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:39213\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:34905/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:42205\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-qwd7v3ij\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 15

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:39195\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:35877/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:42625\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-z4tun56j\n", "
\n", "
\n", "
\n", "
\n", " \n", "\n", "
\n", "
\n", "\n", "
\n", "
\n", "
\n", "
\n", " \n", "\n", "
\n", "
" ], "text/plain": [ "" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "client = Client(LocalCluster(n_workers=16))\n", "client" ] }, { "cell_type": "code", "execution_count": 3, "id": "6537fb1e", "metadata": {}, "outputs": [], "source": [ "specs = {\n", " 'chip_size': 32,\n", " 'folder': (\n", " '/home/jovyan/work/chips_gb/32_temporal/'\n", " ),\n", "}" ] }, { "cell_type": "code", "execution_count": 4, "id": "6f0953cd", "metadata": {}, "outputs": [], "source": [ "s = specs[\"chip_size\"]" ] }, { "cell_type": "code", "execution_count": 5, "id": "2024ed97", "metadata": {}, "outputs": [], "source": [ "tiles = pyogrio.read_dataframe(\"../chips_gb/sentinel_tiles.gpkg\").to_crs(27700)" ] }, { "cell_type": "code", "execution_count": 12, "id": "ba613601", "metadata": {}, "outputs": [], "source": [ "def process_tile(row):\n", " epsg = pandas.read_html(row.description)[0].loc[1, 1]\n", " bounds_in = bounds.iloc[bounds.sindex.query(row.geometry, predicate=\"contains\")].to_crs(int(epsg))\n", " times = glob.glob(f\"../../data/Sentinel2/{row.Name}/*\")\n", " for t in times:\n", " try:\n", " cloud_proba = glob.glob(f\"{t}/*CLDPRB*\")[0]\n", " tci = glob.glob(f\"{t}/*TCI*\")[0]\n", " for tup in bounds_in.itertuples():\n", " with rasterio.open(cloud_proba) as f:\n", " cldprb, transform = rasterio.mask.mask(\n", " f, [tup.geometry], crop=True, all_touched=True\n", " )\n", " if (cldprb > 10).sum() < 10:\n", " with rasterio.open(tci) as src:\n", " profile = src.profile\n", " profile.update(\n", " width=s,\n", " height=s,\n", " driver=\"GTiff\",\n", " dtype=rasterio.uint8,\n", " tiled=False\n", " )\n", " try:\n", " img, transform = rasterio.mask.mask(\n", " src, [tup.geometry], crop=True, all_touched=True\n", " )\n", " _, w, h = img.shape\n", " rw = (w - s) / 2\n", " rh = (h - s) / 2\n", " img = img[:, math.floor(rw):-math.ceil(rw), math.floor(rh):-math.ceil(rh)]\n", " if ((img[0] == img[1]) & (img[1] == img[2])).sum() < 3: # filter missingness\n", "\n", " path = f\"{specs['folder']}{tup.signature_type}/{tup.X}_{tup.Y}_{Path(t).stem}.tif\"\n", " with rasterio.open(path, 'w', **profile) as dst:\n", " dst.write(img.astype(rasterio.uint8))\n", " except ValueError:\n", " pass\n", " except IndexError:\n", " pass\n", " return f\"Tile {row.Name} processed sucessfully.\" " ] }, { "cell_type": "code", "execution_count": null, "id": "27128cdc", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Tile 29VPC processed sucessfully.\n", "Tile 29UQU processed sucessfully.\n", "Tile 29UQV processed sucessfully.\n", "Tile 30UUE processed sucessfully.\n", "Tile 30UUD processed sucessfully.\n", "Tile 30UWA processed sucessfully.\n", "Tile 29UQS processed sucessfully.\n", "Tile 30UUF processed sucessfully.\n", "Tile 29UQR processed sucessfully.\n", "Tile 30UUB processed sucessfully.\n", "Tile 30UUA processed sucessfully.\n", "Tile 29UQT processed sucessfully.\n", "Tile 30UUC processed sucessfully.\n", "Tile 30UVA processed sucessfully.\n", "Tile 30UVD processed sucessfully.\n", "Tile 30UUG processed sucessfully.\n", "Tile 30UVF processed sucessfully.\n", "Tile 30UXG processed sucessfully.\n", "Tile 30UVE processed sucessfully.\n", "Tile 30UVB processed sucessfully.\n", "Tile 30UWB processed sucessfully.\n", "Tile 30UYE processed sucessfully.\n", "Tile 30UWC processed sucessfully.\n", "Tile 30VUH processed sucessfully.\n", "Tile 30VWH processed sucessfully.\n", "Tile 30UYB processed sucessfully.\n", "Tile 30UXF processed sucessfully.\n", "Tile 30UYD processed sucessfully.\n", "Tile 31UCV processed sucessfully.\n", "Tile 31UDT processed sucessfully.\n", "Tile 30VVH processed sucessfully.\n", "Tile 31UDU processed sucessfully.\n", "Tile 30UVG processed sucessfully.\n", "Tile 31UCS processed sucessfully.\n", "Tile 30UWG processed sucessfully.\n", "Tile 31UCU processed sucessfully.\n", "Tile 30UXE processed sucessfully.\n", "Tile 30UVC processed sucessfully.\n", "Tile 30UWF processed sucessfully.\n", "Tile 30UXB processed sucessfully.\n" ] } ], "source": [ "subsets = [\"train\", \"validation\", \"secret\"]\n", "subsets = [\"secret\"]\n", "\n", "for sub in subsets:\n", " specs['folder'] = f'/home/jovyan/work/chips_gb/32_temporal/{sub}/'\n", " \n", " bounds = geopandas.read_parquet(f\"../chips_gb/slided_{sub}_50k.pq\")\n", " centroid = bounds.centroid\n", " bounds['X'] = centroid.x.astype(int)\n", " bounds['Y'] = centroid.y.astype(int)\n", " \n", " for t in bounds.signature_type.unique():\n", " os.makedirs(f\"{specs['folder']}{t}\", exist_ok=True)\n", " \n", " inputs = tiles.itertuples()\n", " futures = [client.submit(process_tile, next(inputs)) for i in range(16)]\n", " ac = as_completed(futures)\n", " for finished_future in ac:\n", " # submit new future \n", " try:\n", " new_future = client.submit(process_tile, next(inputs))\n", " ac.add(new_future)\n", " except StopIteration:\n", " pass\n", " print(finished_future.result())\n", " " ] }, { "cell_type": "code", "execution_count": 13, "id": "b938b6a9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "distributed.nanny - WARNING - Worker process still alive after 1.5999980926513673 seconds, killing\n" ] }, { "data": { "text/html": [ "
\n", "
\n", "
\n", "

Client

\n", "

Client-42df0a96-90dd-11ec-b8ad-33d6c79f778e

\n", " \n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "
Connection method: Cluster objectCluster type: distributed.LocalCluster
\n", " Dashboard: http://127.0.0.1:8787/status\n", "
\n", "\n", " \n", "
\n", "

Cluster Info

\n", "
\n", "
\n", "
\n", "
\n", "

LocalCluster

\n", "

367344c5

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", " \n", "
\n", " Dashboard: http://127.0.0.1:8787/status\n", " \n", " Workers: 16\n", "
\n", " Total threads: 16\n", " \n", " Total memory: 125.54 GiB\n", "
Status: runningUsing processes: True
\n", "\n", "
\n", " \n", "

Scheduler Info

\n", " \n", "\n", "
\n", "
\n", "
\n", "
\n", "

Scheduler

\n", "

Scheduler-552ece40-15b3-4532-8c0e-81ff9a02c8af

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
\n", " Comm: tcp://127.0.0.1:45485\n", " \n", " Workers: 16\n", "
\n", " Dashboard: http://127.0.0.1:8787/status\n", " \n", " Total threads: 16\n", "
\n", " Started: 2 days ago\n", " \n", " Total memory: 125.54 GiB\n", "
\n", "
\n", "
\n", "\n", "
\n", " \n", "

Workers

\n", " \n", "\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 0

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:32787\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:43601/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:33627\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-j8oryagm\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 1

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:34069\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:44247/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:34055\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-1h58gw8c\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 2

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:37003\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:35457/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:37495\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-__ba_jqe\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 3

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:34777\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:36987/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:44175\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-_lbtrlex\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 4

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:45797\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:39637/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:36105\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-cas91jbc\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 5

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:35143\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:45425/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:43973\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ynoiyio4\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 6

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:37719\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:40693/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:33521\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-n9gwy0ys\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 7

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:41083\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:40535/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:42629\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-zk9pyujw\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 8

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:38551\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:41691/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:39577\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ibiaq9w3\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 9

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:40411\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:34633/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:34711\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-kfm98rib\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 10

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:40351\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:38361/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:39319\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-e3dkz0qp\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 11

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:38803\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:36489/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:45477\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-jfekx6or\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 12

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:44583\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:43459/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:37429\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-o46cbhao\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 13

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:42945\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:41571/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:41919\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-_hhso_h2\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 14

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:41585\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:42929/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:42205\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-f8yiasy_\n", "
\n", "
\n", "
\n", "
\n", " \n", "
\n", "
\n", "
\n", "
\n", " \n", "

Worker: 15

\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", " \n", "\n", " \n", "\n", "
\n", " Comm: tcp://127.0.0.1:37511\n", " \n", " Total threads: 1\n", "
\n", " Dashboard: http://127.0.0.1:44685/status\n", " \n", " Memory: 7.85 GiB\n", "
\n", " Nanny: tcp://127.0.0.1:42625\n", "
\n", " Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-r6tp3sta\n", "
\n", "
\n", "
\n", "
\n", " \n", "\n", "
\n", "
\n", "\n", "
\n", "
\n", "
\n", "
\n", " \n", "\n", "
\n", "
" ], "text/plain": [ "" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "client.restart()" ] }, { "cell_type": "code", "execution_count": null, "id": "6cf24708", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }