\\\"]}\\n\",\n \")\\n\",\n \"```\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"70353a53-f0cb-4eae-8c18-93b7ef07eb14\",\n \"metadata\": {},\n \"source\": [\n \"# Dashboard Visualization with `MLEDashboard` 🎞️\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 14,\n \"id\": \"6daefdf4-0fab-4031-bc53-fa56bb8d6bb7\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from mle_monitor import MLEDashboard\\n\",\n \"\\n\",\n \"dashboard = MLEDashboard(protocol, resource)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 15,\n \"id\": \"c65b012e-d8ef-4ef9-b8a3-8ac5098b06a5\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"╭───────────────────────────────────────────────────────────────────────────────────────────╮\\n\",\n \"│ General Settings 💡 __ Thu Dec 9 15:38:18 2021 ⏰ │\\n\",\n \"│ _ __ │\\n\",\n \"│ • GCS Sync Protocol: ✔ ____ ___ / /__ Author: @RobertTLange 🐦 │\\n\",\n \"│ ____ ___ ____ ____ (_) /_____ │\\n\",\n \"│ _____ │\\n\",\n \"╰───────────────────────────────────────────────────────────────────────────────────────────╯\\n\",\n \"╭─ Local - Util ─╮╭──────────── Experiment Protocol Summary ─────────────╮╭─ Total Number o─╮\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ 🔖 🆔 🗓 P… P… … ▶ ♻ … G… … ││ … ✔ │\\n\",\n \"│ ││ … ││ 1 0 │\\n\",\n \"│ ──────── ││ … ││ … … │\\n\",\n \"│ ││ ✔ ││ 0 0 │\\n\",\n \"│ ││ ────────────────────────────────────────────────── ││ - … │\\n\",\n \"│ ││ ⠸ 1 1… M… T… … L… 5 2 1 ││ - 0 │\\n\",\n \"│ ││ P… ││ │\\n\",\n \"│ ──────── ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ──────── ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ … ││ ││ │\\n\",\n \"│ ──────────── ││ │╰─────────────────╯\\n\",\n \"│ … ││ │╭─ Last Experimen─╮\\n\",\n \"│ … ││ ││ │\\n\",\n \"│ … ││ ││ E-ID 1 │\\n\",\n \"│ … ││ ││ Type hyp… │\\n\",\n \"│ … ││ ││ Dir. log… │\\n\",\n \"│ … ││ ││ Scr… tra… │\\n\",\n \"│ … ││ ││ Con… bas… │\\n\",\n \"│ … ││ ││ Sta… Run… │\\n\",\n \"│ … ││ ││ 🏃 │\\n\",\n \"│ … ││ ││ Res… loc… │\\n\",\n \"│ … ││ ││ │\\n\",\n \"│ … ││ ││ │\\n\",\n \"│ ──────────── ││ ││ │\\n\",\n \"│ … ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ │╰─────────────────╯\\n\",\n \"│ ││ │╭─ Experiment Com─╮\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ Con… 2/5 │\\n\",\n \"│ ││ ││ Tot… 10 │\\n\",\n \"│ ││ ││ Jobs │\\n\",\n \"│ ││ ││ # 2 │\\n\",\n \"│ ││ ││ Bat… │\\n\",\n \"│ ││ ││ Job… 5 │\\n\",\n \"│ ││ ││ Tim… 0:0… │\\n\",\n \"│ ││ ││ Sta… 12/… │\\n\",\n \"│ ││ ││ Time 15:… │\\n\",\n \"│ ││ ││ ~ 12/… │\\n\",\n \"│ ││ ││ Stop 01:… │\\n\",\n \"│ ││ ││ Time │\\n\",\n \"│ ││ ││ ~ 0:1… │\\n\",\n \"│ ││ ││ Dur… │\\n\",\n \"│ ││ ││ ---… ---… │\\n\",\n \"│ ││ ││ ⏳ … … │\\n\",\n \"│ ││ ││ Jobs │\\n\",\n \"│ ││ ││ ✔ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ ││ ││ │\\n\",\n \"│ │╰──────────────────────────────────────────────────────╯╰─────────────────╯\\n\",\n \"│ │╭─ CPU: 0/8T - Memory: ─╮╭─ Protocol Timeline: T─╮╭─ Protocol Timeline: E─╮\\n\",\n \"│ ││ ┌───────────────… ││ ┌───────────────… ││ ┌───────────────… │\\n\",\n \"│ ││ 1.00┤ ••• % CPU Util. ││ 2.00┤ ••• Total ││ 1.00┤ ███ SC │\\n\",\n \"│ ││ │ ││ │ ││ █████████████████│ │\\n\",\n \"│ ││ 0.80┤ ••• % Mem Util. ││ 1.60┤ ││ 0.80┤ ███ MC │\\n\",\n \"│ ││ • │ ││ │ ││ █████████████████│ │\\n\",\n \"│ ││ 0.60┤ ││ 1.20┤ ││ 0.60┤ ███ HS │\\n\",\n \"│ ││ ••• • • • • │ ││ • │ ││ █████████████████│ │\\n\",\n \"│ ││ 0.40┤•••••••••••••••… ││ 0.80┤ ││ 0.40┤ │\\n\",\n \"│ ││ 0.20┤•••••••••••••••… ││ │ ││ █████████████████│ │\\n\",\n \"│ ││ 0.00┤•••••••• ••• ││ 0.40┤ ││ 0.21┤███████████████… │\\n\",\n \"│ ││ •• •• • •│ ││ │ ││ █████████████████│ │\\n\",\n \"│ ││ └┬──────────────… ││ 0.00┤ ││ 0.01┤███████████████… │\\n\",\n \"│ ││ 12/05-20:05:03 ││ │ ││ █████████████████│ │\\n\",\n \"│ ││ 12/09-15:38:18 ││ └───────────────… ││ └────────┬──────… │\\n\",\n \"│ ││ ││ 12/0… ││ … │\\n\",\n \"│ ││ ││ 15:38 ││ │\\n\",\n \"│ ││ ││ ││ │\\n\",\n \"│ ││ ││ ││ │\\n\",\n \"│ ││ ││ ││ │\\n\",\n \"│ ││ ││ ││ │\\n\",\n \"│ ││ ││ ││ │\\n\",\n \"│ ││ ││ ││ │\\n\",\n \"╰────────────────╯╰───────────────────────╯╰───────────────────────╯╰───────────────────────╯\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[37;44m╭───────────────────────────────────────────────────────────────────────────────────────────╮\\u001b[0m\\n\",\n \"\\u001b[37;44m│\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[1;37;44mGeneral Settings 💡\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m __ \\u001b[0m\\u001b[1;37;44mThu Dec 9 15:38:18 2021 ⏰\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m│\\u001b[0m\\n\",\n \"\\u001b[37;44m│\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m_ __ \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m│\\u001b[0m\\n\",\n \"\\u001b[37;44m│\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m• GCS Sync Protocol: \\u001b[0m\\u001b[32;44m✔\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m ____ ___ / /__ \\u001b[0m\\u001b[37;44m Author: \\u001b[0m\\u001b]8;id=914304;https://twitter.com/RobertTLange\\u001b\\\\\\u001b[4;37;44m@RobertTLange\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[37;44m 🐦\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m│\\u001b[0m\\n\",\n \"\\u001b[37;44m│\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m____ ___ ____ ____ (_) /_____ \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m│\\u001b[0m\\n\",\n \"\\u001b[37;44m│\\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m_____ \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m \\u001b[0m\\u001b[37;44m│\\u001b[0m\\n\",\n \"\\u001b[37;44m╰───────────────────────────────────────────────────────────────────────────────────────────╯\\u001b[0m\\n\",\n \"\\u001b[31m╭─\\u001b[0m\\u001b[31m Local - Util \\u001b[0m\\u001b[31m─╮\\u001b[0m\\u001b[94m╭─\\u001b[0m\\u001b[94m─────────── Experiment Protocol Summary ────────────\\u001b[0m\\u001b[94m─╮\\u001b[0m\\u001b[33m╭─\\u001b[0m\\u001b[33m Total Number o\\u001b[0m\\u001b[33m─╮\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m🔖\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m🆔\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m🗓 \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34mP…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34mP…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;33m▶\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;33m♻\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34mG…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[32m✔\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m 1 \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m 0 \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m ──────── \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;34m…\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[1;33m✔\\u001b[0m\\u001b[1;34m \\u001b[0m\\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m 0 \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m 0 \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m ────────────────────────────────────────────────── \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m-\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m \\u001b[35m⠸ \\u001b[0m \\u001b[34m \\u001b[0m 1 \\u001b[34m \\u001b[0m 1… \\u001b[34m \\u001b[0m M… \\u001b[34m \\u001b[0m T… \\u001b[34m \\u001b[0m … \\u001b[34m \\u001b[0m L… \\u001b[34m \\u001b[0m 5 \\u001b[34m \\u001b[0m 2 \\u001b[34m \\u001b[0m 1 \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m - \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m 0 \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m\\u001b[2m \\u001b[0m\\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m P… \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m ──────── \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[34m \\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m ──────── \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m\\u001b[1;31m \\u001b[0m\\u001b[1;31m…\\u001b[0m\\u001b[1;31m \\u001b[0m\\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m ──────────── \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m╰─────────────────╯\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m╭─\\u001b[0m\\u001b[33m Last Experimen\\u001b[0m\\u001b[33m─╮\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mE-ID\\u001b[0m \\u001b[33m \\u001b[0m 1 \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mType\\u001b[0m \\u001b[33m \\u001b[0m hyp… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mDir.\\u001b[0m \\u001b[33m \\u001b[0m log… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mScr…\\u001b[0m \\u001b[33m \\u001b[0m tra… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mCon…\\u001b[0m \\u001b[33m \\u001b[0m bas… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mSta…\\u001b[0m \\u001b[33m \\u001b[0m Run… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m 🏃 \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mRes…\\u001b[0m \\u001b[33m \\u001b[0m loc… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m … \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m ──────────── \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m \\u001b[0m\\u001b[1m \\u001b[0m\\u001b[1m…\\u001b[0m\\u001b[1m \\u001b[0m\\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m \\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m╰─────────────────╯\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m╭─\\u001b[0m\\u001b[33m Experiment Com\\u001b[0m\\u001b[33m─╮\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mCon…\\u001b[0m \\u001b[33m \\u001b[0m 2/5 \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mTot…\\u001b[0m \\u001b[33m \\u001b[0m 10 \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mJobs\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m# \\u001b[0m \\u001b[33m \\u001b[0m 2 \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mBat…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mJob…\\u001b[0m \\u001b[33m \\u001b[0m 5 \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mTim…\\u001b[0m \\u001b[33m \\u001b[0m 0:0… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mSta…\\u001b[0m \\u001b[33m \\u001b[0m 12/… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mTime\\u001b[0m \\u001b[33m \\u001b[0m 15:… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m~ \\u001b[0m \\u001b[33m \\u001b[0m 12/… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mStop\\u001b[0m \\u001b[33m \\u001b[0m 01:… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mTime\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m~ \\u001b[0m \\u001b[33m \\u001b[0m 0:1… \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mDur…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m---…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33m---…\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m ⏳ \\u001b[33m \\u001b[0m \\u001b[35m…\\u001b[0m \\u001b[35m …\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;33mJobs\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[1;32m✔\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m│\\u001b[0m \\u001b[94m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[94m╰──────────────────────────────────────────────────────╯\\u001b[0m\\u001b[33m╰─────────────────╯\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m╭─\\u001b[0m\\u001b[33m CPU: 0/8T - Memory: \\u001b[0m\\u001b[33m─╮\\u001b[0m\\u001b[33m╭─\\u001b[0m\\u001b[33m Protocol Timeline: T\\u001b[0m\\u001b[33m─╮\\u001b[0m\\u001b[33m╭─\\u001b[0m\\u001b[33m Protocol Timeline: E\\u001b[0m\\u001b[33m─╮\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m ┌───────────────…\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m ┌───────────────…\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m ┌───────────────…\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m1.00┤\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•••\\u001b[0m\\u001b[37;40m % CPU Util.\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m2.00┤\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[33;40m•••\\u001b[0m\\u001b[37;40m Total \\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m1.00┤\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[34;40m███\\u001b[0m\\u001b[37;40m SC \\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33;40m█████████████████\\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.80┤\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[33;40m•••\\u001b[0m\\u001b[37;40m % Mem Util.\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m1.60┤ \\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.80┤\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m███\\u001b[0m\\u001b[37;40m MC \\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[31;40m•\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33;40m█████████████████\\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.60┤ \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m1.20┤\\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.60┤\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[33;40m███\\u001b[0m\\u001b[37;40m HS \\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[31;40m•••\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33;40m•\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33;40m█████████████████\\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.40┤\\u001b[0m\\u001b[31;40m•••••••••••••••…\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.80┤\\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.40┤ \\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.20┤\\u001b[0m\\u001b[33;40m•••••••••••••••…\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33;40m█████████████████\\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.00┤\\u001b[0m\\u001b[31;40m••••\\u001b[0m\\u001b[33;40m•\\u001b[0m\\u001b[31;40m•••\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•••\\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.40┤\\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.21┤\\u001b[0m\\u001b[33;40m███████████████…\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[31;40m••\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m••\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•\\u001b[0m\\u001b[40m \\u001b[0m\\u001b[31;40m•\\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33;40m█████████████████\\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m └┬──────────────…\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.00┤\\u001b[0m\\u001b[40m \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m0.01┤\\u001b[0m\\u001b[33;40m███████████████…\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m 12/05-20:05:03 \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33;40m█████████████████\\u001b[0m\\u001b[37;40m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m12/09-15:38:18 \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m └───────────────…\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m └────────┬──────…\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m 12/0…\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m …\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[37;40m15:38 \\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m│\\u001b[0m \\u001b[31m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\u001b[33m│\\u001b[0m \\u001b[33m│\\u001b[0m\\n\",\n \"\\u001b[31m╰────────────────╯\\u001b[0m\\u001b[33m╰───────────────────────╯\\u001b[0m\\u001b[33m╰───────────────────────╯\\u001b[0m\\u001b[33m╰───────────────────────╯\\u001b[0m\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"# Get a static snapshot of the protocol & resource utilisation\\n\",\n \"# Note: This will look a lot nicer in your terminal!\\n\",\n \"dashboard.snapshot()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"a0ca1ea3-9824-47ca-92ee-f62ed3a7a3de\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Run monitoring in while loop - dashboard\\n\",\n \"dashboard.live()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"d63912fe-3c40-455a-921d-4bce4422a896\",\n \"metadata\": {},\n \"source\": [\n \"- Add widget animation!/screenshot\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"b8467015-0f7c-4576-95c6-2c698f5e8680\",\n \"metadata\": {},\n \"source\": [\n \"# Integration with the MLE-Infrastructure Ecosystem 🔺\\n\",\n \"## Running a Hyperparameter Search for Multiple Random Seeds\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 16,\n \"id\": \"277fcec3-11d3-4a13-81d6-bc261d6a8ebf\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"try:\\n\",\n \" from mle_hyperopt import RandomSearch\\n\",\n \" from mle_scheduler import MLEQueue\\n\",\n \" from mle_logging import load_meta_log\\n\",\n \"except:\\n\",\n \" !pip install -q mle-hyperopt mle-scheduler mle-logging\\n\",\n \" !pip install --upgrade rich\\n\",\n \" from mle_hyperopt import RandomSearch\\n\",\n \" from mle_scheduler import MLEQueue\\n\",\n \" from mle_logging import load_meta_log\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"1c58b0ec-af23-45a4-95d9-1e54a1040476\",\n \"metadata\": {},\n \"source\": [\n \"We again start by adding an experiment to the protocol at launch time.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 17,\n \"id\": \"cffe7cdd-de5e-4809-9206-a9e87b76f916\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"[15:38:26] INFO Added experiment 2 to protocol. mle_protocol.py:162\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m[15:38:26]\\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Added experiment \\u001b[1;36m2\\u001b[0m to protocol. \\u001b]8;id=165623;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=63651;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#162\\u001b\\\\\\u001b[2m162\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \" INFO Locally stored protocol: mle_protocol.db mle_protocol.py:91\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m \\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Locally stored protocol: mle_protocol.db \\u001b]8;id=144531;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=493805;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#91\\u001b\\\\\\u001b[2m91\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"# Load (existing) protocol database and add experiment data\\n\",\n \"protocol_db = MLEProtocol(\\\"mle_protocol.db\\\", verbose=True)\\n\",\n \"meta_data = {\\n\",\n \" \\\"purpose\\\": \\\"random search\\\", # Purpose of experiment\\n\",\n \" \\\"project_name\\\": \\\"surrogate\\\", # Project name of experiment\\n\",\n \" \\\"exec_resource\\\": \\\"local\\\", # Resource jobs are run on\\n\",\n \" \\\"experiment_dir\\\": \\\"logs_search\\\", # Experiment log storage directory\\n\",\n \" \\\"experiment_type\\\": \\\"hyperparameter-search\\\", # Type of experiment to run\\n\",\n \" \\\"base_fname\\\": \\\"train.py\\\", # Main code script to execute\\n\",\n \" \\\"config_fname\\\": \\\"base_config.json\\\", # Config file path of experiment\\n\",\n \" \\\"num_seeds\\\": 2, # Number of evaluations seeds\\n\",\n \" \\\"num_total_jobs\\\": 4, # Number of total jobs to run\\n\",\n \" \\\"num_jobs_per_batch\\\": 4, # Number of jobs in single batch\\n\",\n \" \\\"num_job_batches\\\": 1, # Number of sequential job batches\\n\",\n \" \\\"time_per_job\\\": \\\"00:00:02\\\", # Expected duration: days-hours-minutes\\n\",\n \"}\\n\",\n \"new_experiment_id = protocol_db.add(meta_data)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"8ace6804-6f2b-457a-9f6d-4dc78ed912e0\",\n \"metadata\": {},\n \"source\": [\n \"Afterwards, we leverage `mle-hyperopt` to instantiate a random search strategy with its parameter space. We then ask for two configurations and store them as `.yaml` files in our working directory:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 18,\n \"id\": \"7cd0cf2e-dce9-4e5f-8270-67ffa56809df\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \" MLE-Hyperopt Random Search Hyperspace 🚀 \\n\",\n \" \\n\",\n \" 🌻 Variable Type Search Range ↔ \\n\",\n \" ────────────────────────────────────────────────────────────────────── \\n\",\n \" arch categorical ['mlp', 'cnn'] \\n\",\n \" lrate real Begin: 0.1, End: 0.5, Prior: log-uniform \\n\",\n \" batch_size integer Begin: 1, End: 5, Prior: uniform \\n\",\n \" \\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \" \\u001b[3m MLE-Hyperopt Random Search Hyperspace 🚀 \\u001b[0m \\n\",\n \" \\n\",\n \" \\u001b[1m \\u001b[0m\\u001b[1m🌻 Variable\\u001b[0m\\u001b[1m \\u001b[0m \\u001b[1m \\u001b[0m\\u001b[1mType \\u001b[0m\\u001b[1m \\u001b[0m \\u001b[1;31m \\u001b[0m\\u001b[1;31mSearch Range ↔ \\u001b[0m\\u001b[1;31m \\u001b[0m \\n\",\n \" ────────────────────────────────────────────────────────────────────── \\n\",\n \" arch categorical \\u001b[31m \\u001b[0m\\u001b[31m['mlp', 'cnn'] \\u001b[0m\\u001b[31m \\u001b[0m \\n\",\n \" lrate real \\u001b[31m \\u001b[0m\\u001b[31mBegin: 0.1, End: 0.5, Prior: log-uniform\\u001b[0m\\u001b[31m \\u001b[0m \\n\",\n \" batch_size integer \\u001b[31m \\u001b[0m\\u001b[31mBegin: 1, End: 5, Prior: uniform \\u001b[0m\\u001b[31m \\u001b[0m \\n\",\n \" \\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/plain\": [\n \"[{'arch': 'mlp', 'lrate': 0.360379148648584, 'batch_size': 3},\\n\",\n \" {'arch': 'cnn', 'lrate': 0.26208630215377515, 'batch_size': 2}]\"\n ]\n },\n \"execution_count\": 18,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"# Instantiate random search class\\n\",\n \"strategy = RandomSearch(\\n\",\n \" real={\\\"lrate\\\": {\\\"begin\\\": 0.1, \\\"end\\\": 0.5, \\\"prior\\\": \\\"log-uniform\\\"}},\\n\",\n \" integer={\\\"batch_size\\\": {\\\"begin\\\": 1, \\\"end\\\": 5, \\\"prior\\\": \\\"uniform\\\"}},\\n\",\n \" categorical={\\\"arch\\\": [\\\"mlp\\\", \\\"cnn\\\"]},\\n\",\n \" verbose=True,\\n\",\n \")\\n\",\n \"\\n\",\n \"# Ask for configurations to evaluate & run parallel eval of seeds * configs\\n\",\n \"configs, config_fnames = strategy.ask(2, store=True)\\n\",\n \"configs\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"672d70dd-ee77-40c7-a720-e0015c78e5ed\",\n \"metadata\": {},\n \"source\": [\n \"Next, we can use a `MLEQueue` from `mle-scheduler` to run our training script `train.py` for our two configurations and two different random seeds. `train.py` implements a simple surrogate training loop, which logs some statistics with the help of `mle-logging`. Afterwards, we merge the resulting logs into a single `meta_log.hdf5` and retrieve the mean (over seeds) test loss score for both configurations.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 19,\n \"id\": \"6b700165-75b7-4b7c-a83c-1338c1645c43\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"d9861ce83a384229baf7afaf9b94ee79\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"Output()\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \"[15:38:35] INFO Locally stored protocol: mle_protocol.db mle_protocol.py:91\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m[15:38:35]\\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Locally stored protocol: mle_protocol.db \\u001b]8;id=974722;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=774122;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#91\\u001b\\\\\\u001b[2m91\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \" INFO Locally stored protocol: mle_protocol.db mle_protocol.py:91\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m \\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Locally stored protocol: mle_protocol.db \\u001b]8;id=575975;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=875911;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#91\\u001b\\\\\\u001b[2m91\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \" INFO Locally stored protocol: mle_protocol.db mle_protocol.py:91\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m \\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Locally stored protocol: mle_protocol.db \\u001b]8;id=189310;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=640619;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#91\\u001b\\\\\\u001b[2m91\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \"[15:38:36] INFO Locally stored protocol: mle_protocol.db mle_protocol.py:91\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m[15:38:36]\\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Locally stored protocol: mle_protocol.db \\u001b]8;id=706452;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=688773;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#91\\u001b\\\\\\u001b[2m91\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \" MLEQueue - local • 4/4 Jobs ━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% • 0:00:01 ⌛\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \" \\u001b[1;34mMLEQueue - local •\\u001b[0m 4/4 Jobs \\u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━\\u001b[0m \\u001b[35m100% •\\u001b[0m \\u001b[33m0:00:01\\u001b[0m ⌛\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \"\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"queue = MLEQueue(\\n\",\n \" resource_to_run=\\\"local\\\",\\n\",\n \" job_filename=\\\"train.py\\\",\\n\",\n \" config_filenames=config_fnames,\\n\",\n \" random_seeds=[1, 2],\\n\",\n \" experiment_dir=\\\"logs_search\\\",\\n\",\n \" protocol_db=protocol_db,\\n\",\n \")\\n\",\n \"queue.run()\\n\",\n \"\\n\",\n \"# Merge logs of random seeds & configs -> load & get final scores\\n\",\n \"queue.merge_configs(merge_seeds=True)\\n\",\n \"meta_log = load_meta_log(\\\"logs_search/meta_log.hdf5\\\")\\n\",\n \"test_scores = [meta_log[r].stats.test_loss.mean[-1] for r in queue.mle_run_ids]\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"7d1f1508-d527-43f2-98ba-eff277328684\",\n \"metadata\": {},\n \"source\": [\n \"Finally, we update the random search strategy and tell the protocol that the experiment has been completed:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 20,\n \"id\": \"fa05ce77-20e4-473e-a7ff-162b0b1b9e19\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"┏━━━━━━━━━━━━━━━┳━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\\n\",\n \"┃ 📥 Total: 2 ┃ ID ┃ Obj. 📉 ┃ Configuration 🔖 - 12/09/2021 15:38:43 ┃\\n\",\n \"┡━━━━━━━━━━━━━━━╇━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\\n\",\n \"│ Best Overall │ 0 │ 1.193 │ 'arch': 'mlp', 'lrate': 0.360379148648584, │\\n\",\n \"│ │ │ │ 'batch_size': 3 │\\n\",\n \"│ Best in Batch │ 0 │ 1.193 │ 'arch': 'mlp', 'lrate': 0.360379148648584, │\\n\",\n \"│ │ │ │ 'batch_size': 3 │\\n\",\n \"└───────────────┴────┴─────────┴───────────────────────────────────────────────┘\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"┏━━━━━━━━━━━━━━━┳━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\\n\",\n \"┃\\u001b[1m \\u001b[0m\\u001b[1m📥 Total: 2 \\u001b[0m\\u001b[1m \\u001b[0m┃\\u001b[1m \\u001b[0m\\u001b[1mID\\u001b[0m\\u001b[1m \\u001b[0m┃\\u001b[1m \\u001b[0m\\u001b[1mObj. 📉\\u001b[0m\\u001b[1m \\u001b[0m┃\\u001b[1m \\u001b[0m\\u001b[1mConfiguration 🔖 - 12/09/2021 15:38:43 \\u001b[0m\\u001b[1m \\u001b[0m┃\\n\",\n \"┡━━━━━━━━━━━━━━━╇━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\\n\",\n \"│\\u001b[2m \\u001b[0m\\u001b[2mBest Overall \\u001b[0m\\u001b[2m \\u001b[0m│ 0 │ 1.193 │ 'arch': 'mlp', 'lrate': 0.360379148648584, │\\n\",\n \"│ │ │ │ 'batch_size': 3 │\\n\",\n \"│\\u001b[2m \\u001b[0m\\u001b[2mBest in Batch\\u001b[0m\\u001b[2m \\u001b[0m│ 0 │ 1.193 │ 'arch': 'mlp', 'lrate': 0.360379148648584, │\\n\",\n \"│ │ │ │ 'batch_size': 3 │\\n\",\n \"└───────────────┴────┴─────────┴───────────────────────────────────────────────┘\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \"[15:38:43] INFO Locally stored protocol: mle_protocol.db mle_protocol.py:91\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m[15:38:43]\\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Locally stored protocol: mle_protocol.db \\u001b]8;id=388358;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=128964;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#91\\u001b\\\\\\u001b[2m91\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"data\": {\n \"text/html\": [\n \" INFO Updated protocol - COMPLETED: 2 mle_protocol.py:253\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[2;36m \\u001b[0m\\u001b[2;36m \\u001b[0m\\u001b[34mINFO \\u001b[0m Updated protocol - COMPLETED: \\u001b[1;36m2\\u001b[0m \\u001b]8;id=549681;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py\\u001b\\\\\\u001b[2mmle_protocol.py\\u001b[0m\\u001b]8;;\\u001b\\\\\\u001b[2m:\\u001b[0m\\u001b]8;id=588184;file:///Users/rob/Dropbox/core-code/mle-infrastructure/mle-monitor/mle_monitor/mle_protocol.py#253\\u001b\\\\\\u001b[2m253\\u001b[0m\\u001b]8;;\\u001b\\\\\\n\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"# Update the hyperparameter search strategy\\n\",\n \"strategy.tell(configs, test_scores)\\n\",\n \"\\n\",\n \"# Wrap up experiment (store completion time, etc.)\\n\",\n \"protocol_db.complete(new_experiment_id)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"80f0479f-ca08-41d8-a696-2517a7bc7a23\",\n \"metadata\": {},\n \"source\": [\n \"Give it a try and let me know what you think! If you find a bug or are missing your favourite feature, feel free to contact me [@RobertTLange](https://twitter.com/RobertTLange) or create an issue!\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"mle-toolbox\",\n \"language\": \"python\",\n \"name\": \"mle-toolbox\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.7\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"
},
{
"path": "examples/protocol_gcs.py",
"content": "# Import of helpers for protocoling experiments\nimport os\nfrom mle_monitor import MLEProtocol\n\n\ndef run_protocol_gcs():\n \"\"\"Protocol an experiment and sync it + results in GCS bucket.\"\"\"\n # Define GCS settings - requires 'GOOGLE_APPLICATION_CREDENTIALS' env var.\n cloud_settings = {\n \"project_name\": \"mle-toolbox\", # GCP project name\n \"bucket_name\": \"mle-protocol\", # GCS bucket name\n \"use_protocol_sync\": True, # Whether to sync the protocol to GCS\n \"use_results_storage\": True, # Whether to sync experiment_dir to GCS\n }\n protocol_db = MLEProtocol(\"mle_protocol.db\", cloud_settings, verbose=True)\n\n # Draft data to store in protocol\n meta_data = {\n \"purpose\": \"Test 123\", # Purpose of experiment\n \"project_name\": \"MNIST\", # Project name of experiment\n \"exec_resource\": \"local\", # Resource jobs are run on\n \"experiment_dir\": \"log_dir\", # Experiment log storage directory\n \"experiment_type\": \"hyperparameter-search\", # Type of experiment to run\n \"config_fname\": \"base_config.json\", # Config file path of experiment\n }\n new_experiment_id = protocol_db.add(meta_data)\n\n # ... train your network - or create a placeholder directory + file\n os.makedirs(meta_data[\"experiment_dir\"])\n with open(os.path.join(meta_data[\"experiment_dir\"], \"readme.txt\"), \"w\") as f:\n f.write(\"This is some example file for the mle-monitor example.\")\n\n # Store experiment_dir as .zip in GCS bucket\n protocol_db.complete(new_experiment_id)\n\n\ndef retrieve_results():\n \"\"\"Retrieve the stored experiment dir from GCS bucket and unzip it.\"\"\"\n # Define GCS settings - requires 'GOOGLE_APPLICATION_CREDENTIALS' env var.\n cloud_settings = {\n \"project_name\": \"mle-toolbox\", # GCP project name\n \"bucket_name\": \"mle-protocol\", # GCS bucket name\n \"use_protocol_sync\": True, # Whether to sync the protocol to GCS\n \"use_results_storage\": True, # Whether to sync experiment_dir to GCS\n }\n protocol_db = MLEProtocol(\"mle_protocol.db\", cloud_settings, verbose=True)\n\n # Retrieve last stored experiment from GCS bucket -> retrieved_log_dir\n protocol_db.retrieve(protocol_db.last_experiment_id, \"retrieved_log_dir/\")\n\n\nif __name__ == \"__main__\":\n import argparse\n\n parser = argparse.ArgumentParser(description=\"Let's train a network.\")\n parser.add_argument(\"-retrieve\", \"--retrieve\", action=\"store_true\", default=False)\n args = vars(parser.parse_args())\n if args[\"retrieve\"]:\n retrieve_results()\n else:\n run_protocol_gcs()\n"
},
{
"path": "examples/protocol_local.py",
"content": "# Import of helpers for protocoling experiments\nimport time\nfrom mle_monitor import MLEProtocol, MLEResource, MLEDashboard\n\n\ndef run_protocol_local():\n \"\"\"Protocol an experiment and update the progress status.\"\"\"\n # Load (existing) protocol database and print summary\n protocol_db = MLEProtocol(\"mle_protocol.db\", verbose=False)\n protocol_db.summary(tail=10, verbose=True)\n\n # Ask whether to abort/delete experiment data - only if previously used!\n if len(protocol_db) > 0:\n protocol_db.ask_for_e_id(action=\"delete\")\n protocol_db.ask_for_e_id(action=\"abort\")\n\n # Ask for command line input -> purpose of the experiment\n purpose = protocol_db.ask_for_purpose()\n\n # Draft data to store in protocol & add it to the protocol\n meta_data = {\n \"purpose\": purpose, # Purpose of experiment\n \"project_name\": \"MNIST\", # Project name of experiment\n \"exec_resource\": \"local\", # Resource jobs are run on\n \"experiment_dir\": \"log_dir\", # Experiment log storage directory\n \"experiment_type\": \"hyperparameter-search\", # Type of experiment to run\n \"base_fname\": \"main.py\", # Main code script to execute\n \"config_fname\": \"base_config.json\", # Config file path of experiment\n \"num_seeds\": 5, # Number of evaluations seeds\n \"num_total_jobs\": 10, # Number of total jobs to run\n \"num_jobs_per_batch\": 5, # Number of jobs in single batch\n \"num_job_batches\": 2, # Number of sequential job batches\n \"time_per_job\": \"00:05:00\", # Expected duration: days-hours-minutes\n \"num_cpus\": 2, # Number of CPUs used in job\n \"num_gpus\": 1, # Number of GPUs used in job\n }\n new_experiment_id = protocol_db.add(meta_data)\n\n # ... train your 10 (pseudo) networks/complete respective jobs\n for i in range(10):\n protocol_db.update_progress_bar(new_experiment_id)\n time.sleep(3)\n\n # Wrap up experiment (store completion time, etc.)\n protocol_db.complete(new_experiment_id)\n\n\ndef launch_dashboard():\n \"\"\"Launch a dashboard displaying protocol summary and resource status.\"\"\"\n # Load the protocol & define a resource monitor instance (on local machine)\n protocol = MLEProtocol(\"mle_protocol.db\")\n resource = MLEResource(resource_name=\"local\", monitor_config={})\n # You can also monitor slurm or grid engine clusters\n # resource = MLEResource(\n # resource_name=\"slurm-cluster\",\n # monitor_config={\"partitions\": [\"partition-1\", \"partition-2\"]},\n # )\n # resource = MLEResource(\n # resource_name=\"sge-cluster\",\n # monitor_config={\"queues\": [\"queue-1\", \"queue-2\"]}\n # )\n dashboard = MLEDashboard(protocol, resource)\n\n # Run the dashboard in a while loop\n dashboard.live()\n\n\nif __name__ == \"__main__\":\n import argparse\n\n parser = argparse.ArgumentParser(description=\"Let's train a network.\")\n parser.add_argument(\"-dash\", \"--dashboard\", action=\"store_true\", default=False)\n args = vars(parser.parse_args())\n if args[\"dashboard\"]:\n launch_dashboard()\n else:\n run_protocol_local()\n"
},
{
"path": "examples/run_infrastructure.py",
"content": "from mle_hyperopt import RandomSearch\nfrom mle_scheduler import MLEQueue\nfrom mle_monitor import MLEProtocol\nfrom mle_logging import load_meta_log\n\n\ndef main():\n \"\"\"Full mle-infrastructure example: logging, search, queue, protocol.\"\"\"\n # Load (existing) protocol database and add experiment data\n protocol_db = MLEProtocol(\"mle_protocol.db\")\n meta_data = {\n \"purpose\": \"random search\", # Purpose of experiment\n \"project_name\": \"surrogate\", # Project name of experiment\n \"exec_resource\": \"local\", # Resource jobs are run on\n \"experiment_dir\": \"logs_search\", # Experiment log storage directory\n \"experiment_type\": \"hyperparameter-search\", # Type of experiment to run\n \"base_fname\": \"train.py\", # Main code script to execute\n \"config_fname\": \"base_config.json\", # Config file path of experiment\n \"num_seeds\": 2, # Number of evaluations seeds\n \"num_total_jobs\": 4, # Number of total jobs to run\n \"num_jobs_per_batch\": 4, # Number of jobs in single batch\n \"num_job_batches\": 1, # Number of sequential job batches\n \"time_per_job\": \"00:00:02\", # Expected duration: days-hours-minutes\n }\n new_experiment_id = protocol_db.add(meta_data)\n\n # Instantiate random search class\n strategy = RandomSearch(\n real={\"lrate\": {\"begin\": 0.1, \"end\": 0.5, \"prior\": \"log-uniform\"}},\n integer={\"batch_size\": {\"begin\": 1, \"end\": 5, \"prior\": \"uniform\"}},\n categorical={\"arch\": [\"mlp\", \"cnn\"]},\n verbose=True,\n )\n\n # Ask for configurations to evaluate & run parallel eval of seeds * configs\n configs, config_fnames = strategy.ask(2, store=True)\n queue = MLEQueue(\n resource_to_run=\"local\",\n job_filename=\"train.py\",\n config_filenames=config_fnames,\n random_seeds=[1, 2],\n experiment_dir=\"logs_search\",\n protocol_db=protocol_db,\n )\n queue.run()\n\n # Merge logs of random seeds & configs -> load & get final scores\n queue.merge_configs(merge_seeds=True)\n meta_log = load_meta_log(\"logs_search/meta_log.hdf5\")\n test_scores = [meta_log[r].stats.test_loss.mean[-1] for r in queue.mle_run_ids]\n\n # Update the hyperparameter search strategy\n strategy.tell(configs, test_scores)\n\n # Wrap up experiment (store completion time, etc.)\n protocol_db.complete(new_experiment_id)\n\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "examples/train.py",
"content": "import argparse\nimport random\nfrom mle_logging import MLELogger\nfrom mle_logging.utils import load_yaml_config\n\n\ndef train_your_net(epoch: int, seed_id: int, lrate: float, batch_size: int, arch: str):\n \"\"\"Optimum: lrate=0.2, batch_size=4, arch='conv'.\"\"\"\n f1 = (lrate - 0.2) ** 2 + (batch_size - 4) ** 2 + (0 if arch == \"conv\" else 10)\n train_loss = f1 + seed_id * 0.5\n test_loss = f1 + seed_id * 0.5 + random.uniform(0, 0.3)\n return train_loss / epoch, test_loss / epoch\n\n\ndef main(experiment_dir: str, config_fname: str, seed_id: int):\n \"\"\"Example training 'loop' using MLE-Logging.\"\"\"\n train_config = load_yaml_config(config_fname)\n log = MLELogger(\n experiment_dir=experiment_dir,\n config_fname=config_fname,\n seed_id=seed_id,\n time_to_track=[\"num_epochs\"],\n what_to_track=[\"train_loss\", \"test_loss\"],\n )\n for epoch in range(1, 11):\n train_loss, test_loss = train_your_net(epoch, seed_id, **train_config)\n log.update(\n {\"num_epochs\": epoch},\n {\"train_loss\": train_loss, \"test_loss\": test_loss},\n save=True,\n )\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(description=\"Let's train a network.\")\n parser.add_argument(\n \"-exp_dir\", \"--experiment_dir\", type=str, default=\"experiments/\"\n )\n parser.add_argument(\n \"-config\", \"--config_fname\", type=str, default=\"base_config_1.yaml\"\n )\n parser.add_argument(\"-seed\", \"--seed_id\", type=int, default=1)\n args = vars(parser.parse_args())\n main(args[\"experiment_dir\"], args[\"config_fname\"], args[\"seed_id\"])\n"
},
{
"path": "mle_monitor/__init__.py",
"content": "from ._version import __version__\nfrom .mle_protocol import MLEProtocol\nfrom .mle_resource import MLEResource\nfrom .mle_dashboard import MLEDashboard\n\n\n__all__ = [\n \"__version__\",\n \"MLEProtocol\",\n \"MLEResource\",\n \"MLEDashboard\",\n]\n"
},
{
"path": "mle_monitor/_version.py",
"content": "__version__ = \"0.0.2\"\n"
},
{
"path": "mle_monitor/dashboard/__init__.py",
"content": "from .layout import layout_dashboard\nfrom .update import update_dashboard\n\n\n__all__ = [\"layout_dashboard\", \"update_dashboard\"]\n"
},
{
"path": "mle_monitor/dashboard/components/__init__.py",
"content": "from .plots import (\n make_util_plot,\n make_protocol_total_plot,\n make_protocol_daily_plot,\n)\nfrom .protocol import (\n make_protocol,\n make_total_experiments,\n make_last_experiment,\n make_est_completion,\n)\nfrom .usage import (\n make_user_jobs_cluster,\n make_node_jobs_cluster,\n make_device_panel_local,\n make_process_panel_local,\n)\n\n__all__ = [\n \"make_util_plot\",\n \"make_protocol_total_plot\",\n \"make_protocol_daily_plot\",\n \"make_protocol\",\n \"make_total_experiments\",\n \"make_last_experiment\",\n \"make_est_completion\",\n \"make_user_jobs_cluster\",\n \"make_node_jobs_cluster\",\n \"make_device_panel_local\",\n \"make_process_panel_local\",\n]\n"
},
{
"path": "mle_monitor/dashboard/components/plots.py",
"content": "import numpy as np\nimport plotext as plt\nfrom rich.ansi import AnsiDecoder\nfrom rich.align import Align\nfrom rich.table import Table\n\n\ndef make_util_plot(util_hist) -> Align:\n \"\"\"Plot curve displaying a CPU usage times series for the cluster.\"\"\"\n x = np.arange(len(util_hist[\"rel_cpu_util\"])).tolist()\n y = np.array(util_hist[\"rel_cpu_util\"]).tolist()\n\n x_mem = np.arange(len(util_hist[\"rel_mem_util\"])).tolist()\n y_mem = np.array(util_hist[\"rel_mem_util\"]).tolist()\n # Clear the plot and draw the utilisation lines\n plt.clear_plot()\n plt.plot(x, y, marker=\"dot\", color=\"red\", label=\"% CPU Util.\")\n plt.plot(x_mem, y_mem, marker=\"dot\", color=\"yellow\", label=\"% Mem Util.\")\n plt.figure.plot_size(42, 9)\n plt.canvas_color(\"black\")\n plt.axes_color(\"black\")\n plt.ticks_color(\"white\")\n plt.ylim(0, 1)\n\n # Get time points start and end of monitored period\n xticks = [0, len(util_hist[\"times_hour\"]) - 1]\n xlabels = [\n util_hist[\"times_date\"][i][:5] + \"-\" + util_hist[\"times_hour\"][i]\n for i in xticks\n ]\n plt.xticks(xticks, xlabels)\n plot_str = plotext_helper()\n decoder = AnsiDecoder()\n lines = list(decoder.decode(plot_str))\n\n # Build the rich table graph\n message = Table.grid()\n message.add_column()\n for line in lines:\n message.add_row(line)\n return Align.center(message)\n\n\ndef make_protocol_total_plot(experiment_hist) -> Align:\n \"\"\"Plot curve displaying a memory usage times series for the cluster.\"\"\"\n plt.clear_plot()\n plt.datetime.set_datetime_form(date_form=\"%m/%d/%y %H:%M\")\n\n plt.canvas_color(\"black\")\n plt.axes_color(\"black\")\n plt.ticks_color(\"white\")\n plt.plot_date(\n experiment_hist[\"time\"],\n experiment_hist[\"total_exp\"][\"all\"],\n marker=\"dot\",\n color=\"yellow\",\n label=\"Total\",\n )\n plt.figure.plot_size(42, 9)\n\n plot_str = plotext_helper()\n decoder = AnsiDecoder()\n lines = list(decoder.decode(plot_str))\n\n # Build the rich table graph\n message = Table.grid()\n message.add_column()\n for line in lines:\n message.add_row(line)\n return Align.center(message)\n\n\ndef make_protocol_daily_plot(experiment_hist) -> Align:\n \"\"\"Plot curve displaying a memory usage times series for the cluster.\"\"\"\n plt.clear_plot()\n try:\n day_short = [e[:5] for e in experiment_hist[\"day\"]]\n # Add empty string for label legend visability\n plt.stacked_bar(\n [\"\"] + day_short,\n [\n [0.1] + experiment_hist[\"day_exp\"][\"hyperparameter-search\"],\n [0] + experiment_hist[\"day_exp\"][\"multiple-configs\"],\n [0] + experiment_hist[\"day_exp\"][\"single-config\"],\n ],\n label=[\"HS\", \"MC\", \"SC\"],\n marker=\"sd\",\n color=[\"yellow\", \"red\", \"blue\"],\n )\n except Exception:\n pass\n plt.figure.plot_size(42, 9)\n plt.canvas_color(\"black\")\n plt.axes_color(\"black\")\n plt.ticks_color(\"white\")\n\n plot_str = plotext_helper()\n decoder = AnsiDecoder()\n lines = list(decoder.decode(plot_str))\n\n # Build the rich table graph\n message = Table.grid()\n message.add_column()\n for line in lines:\n message.add_row(line)\n return Align.center(message)\n\n\ndef plotext_helper():\n \"\"\"Helper fct. that generates ansi string to plot.\"\"\"\n plt.figure.build()\n return plt.figure.canvas\n"
},
{
"path": "mle_monitor/dashboard/components/protocol.py",
"content": "from rich import box\nfrom rich.align import Align\nfrom rich.table import Table\nfrom rich.text import Text\nfrom rich.progress import (\n BarColumn,\n Progress,\n TextColumn,\n)\n\n\ndef make_protocol(protocol_table) -> Table:\n \"\"\"Generate rich table summarizing experiment protocol summary.\"\"\"\n return Align.center(protocol_table)\n\n\ndef make_total_experiments(total_data) -> Align:\n \"\"\"Generate rich table summarizing all experiments in protocol db.\"\"\"\n table = Table(\n show_header=False,\n show_footer=False,\n header_style=\"bold yellow\",\n border_style=\"yellow\",\n box=box.SIMPLE_HEAD,\n )\n table.add_column()\n table.add_column()\n table.add_column()\n table.add_column()\n table.add_row(\n \"[b yellow]Total\",\n \":running:\",\n \"[green]:heavy_check_mark:\",\n \"[red]:heavy_multiplication_x:\",\n )\n table.add_row(\n total_data[\"total\"],\n total_data[\"run\"],\n total_data[\"done\"],\n total_data[\"aborted\"],\n )\n table.add_row(\n Text.from_markup(\"[b yellow]SGE\"),\n Text.from_markup(\"[b yellow]Slurm\"),\n Text.from_markup(\"[b yellow]GCP\"),\n Text.from_markup(\"[b yellow]Local\"),\n )\n table.add_row(\n total_data[\"sge\"], total_data[\"slurm\"], total_data[\"gcp\"], total_data[\"local\"]\n )\n table.add_row(\n Text.from_markup(\"[b yellow]-\"),\n Text.from_markup(\"[b yellow]Report\"),\n Text.from_markup(\"[b yellow]GCS\"),\n Text.from_markup(\"[b yellow]Sync\"),\n )\n table.add_row(\n \"-\", total_data[\"report_gen\"], total_data[\"gcs_stored\"], total_data[\"retrieved\"]\n )\n return Align.center(table)\n\n\ndef make_last_experiment(last_data) -> Align:\n \"\"\"Generate rich table summarizing last scheduled experiment settings.\"\"\"\n table = Table(\n show_header=False,\n show_footer=False,\n box=box.SIMPLE_HEAD,\n header_style=\"bold yellow\",\n border_style=\"yellow\",\n )\n table.add_column()\n table.add_column()\n table.add_row(\n Text.from_markup(\"[b yellow]E-ID\"),\n last_data[\"e_id\"],\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Type\"),\n last_data[\"e_type\"],\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Dir.\"),\n last_data[\"e_dir\"],\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Script\"),\n last_data[\"e_script\"],\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Config\"),\n last_data[\"e_config\"],\n )\n\n if last_data[\"job_status\"] == \"running\":\n table.add_row(\n Text.from_markup(\"[b yellow]Status\"),\n \"Running :running:\",\n )\n elif last_data[\"job_status\"] == \"completed\":\n table.add_row(\n Text.from_markup(\"[b yellow]Status\"),\n \"Completed [green]:heavy_check_mark:\",\n )\n elif last_data[\"job_status\"] == \"aborted\":\n table.add_row(\n Text.from_markup(\"[b yellow]Status\"),\n \"Aborted [red]:heavy_multiplication_x:\",\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Resource\"),\n last_data[\"resource\"],\n )\n return Align.center(table)\n\n\ndef make_est_completion(time_data) -> Align:\n \"\"\"Generate rich table summarizing estim. time of experim. completion.\"\"\"\n table = Table(\n show_header=False,\n show_footer=False,\n box=box.SIMPLE_HEAD,\n header_style=\"bold yellow\",\n border_style=\"yellow\",\n )\n table.add_column()\n table.add_column()\n table.add_row(\n Text.from_markup(\"[b yellow]Conf/Seeds\"),\n f\"{int(time_data['total_jobs']/time_data['num_seeds'])}/{time_data['num_seeds']}\",\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Total Jobs\"), str(time_data[\"total_jobs\"])\n )\n table.add_row(\n Text.from_markup(\"[b yellow]# Batches\"), str(time_data[\"total_batches\"])\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Jobs/Batch\"), str(time_data[\"jobs_per_batch\"])\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Time/Batch\"),\n str(time_data[\"time_per_batch\"][1:]),\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Start Time\"), str(time_data[\"start_time\"])\n )\n if time_data[\"job_status\"] == \"completed\":\n table.add_row(\n Text.from_markup(\"[b yellow]Stop Time\"), str(time_data[\"stop_time\"])\n )\n table.add_row(\n Text.from_markup(\"[b yellow]Duration\"),\n str(time_data[\"duration\"]),\n )\n else:\n table.add_row(\n Text.from_markup(\"[b yellow]~ Stop Time\"), str(time_data[\"stop_time\"])\n )\n table.add_row(\n Text.from_markup(\"[b yellow]~ Duration\"),\n str(time_data[\"duration\"]),\n )\n progress = Progress(\n TextColumn(\"{task.completed}/{task.total}\", justify=\"left\", style=\"magenta\"),\n BarColumn(bar_width=10, style=\"magenta\"),\n TextColumn(\"[progress.percentage]{task.percentage:>3.0f}%\"),\n auto_refresh=False,\n )\n\n task = progress.add_task(\"queue\", total=time_data[\"total_jobs\"])\n progress.update(task, completed=time_data[\"completed_jobs\"], refresh=True)\n table.add_row(\n \"[b yellow]-----------\",\n \"[b yellow]-----------------\",\n )\n table.add_row(\n Text.from_markup(\n \":hourglass_flowing_sand: [b yellow]Jobs [green]:heavy_check_mark:\"\n ),\n progress,\n )\n return Align.center(table)\n"
},
{
"path": "mle_monitor/dashboard/components/usage.py",
"content": "from rich import box\nfrom rich.align import Align\nfrom rich.table import Table\nfrom rich.text import Text\n\n\ndef make_user_jobs_cluster(user_data: dict) -> Align:\n \"\"\"Generate rich table summarizing jobs scheduled by users.\"\"\"\n all_active_users = min(len(user_data[\"total\"]), 13)\n sum_all = str(sum(user_data[\"total\"]))\n sum_running = str(sum(user_data[\"run\"]))\n # sum_login = str(sum(user_data[\"login\"]))\n sum_wait = str(sum(user_data[\"wait\"]))\n\n # Create table with structure with aggregated numbers\n table = Table(\n show_header=True,\n show_footer=True,\n header_style=\"bold red\",\n row_styles=[\"none\", \"dim\"],\n border_style=\"red\",\n box=box.SIMPLE,\n )\n table.add_column(\n \"USER\",\n Text.from_markup(\"[b]Sum\", justify=\"right\"),\n style=\"white\",\n justify=\"left\",\n )\n table.add_column(\"ALL\", sum_all, justify=\"center\")\n table.add_column(\":running:\", sum_running, justify=\"center\")\n # table.add_column(\":astronaut:\", sum_login, justify=\"center\")\n table.add_column(\":pause_button:\", sum_wait, justify=\"center\")\n\n # Add row for each individual user\n for u_id in range(all_active_users):\n table.add_row(\n user_data[\"user\"][u_id],\n str(user_data[\"total\"][u_id]),\n str(user_data[\"run\"][u_id]),\n # str(user_data[\"login\"][u_id]),\n str(user_data[\"wait\"][u_id]),\n )\n return table\n\n\ndef make_node_jobs_cluster(host_data: dict, col_title: str) -> Align:\n \"\"\"Generate rich table summarizing jobs running on different nodes.\"\"\"\n all_nodes = len(host_data[\"total\"])\n sum_all = str(sum(host_data[\"total\"]))\n sum_running = str(sum(host_data[\"run\"]))\n sum_login = str(sum(host_data[\"login\"]))\n\n table = Table(\n show_header=True,\n show_footer=True,\n header_style=\"bold red\",\n row_styles=[\"none\", \"dim\"],\n border_style=\"red\",\n box=box.SIMPLE,\n )\n table.add_column(\n col_title,\n Text.from_markup(\"[b]Sum\", justify=\"right\"),\n style=\"white\",\n justify=\"left\",\n )\n table.add_column(\"ALL\", sum_all, justify=\"center\")\n table.add_column(\":running:\", sum_running, justify=\"center\")\n table.add_column(\":construction_worker:\", sum_login, justify=\"center\")\n\n # Add row for each individual cluster/queue/partition node\n for h_id in range(all_nodes):\n table.add_row(\n host_data[\"host_id\"][h_id],\n str(host_data[\"total\"][h_id]),\n str(host_data[\"run\"][h_id]),\n str(host_data[\"login\"][h_id]),\n )\n return table\n\n\ndef make_device_panel_local(device_data: dict) -> Align:\n \"\"\"Generate rich table summarizing jobs running on different nodes.\"\"\"\n sum_all = str(round(sum(device_data[\"percent_util\"]), 1))\n t1 = Table(\n show_header=True,\n show_footer=True,\n header_style=\"bold red\",\n row_styles=[\"none\", \"dim\"],\n border_style=\"red\",\n box=box.SIMPLE,\n )\n t1.add_column(\n \":computer:\",\n Text.from_markup(\"[b]Sum\", justify=\"right\"),\n style=\"white\",\n justify=\"left\",\n )\n t1.add_column(\"%\", sum_all, justify=\"center\")\n t1.add_column(\":computer:\", \"---\", justify=\"center\")\n t1.add_column(\"%\", \"---\", justify=\"center\")\n\n # Add row for each individual core\n num_cores = len(device_data[\"core_id\"])\n for i in range(int(num_cores / 2)):\n t1.add_row(\n \"C\" + str(device_data[\"core_id\"][i]),\n str(device_data[\"percent_util\"][i]),\n \"C\" + str(device_data[\"core_id\"][int(i + num_cores / 2 - 1)]),\n str(device_data[\"percent_util\"][int(i + num_cores / 2 - 1)]),\n )\n\n t2 = Table(\n show_header=True,\n show_footer=False,\n box=box.SIMPLE,\n border_style=\"red\",\n header_style=\"bold red\",\n )\n t2.add_column(\"GPU\", style=\"white\", justify=\"left\")\n t2.add_column(\":computer: %\", justify=\"center\")\n t2.add_column(\":floppy_disk: %\", justify=\"center\")\n t2.add_column(\"GB\", justify=\"center\")\n\n for i in range(len(device_data[\"gpu_name\"])):\n t2.add_row(\n str(device_data[\"gpu_name\"][i]),\n str(device_data[\"gpu_load\"][i]),\n str(device_data[\"gpu_mem_util\"][i]),\n str(device_data[\"gpu_mem_total\"][i]),\n )\n\n table = Table(box=box.SIMPLE_HEAD, show_header=False, show_footer=False)\n table.add_column()\n table.add_row(t1)\n table.add_row(t2)\n return Align.center(table)\n\n\ndef make_process_panel_local(proc_data) -> Align:\n \"\"\"Generate rich table summarizing jobs running on different nodes.\"\"\"\n num_procs = len(proc_data[\"pid\"])\n t1 = Table(\n show_header=True,\n show_footer=True,\n header_style=\"bold red\",\n border_style=\"red\",\n box=box.SIMPLE,\n )\n t1.add_column(\"PID\", \"---\", style=\"white\", justify=\"left\")\n t1.add_column(\"NAME\", \"---\", justify=\"center\")\n t1.add_column(\n \":computer: %\", str(round(proc_data[\"total_cpu_util\"], 1)), justify=\"center\"\n )\n t1.add_column(\n \":floppy_disk:\", str(round(proc_data[\"total_mem_util\"], 1)), justify=\"center\"\n )\n\n for i in range(int(num_procs / 2)):\n t1.add_row(\n str(proc_data[\"pid\"][i]),\n str(proc_data[\"p_name\"][i][:6]),\n str(round(proc_data[\"cpu_util\"][i], 1)),\n str(round(proc_data[\"mem_util\"][i], 1)),\n )\n\n for i in range(int(num_procs / 2), num_procs):\n t1.add_row(\n str(proc_data[\"pid\"][i]),\n str(proc_data[\"p_name\"][i][:6]),\n str(round(proc_data[\"cpu_util\"][i], 1)),\n str(round(proc_data[\"mem_util\"][i], 1)),\n )\n return t1\n\n\ndef make_help_commands() -> Align:\n \"\"\"Generate rich table summarizing core toolbox subcommands.\"\"\"\n table = Table(\n show_header=True,\n show_footer=False,\n border_style=\"white\",\n header_style=\"bold magenta\",\n box=box.SIMPLE_HEAD,\n )\n table.add_column(\"Command\", style=\"white\", justify=\"left\")\n table.add_column(\n \"Function\",\n )\n table.add_row(\n \"mle run\",\n \"[b red] Experiment w. config .yaml\",\n )\n table.add_row(\n \"mle retrieve\",\n \"[b red] Retrieve from cluster/GCS\",\n )\n table.add_row(\n \"mle report\",\n \"[b red] Generate results .md report\",\n )\n table.add_row(\n \"mle monitor\",\n \"[b red] Monitor resource usage\",\n )\n table.add_row(\n \"mle init\",\n \"[b red] Init creds/default setup\",\n )\n return table\n\n\ndef make_gcp_util(gcp_data) -> Align:\n \"\"\"Generate rich table summarizing running GCP VM machines.\"\"\"\n table = Table(\n show_header=True,\n show_footer=False,\n border_style=\"white\",\n header_style=\"bold magenta\",\n box=box.SIMPLE_HEAD,\n )\n table.add_column(\"Machine\", style=\"white\", justify=\"left\")\n table.add_column(\"Run\", justify=\"center\")\n table.add_column(\"Stage\", justify=\"center\")\n table.add_column(\"Stop\", justify=\"center\")\n for m_id in range(len(gcp_data[\"experiment_type\"])):\n table.add_row(\n str(gcp_data[\"experiment_type\"][m_id]),\n str(gcp_data[\"run\"][m_id]),\n str(gcp_data[\"stage\"][m_id]),\n str(gcp_data[\"stop\"][m_id]),\n )\n return table\n"
},
{
"path": "mle_monitor/dashboard/layout.py",
"content": "from rich.layout import Layout\nfrom rich.panel import Panel\nfrom rich.table import Table\nimport datetime as dt\n\n\ndef layout_dashboard(resource: str, use_gcs_sync: bool, protocol_fname: str) -> Layout:\n \"\"\"Define the MLE-Toolbox `monitor` base dashboard layout.\"\"\"\n layout = Layout(name=\"root\")\n # Split in three vertical sections: Welcome, core info, help + util plots\n layout.split_column(\n Layout(name=\"header\", size=7),\n Layout(name=\"main\"),\n )\n # Split center into 3 horizontal sections\n layout[\"main\"].split_row(\n Layout(name=\"left\", ratio=2),\n Layout(name=\"right\", ratio=8),\n )\n\n # Split center into 3 horizontal sections\n layout[\"right\"].split_column(\n Layout(name=\"top-right\", ratio=6),\n Layout(name=\"bottom-right\", ratio=2),\n )\n\n # Split center into 3 horizontal sections\n layout[\"top-right\"].split_row(\n Layout(name=\"top-right-left\", ratio=6),\n Layout(name=\"top-right-right\", ratio=2),\n )\n\n # Split center right into total experiments, last experiments, ETA\n layout[\"top-right-right\"].split_column(\n Layout(name=\"top-right-right-1\", ratio=3),\n Layout(name=\"top-right-right-2\", ratio=3),\n Layout(name=\"top-right-right-3\", ratio=4),\n )\n\n # Split bottom into toolbox command info and cluster util termplots\n layout[\"bottom-right\"].split_row(\n Layout(name=\"bottom-right-1\", ratio=1),\n Layout(name=\"bottom-right-2\", ratio=1),\n Layout(name=\"bottom-right-3\", ratio=1),\n )\n # # Fill the header with life!\n layout[\"header\"].update(Header(resource, use_gcs_sync, protocol_fname))\n return layout\n\n\nclass Header:\n \"\"\"Display header with clock and general toolbox configurations.\"\"\"\n\n welcome_ascii = \"\"\" __ _ __\n ____ ___ / /__ ____ ___ ____ ____ (_) /_____ _____\n / __ `__ \\/ / _ \\______/ __ `__ \\/ __ \\/ __ \\/ / __/ __ \\/ ___/\n / / / / / / / __/_____/ / / / / / /_/ / / / / / /_/ /_/ / /\n /_/ /_/ /_/_/\\___/ /_/ /_/ /_/\\____/_/ /_/_/\\__/\\____/_/\n\"\"\".splitlines()\n\n def __init__(self, resource: str, use_gcs_sync: bool, protocol_fname: str):\n self.resource = resource\n self.use_gcs_sync = use_gcs_sync\n self.protocol_fname = protocol_fname\n\n def __rich__(self) -> Panel:\n grid = Table.grid(expand=True)\n grid.add_column(justify=\"left\")\n grid.add_column(justify=\"left\")\n grid.add_column(justify=\"right\")\n grid.add_row(\n \"[bold]General Settings :bulb:\",\n Header.welcome_ascii[0],\n \"[bold]\" + dt.datetime.now().ctime().replace(\" \", \" \") + \" :alarm_clock:\",\n )\n grid.add_row(\n \"\\u2022 GCS Sync Protocol: [green]:heavy_check_mark:\"\n if self.use_gcs_sync\n else \"\\u2022 GCS Sync Protocol: [red]:heavy_multiplication_x:\",\n Header.welcome_ascii[1],\n \"Author: [u link=https://twitter.com/RobertTLange]@RobertTLange[/] :bird:\",\n )\n\n grid.add_row(\n \"\\u2022 GCS Sync Results: [green]:heavy_check_mark:\"\n if self.use_gcs_sync\n else \"\\u2022 GCS Sync Results: [red]:heavy_multiplication_x:\",\n Header.welcome_ascii[2],\n \":point_right: [u link=https://github.com/mle-infrastructure]MLE-Infrastructure[/] :small_red_triangle:\",\n )\n grid.add_row(\n f\"\\u2022 DB Path: {self.protocol_fname}\",\n Header.welcome_ascii[3],\n f\"Resource: {self.resource.resource_name} :computer:\",\n )\n grid.add_row(\n \"[bold]Carpe Diem[/bold] :city_sunrise:\",\n Header.welcome_ascii[4],\n \"[bold]Hi there - You rock! [not italic]:hugging_face:[/]\",\n )\n return Panel(grid, style=\"white on blue\")\n"
},
{
"path": "mle_monitor/dashboard/update.py",
"content": "from rich.panel import Panel\nfrom rich.align import Align\nfrom rich.table import Table\nfrom .components import (\n make_user_jobs_cluster,\n make_node_jobs_cluster,\n make_device_panel_local,\n make_process_panel_local,\n make_protocol,\n make_total_experiments,\n make_last_experiment,\n make_est_completion,\n make_util_plot,\n make_protocol_total_plot,\n make_protocol_daily_plot,\n)\n\n\ndef update_dashboard(layout, resource_data, protocol_data, usage_data):\n \"\"\"Helper function that fills dashboard with life!\"\"\"\n # Fill the left-main with life!\n if resource_data[\"resource_name\"] in [\"sge-cluster\", \"slurm-cluster\"]:\n table_user = make_user_jobs_cluster(resource_data[\"user_data\"])\n queue_var = (\n \"PARTITION\"\n if resource_data[\"resource_name\"] == \"slurm-cluster\"\n else \"QUEUE\"\n )\n table_host = make_node_jobs_cluster(resource_data[\"host_data\"], queue_var)\n table_node = make_node_jobs_cluster(resource_data[\"node_data\"], \"NODE\")\n grid = Table.grid(expand=True)\n grid.add_column()\n grid.add_row(table_user)\n grid.add_row(table_host)\n grid.add_row(table_node)\n layout[\"left\"].update(\n Panel(\n Align.center(grid),\n border_style=\"red\",\n title=\"Jobs by User/Queue/Node\",\n )\n )\n else:\n table_device = make_device_panel_local(resource_data[\"host_data\"])\n table_process = make_process_panel_local(resource_data[\"user_data\"])\n grid = Table.grid()\n grid.add_column()\n grid.add_row(table_device)\n grid.add_row(table_process)\n\n layout[\"left\"].update(\n Panel(\n Align.center(grid),\n border_style=\"red\",\n title=\"Local - Util by Device/Process\",\n )\n )\n\n # Fill the center-main with life!\n layout[\"top-right-left\"].update(\n Panel(\n make_protocol(protocol_data[\"protocol_table\"]),\n border_style=\"bright_blue\",\n title=\"Experiment Protocol Summary\",\n )\n )\n\n # Fill the right-main with life!\n layout[\"top-right-right-1\"].update(\n Panel(\n make_total_experiments(protocol_data[\"total_data\"]),\n border_style=\"yellow\",\n title=\"Total Number of Experiment Runs\",\n )\n )\n layout[\"top-right-right-2\"].update(\n Panel(\n make_last_experiment(protocol_data[\"last_data\"]),\n border_style=\"yellow\",\n title=\"Last Experiment Configuration\",\n )\n )\n layout[\"top-right-right-3\"].update(\n Panel(\n make_est_completion(protocol_data[\"time_data\"]),\n border_style=\"yellow\",\n title=\"Experiment Completion Time\",\n )\n )\n\n # Fill the footer with life!\n layout[\"bottom-right-1\"].update(\n Panel(\n make_util_plot(usage_data),\n title=(\n f\"CPU: {int(resource_data['util_data']['cores_util'])}/\"\n f\"{int(resource_data['util_data']['cores'])}T\"\n f\" - Memory: {int(resource_data['util_data']['mem_util'])}/\"\n f\"{int(resource_data['util_data']['mem'])}G\"\n ),\n border_style=\"yellow\",\n ),\n )\n\n layout[\"bottom-right-2\"].update(\n Panel(\n make_protocol_total_plot(protocol_data[\"summary_data\"]),\n title=(\"Protocol Timeline: Total Experiments\"),\n border_style=\"yellow\",\n ),\n )\n\n layout[\"bottom-right-3\"].update(\n Panel(\n make_protocol_daily_plot(protocol_data[\"summary_data\"]),\n title=(\"Protocol Timeline: Experiments/Day\"),\n border_style=\"yellow\",\n )\n )\n return layout\n"
},
{
"path": "mle_monitor/mle_dashboard.py",
"content": "import time\nfrom rich.live import Live\nfrom rich.console import Console\nfrom . import MLEProtocol, MLEResource\nfrom .utils import Tracker\nfrom .dashboard import layout_dashboard, update_dashboard\n\n\nclass MLEDashboard(object):\n def __init__(self, protocol: MLEProtocol, resource: MLEResource):\n \"\"\"MLE Resource Dashboard - Rich-based terminal output.\"\"\"\n self.protocol = protocol\n self.resource = resource\n self.tracker = Tracker()\n\n def snapshot(self):\n \"\"\"Get single console output snapshot.\"\"\"\n # Create the layout\n layout = layout_dashboard(\n self.resource,\n self.protocol.use_gcs_protocol_sync,\n self.protocol.protocol_fname,\n )\n # Retrieve the data\n resource_data = self.resource.monitor()\n protocol_data = self.protocol.monitor()\n usage_data = self.tracker.update(resource_data[\"util_data\"])\n # Update the layout and print it\n layout = update_dashboard(\n layout, resource_data, protocol_data, usage_data\n )\n Console().print(layout)\n\n def live(self, pull_gcs: bool = False):\n \"\"\"Run constant monitoring in while loop.\"\"\"\n # Generate the dashboard layout and display first data\n layout = layout_dashboard(\n self.resource,\n self.protocol.use_gcs_protocol_sync,\n self.protocol.protocol_fname,\n )\n\n resource_data = self.resource.monitor()\n protocol_data = self.protocol.monitor()\n usage_data = self.tracker.update(resource_data[\"util_data\"])\n layout = update_dashboard(\n layout, resource_data, protocol_data, usage_data\n )\n\n # Start timers for GCS pulling and reloading of local protocol db\n timer_gcs = time.time()\n timer_db = time.time()\n\n # Run the live updating of the dashboard\n with Live(console=Console(), screen=True, auto_refresh=True) as live:\n live.update(layout)\n while True:\n try:\n resource_data = self.resource.monitor()\n protocol_data = self.protocol.monitor()\n usage_data = self.tracker.update(\n resource_data[\"util_data\"],\n protocol_data[\"summary_data\"],\n )\n layout = update_dashboard(\n layout, resource_data, protocol_data, usage_data\n )\n\n # Every 10 seconds reload local database file\n if time.time() - timer_db > 2:\n self.protocol.load(pull_gcs=False)\n timer_db = time.time()\n\n # Every 5 minutes pull the newest DB from GCS\n if pull_gcs:\n if time.time() - timer_gcs > 300:\n self.protocol.load()\n timer_gcs = time.time()\n except Exception:\n pass\n"
},
{
"path": "mle_monitor/mle_protocol.py",
"content": "from typing import Union, List\nfrom datetime import datetime\nimport sys\nimport select\nimport logging\nfrom .protocol import (\n load_protocol_db,\n protocol_summary,\n protocol_experiment,\n protocol_table,\n get_monitor_db_data,\n)\nfrom .utils import setup_logger\n\n\nclass MLEProtocol(object):\n def __init__(\n self,\n protocol_fname: str,\n cloud_settings: Union[dict, None] = None,\n verbose: bool = False,\n ):\n \"\"\"MLE Protocol DB Instance.\"\"\"\n self.protocol_fname = protocol_fname\n self.cloud_settings = cloud_settings\n self.verbose = verbose\n if self.verbose:\n self.logger = setup_logger(logging.INFO)\n else:\n self.logger = setup_logger(logging.WARNING)\n\n # Setup GCS credentials/data\n if self.cloud_settings is not None:\n # Don't use sync if cloud_setting dict is empty DotMap\n if len(self.cloud_settings.keys()) > 0:\n assert self.cloud_settings[\"project_name\"] is not None\n assert self.cloud_settings[\"bucket_name\"] is not None\n self.use_gcs_protocol_sync = self.cloud_settings[\n \"use_protocol_sync\"\n ]\n self.use_gcs_protocol_storage = self.cloud_settings[\n \"use_results_storage\"\n ]\n\n if \"credentials_path\" in self.cloud_settings:\n # Set the path to the GCP credentials json file & pull recent db\n from .protocol import set_gcp_credentials\n\n set_gcp_credentials(cloud_settings[\"credentials_path\"])\n\n if \"protocol_fname\" not in self.cloud_settings:\n self.cloud_settings[\"protocol_fname\"] = self.protocol_fname\n else:\n self.use_gcs_protocol_sync = False\n self.use_gcs_protocol_storage = False\n else:\n self.use_gcs_protocol_sync = False\n self.use_gcs_protocol_storage = False\n self.load()\n\n def load(self, pull_gcs: bool = True):\n \"\"\"Load the protocol data from a local pickle file.\"\"\"\n if self.use_gcs_protocol_sync:\n if pull_gcs:\n self.accessed_gcs = self.gcs_pull()\n else:\n self.accessed_gcs = False\n else:\n self.accessed_gcs = False\n (\n self.db,\n self.experiment_ids,\n self.last_experiment_id,\n ) = load_protocol_db(self.protocol_fname)\n\n def get(\n self,\n experiment_id: Union[str, int, None] = None,\n var_name: Union[str, None] = None,\n ):\n \"\"\"Retrieve variable from database.\"\"\"\n if experiment_id is None:\n experiment_id = str(self.last_experiment_id)\n elif type(experiment_id) == int:\n experiment_id = str(experiment_id)\n\n if var_name is None:\n return self.db.get(experiment_id)\n else:\n return self.db.dget(experiment_id, var_name)\n\n def save(self, send_gcs: bool = True):\n \"\"\"Dump the protocol db to its pickle file.\"\"\"\n self.db.dump()\n if self.verbose:\n self.logger = setup_logger(logging.INFO)\n else:\n self.logger = setup_logger(logging.WARNING)\n self.logger.info(f\"Locally stored protocol: {self.protocol_fname}\")\n\n # Send recent/up-to-date experiment DB to Google Cloud Storage\n if send_gcs and self.use_gcs_protocol_sync:\n if self.accessed_gcs:\n self.gcs_send()\n self.logger.info(f\"GCS synced protocol: {self.protocol_fname}\")\n\n @property\n def standard_keys(self):\n \"\"\"All required keys in standard dict to supply in `add`.\"\"\"\n return [\n \"purpose\",\n \"project_name\",\n \"exec_resource\",\n \"experiment_dir\",\n \"experiment_type\",\n \"base_fname\",\n \"config_fname\",\n \"num_seeds\",\n \"num_total_jobs\",\n \"num_job_batches\",\n \"num_jobs_per_batch\",\n \"time_per_job\",\n \"num_cpus\",\n \"num_gpus\",\n ]\n\n @property\n def standard_default(self):\n \"\"\"All required keys in standard dict to supply in `add`.\"\"\"\n return {\n \"purpose\": \"None provided\",\n \"project_name\": \"default\",\n \"exec_resource\": \"local\",\n \"experiment_dir\": \"experiments\",\n \"experiment_type\": \"single\",\n \"base_fname\": \"main.py\",\n \"config_fname\": \"base_config.yaml\",\n \"num_seeds\": 1,\n \"num_total_jobs\": 1,\n \"num_job_batches\": 1,\n \"num_jobs_per_batch\": 1,\n \"time_per_job\": \"00:01:00\",\n \"num_cpus\": 1,\n \"num_gpus\": 0,\n }\n\n def add(\n self,\n standard: dict,\n extra: Union[dict, None] = None,\n save: bool = True,\n send_gcs: bool = True,\n ):\n \"\"\"Add an experiment to the database.\"\"\"\n for k in self.standard_keys:\n if k not in standard.keys():\n standard[k] = self.standard_default[k]\n assert standard[\"experiment_type\"] in [\n \"hyperparameter-search\",\n \"multiple-configs\",\n \"single-config\",\n ]\n self.db, new_experiment_id = protocol_experiment(\n self.db, self.last_experiment_id, standard, extra\n )\n self.experiment_ids.append(new_experiment_id)\n self.last_experiment_id = new_experiment_id\n self.added_experiment_id = new_experiment_id\n self.completed_jobs_counter = 0\n self.logger.info(f\"Added experiment {new_experiment_id} to protocol.\")\n if save:\n self.save(send_gcs)\n return new_experiment_id\n\n def abort(\n self,\n experiment_id: Union[int, str],\n save: bool = True,\n send_gcs: bool = True,\n ):\n \"\"\"Abort an experiment - change status in db.\"\"\"\n self.db.dadd(str(experiment_id), (\"job_status\", \"aborted\"))\n if save:\n self.save(send_gcs)\n\n def delete(\n self,\n experiment_id: Union[int, str],\n save: bool = True,\n send_gcs: bool = True,\n ):\n \"\"\"Delete an experiment - change status in db.\"\"\"\n self.db.drem(str(experiment_id))\n self.all_experiment_ids = list(self.db.getall())\n try:\n self.all_experiment_ids.remove(\"summary\")\n except Exception:\n pass\n if len(self.all_experiment_ids) > 0:\n self.last_experiment_id = int(self.all_experiment_ids[-1])\n else:\n self.last_experiment_id = 0\n if save:\n self.save(send_gcs)\n\n def update_progress_bar(\n self,\n experiment_id: Union[int, str, None] = None,\n completed_increment: int = 1,\n pull_gcs: bool = False,\n save: bool = True,\n send_gcs: bool = False,\n ):\n \"\"\"Update progress bar of completed jobs using an integer increment.\"\"\"\n if experiment_id is None:\n experiment_id = self.added_experiment_id\n self.completed_jobs_counter += completed_increment\n try:\n self.load(pull_gcs)\n self.update(\n experiment_id,\n \"completed_jobs\",\n self.completed_jobs_counter,\n save=save,\n send_gcs=send_gcs,\n )\n except Exception:\n pass\n\n def complete(\n self,\n experiment_id: Union[int, str],\n report: bool = False,\n save: bool = True,\n ):\n \"\"\"Set status of an experiment to completed - change status in db.\"\"\"\n self.load()\n experiment_data = self.get(experiment_id)\n # Store experiment directory in GCS bucket under hash\n if self.use_gcs_protocol_storage:\n from .utils import send_gcloud_zip\n\n zip_to_store = experiment_data[\"e-hash\"] + \".zip\"\n experiment_dir = experiment_data[\"experiment_dir\"]\n send_gcloud_zip(\n self.cloud_settings, experiment_dir, zip_to_store, True\n )\n self.logger.info(f\"Send results to GCS: {zip_to_store}\")\n\n # Update and send protocol db\n time_t = datetime.now().strftime(\"%m/%d/%y %H:%M\")\n stop_time = datetime.strptime(time_t, \"%m/%d/%y %H:%M\")\n start_time = datetime.strptime(\n experiment_data[\"start_time\"], \"%m/%d/%y %H:%M\"\n )\n duration = str(stop_time - start_time)\n self.update(\n experiment_id,\n var_name=[\n \"job_status\",\n \"stop_time\",\n \"duration\",\n \"report_generated\",\n \"stored_in_gcloud\",\n \"completed_jobs\",\n ],\n var_value=[\n \"completed\",\n time_t,\n duration,\n report,\n self.use_gcs_protocol_storage,\n experiment_data[\"num_total_jobs\"],\n ],\n save=save,\n )\n self.logger.info(f\"Updated protocol - COMPLETED: {experiment_id}\")\n\n def status(self, experiment_id: Union[int, str]) -> str:\n \"\"\"Get the status of an experiment.\"\"\"\n return self.db.dget(str(experiment_id), \"job_status\")\n\n def update(\n self,\n experiment_id: Union[int, str],\n var_name: Union[List[str], str],\n var_value: Union[list, str, dict],\n save: bool = True,\n send_gcs: bool = True,\n ):\n \"\"\"Update the data of an experiment.\"\"\"\n # Update the variable(s) of the experiment\n if type(var_name) == list:\n for db_v_id in range(len(var_name)):\n self.db.dadd(\n str(experiment_id), (var_name[db_v_id], var_value[db_v_id])\n )\n else:\n self.db.dadd(str(experiment_id), (var_name, var_value))\n if save:\n self.save(send_gcs)\n\n def summary(\n self,\n tail: int = 5,\n verbose: bool = True,\n return_table: bool = False,\n full: bool = False,\n ):\n \"\"\"Print a rich summary table of all experiments in db.\"\"\"\n summary = protocol_summary(\n self.db, self.experiment_ids, tail, verbose, full\n )\n if return_table:\n return protocol_table(summary, full)\n return summary\n\n def monitor(self):\n \"\"\"Get monitoring data used in dashboard.\"\"\"\n total_data, last_data, time_data = get_monitor_db_data(self)\n protocol_table = self.summary(\n tail=50, verbose=False, return_table=True, full=True\n )\n summary_data = self.get(\"summary\")\n if not summary_data:\n summary_data = {\n \"time\": [],\n \"day\": [],\n \"total_exp\": {\n \"all\": [],\n \"hyperparameter-search\": [],\n \"multiple-configs\": [],\n \"single-config\": [],\n },\n \"day_exp\": {\n \"all\": [],\n \"hyperparameter-search\": [],\n \"multiple-configs\": [],\n \"single-config\": [],\n },\n }\n return {\n \"total_data\": total_data,\n \"last_data\": last_data,\n \"time_data\": time_data,\n \"summary_data\": summary_data,\n \"protocol_table\": protocol_table,\n }\n\n def retrieve(\n self,\n experiment_id: Union[int, str],\n local_dir_name: Union[None, str] = None,\n ):\n \"\"\"Retrieve experiment from GCS.\"\"\"\n from .utils import get_gcloud_zip\n\n while True:\n if str(experiment_id) not in self.experiment_ids:\n time_t = datetime.now().strftime(\"%m/%d/%Y %I:%M:%S %p\")\n print(\n time_t,\n \"The experiment you are trying to retrieve does not exist\",\n )\n experiment_id = input(\n time_t + \" Which experiment do you want to retrieve?\"\n )\n else:\n break\n\n # Update protocol retrieval status of the experiment\n hash_to_store = self.get(experiment_id, \"e-hash\")\n get_gcloud_zip(\n self.cloud_settings, hash_to_store, experiment_id, local_dir_name\n )\n self.update(experiment_id, \"retrieved_results\", True)\n\n self.logger.info(f\"Retrieved results from GCS bucket: {experiment_id}.\")\n\n def gcs_send(self):\n \"\"\"Send the local protocol to a GCS bucket.\"\"\"\n # First store the most recent log\n from .protocol import send_gcloud_db\n\n send_db = send_gcloud_db(\n self.cloud_settings[\"project_name\"],\n self.cloud_settings[\"bucket_name\"],\n self.cloud_settings[\"protocol_fname\"],\n self.protocol_fname,\n )\n self.logger.info(\n \"Send protocol to GCS bucket:\"\n f\" {self.cloud_settings['bucket_name']}.\"\n )\n return send_db\n\n def gcs_pull(self):\n \"\"\"Pull the remote protocol from a GCS bucket.\"\"\"\n from .protocol import get_gcloud_db\n\n accessed_db = get_gcloud_db(\n self.cloud_settings[\"project_name\"],\n self.cloud_settings[\"bucket_name\"],\n self.cloud_settings[\"protocol_fname\"],\n self.protocol_fname,\n )\n self.logger.info(\n \"Pulled protocol from GCS bucket:\"\n f\" {self.cloud_settings['bucket_name']}.\"\n )\n return accessed_db\n\n def __len__(self) -> int:\n \"\"\"Return number of experiments stored in protocol.\"\"\"\n return len(self.experiment_ids)\n\n def ask_for_e_id(self, action: str = \"delete\"):\n \"\"\"Ask user if they want to delete/abort previous experiment by id.\"\"\"\n time_t = datetime.now().strftime(\"%m/%d/%Y %I:%M:%S %p\")\n q_str = \"{} Want to {} an experiment? - state its id: [e_id/N]\"\n print(q_str.format(time_t, action), end=\" \")\n sys.stdout.flush()\n\n # Loop over experiments to delete until \"N\" given or timeout after 60 secs\n while True:\n i, o, e = select.select([sys.stdin], [], [], 60)\n if i:\n e_id = sys.stdin.readline().strip()\n if e_id == \"N\":\n return e_id\n else:\n break\n\n if e_id not in self.experiment_ids:\n time_t = datetime.now().strftime(\"%m/%d/%Y %I:%M:%S %p\")\n print(\n \"\\n{} The e_id is not in the protocol db. \"\n \"Please try again: [e_id/N]\".format(time_t),\n end=\" \",\n )\n sys.stdout.flush()\n continue\n\n # Delete the dictionary in DB corresponding to e-id\n if action == \"delete\":\n self.delete(e_id, save=True)\n elif action == \"abort\":\n self.abort(e_id, save=True)\n else:\n return e_id\n\n print(\n \"{} Another one? - state the next id: [e_id/N]\".format(time_t),\n end=\" \",\n )\n sys.stdout.flush()\n\n def ask_for_purpose(self):\n \"\"\"Ask user for purpose of experiment.\"\"\"\n # Add purpose of experiment - cmd args or timeout input after 30 secs\n time_t = datetime.now().strftime(\"%m/%d/%Y %I:%M:%S %p\")\n print(f\"{time_t} Purpose of experiment?\", end=\" \"),\n sys.stdout.flush()\n i, o, e = select.select([sys.stdin], [], [], 60)\n\n if i:\n purpose = sys.stdin.readline().strip()\n else:\n purpose = \"default\"\n return purpose\n"
},
{
"path": "mle_monitor/mle_resource.py",
"content": "from typing import Union\nfrom .resource import SGEResource, SlurmResource, LocalResource, GCPResource\n\n\nclass MLEResource(object):\n def __init__(\n self, resource_name: str = \"local\", monitor_config: Union[dict, None] = None\n ):\n \"\"\"MLE Resource Instance - Get Monitoring Data.\"\"\"\n assert resource_name in [\"local\", \"sge-cluster\", \"slurm-cluster\", \"gcp-cloud\"]\n self.resource_name = resource_name\n self.monitor_config = monitor_config\n\n if self.resource_name == \"sge-cluster\":\n self.resource = SGEResource(self.monitor_config)\n elif self.resource_name == \"slurm-cluster\":\n self.resource = SlurmResource(self.monitor_config)\n elif self.resource_name == \"gcp-cloud\":\n self.resource = GCPResource(self.monitor_config)\n else:\n self.resource = LocalResource(self.monitor_config)\n\n def monitor(self):\n \"\"\"Get utilization data.\"\"\"\n # Get resource dependent data\n if self.resource_name == \"local\":\n user_data, host_data, util_data = self.resource.monitor()\n return {\n \"resource_name\": self.resource_name,\n \"user_data\": user_data,\n \"host_data\": host_data,\n \"util_data\": util_data,\n }\n elif self.resource_name in [\"sge-cluster\", \"slurm-cluster\"]:\n user_data, host_data, util_data, node_data = self.resource.monitor()\n return {\n \"resource_name\": self.resource_name,\n \"user_data\": user_data,\n \"host_data\": host_data,\n \"util_data\": util_data,\n \"node_data\": node_data,\n }\n else:\n gcp_data = self.resource.monitor()\n return gcp_data\n"
},
{
"path": "mle_monitor/protocol/__init__.py",
"content": "from .load import load_protocol_db\nfrom .tables import protocol_summary, protocol_table\nfrom .add import protocol_experiment\nfrom .summary import get_monitor_db_data\nfrom .gcs_sync import set_gcp_credentials, send_gcloud_db, get_gcloud_db\n\n\n__all__ = [\n \"load_protocol_db\",\n \"protocol_summary\",\n \"protocol_table\",\n \"protocol_experiment\",\n \"get_monitor_db_data\",\n \"set_gcp_credentials\",\n \"send_gcloud_db\",\n \"get_gcloud_db\",\n]\n"
},
{
"path": "mle_monitor/protocol/add.py",
"content": "import os\nimport hashlib\nimport json\nimport datetime as dt\nfrom datetime import datetime\nfrom typing import Union, Tuple\nfrom ..utils import load_json_config, load_yaml_config\n\n\ndef protocol_experiment(\n db, last_experiment_id, standard: dict, extra: Union[dict, None] = None\n):\n \"\"\"Add the new experiment to the protocol database.\"\"\"\n # Add experiment summary data\n add_experiment_summary(db, standard[\"experiment_type\"])\n\n # Create a new db experiment entry\n new_experiment_id = str(last_experiment_id + 1)\n db.dcreate(new_experiment_id)\n\n # Add all standard & extra data points to protocol\n for k, v in standard.items():\n db.dadd(new_experiment_id, (k, v))\n\n if extra is not None:\n for k, v in extra.items():\n db.dadd(new_experiment_id, (k, v))\n\n try:\n import git\n\n except ImportError:\n print(\"You need to install `GitPython` to use git hash protocolling.\")\n\n # Add the git latest commit hash\n try:\n g = git.Repo(search_parent_directories=True)\n git_hash = g.head.object.hexsha\n except Exception:\n git_hash = \"no-git-repo\"\n db.dadd(new_experiment_id, (\"git_hash\", git_hash))\n\n # Add the base config - load from given configuration file(s)\n if type(standard[\"config_fname\"]) == str:\n all_config_paths = [standard[\"config_fname\"]]\n elif type(standard[\"base_train_config\"]) == list:\n all_config_paths = standard[\"config_fname\"]\n else:\n all_config_paths = []\n\n loaded_configs = []\n for config_path in all_config_paths:\n _, fext = os.path.splitext(config_path)\n if fext == \".json\":\n base_config = load_json_config(config_path)\n elif fext == \".yaml\":\n base_config = load_yaml_config(config_path)\n else:\n base_config = {}\n loaded_configs.append(base_config)\n\n db.dadd(new_experiment_id, (\"loaded_config\", loaded_configs))\n\n # Hash to store results by in GCS bucket (timestamp, config_fname)\n config_to_hash = {\n **{\"time\": datetime.now().strftime(\"%m/%d/%Y %H:%M:%S\")},\n **{\"config_fname\": standard[\"config_fname\"]},\n }\n hash_object = hashlib.md5(json.dumps(config_to_hash).encode(\"ASCII\"))\n experiment_hash = hash_object.hexdigest()\n\n # Set of booleans: previously retrieved, stored in GCS, report generated\n db.dadd(new_experiment_id, (\"e-hash\", experiment_hash))\n db.dadd(new_experiment_id, (\"retrieved_results\", False))\n db.dadd(new_experiment_id, (\"stored_in_cloud\", False))\n db.dadd(new_experiment_id, (\"report_generated\", False))\n\n # Set the job status to running & calculate time till completion\n db.dadd(new_experiment_id, (\"job_status\", \"running\"))\n db.dadd(new_experiment_id, (\"completed_jobs\", 0))\n start_time = datetime.now().strftime(\"%m/%d/%y %H:%M\")\n est_duration, stop_time = estimate_experiment_duration(\n start_time, standard[\"time_per_job\"], standard[\"num_job_batches\"]\n )\n db.dadd(new_experiment_id, (\"start_time\", start_time))\n db.dadd(new_experiment_id, (\"duration\", est_duration))\n db.dadd(new_experiment_id, (\"stop_time\", stop_time))\n return db, int(new_experiment_id)\n\n\ndef estimate_experiment_duration(\n start_time: str, time_per_batch: str, total_batches: int\n) -> Tuple[str, str]:\n \"\"\"Estimate total duration and completion time of experiment.\"\"\"\n days, hours, minutes = time_per_batch.split(\":\")\n hours_add, tot_mins = divmod(total_batches * int(minutes), 60)\n days_add, tot_hours = divmod(total_batches * int(hours) + hours_add, 24)\n tot_days = total_batches * int(days) + days_add\n tot_days, tot_hours, tot_mins = (\n str(tot_days),\n str(tot_hours),\n str(tot_mins),\n )\n if len(tot_days) < 2:\n tot_days = tot_days\n if len(tot_hours) < 2:\n tot_hours = \"0\" + tot_hours\n if len(tot_mins) < 2:\n tot_mins = \"0\" + tot_mins\n\n start_date = dt.datetime.strptime(start_time, \"%m/%d/%y %H:%M\")\n end_date = start_date + dt.timedelta(\n days=int(float(tot_days)),\n hours=int(float(tot_hours)),\n minutes=int(float(tot_mins)),\n )\n est_duration = tot_days + \":\" + tot_hours + \":\" + tot_mins\n stop_time = end_date.strftime(\"%m/%d/%y %H:%M\")\n return est_duration, stop_time\n\n\ndef add_experiment_summary(db, experiment_type: str):\n \"\"\"Update the summary data of the protocol.\"\"\"\n all_experiment_types = [\n \"hyperparameter-search\",\n \"multiple-configs\",\n \"single-config\",\n ]\n start_time = datetime.now().strftime(\"%m/%d/%y %H:%M\")\n start_day = datetime.now().strftime(\"%m/%d/%y\")\n if \"summary\" not in list(db.getall()):\n db.dcreate(\"summary\")\n # Create dict that stores timeseries of cumulative experiments by type\n db.dadd(\"summary\", (\"time\", [start_time]))\n total_dict = {\"all\": [1]}\n for k in all_experiment_types:\n total_dict[k] = [0]\n total_dict[experiment_type][-1] += 1\n db.dadd(\"summary\", (\"total_exp\", total_dict))\n\n # Create dict that stores timeseries of daily experiments by type\n db.dadd(\"summary\", (\"day\", [start_day]))\n day_dict = {\"all\": [1]}\n for k in all_experiment_types:\n day_dict[k] = [0]\n day_dict[experiment_type][-1] += 1\n db.dadd(\"summary\", (\"day_exp\", day_dict))\n else:\n data = db.get(\"summary\")\n # Add new experiment to total dict\n total_dict = data[\"total_exp\"]\n data[\"time\"].append(start_time)\n db.dadd(\"summary\", (\"time\", data[\"time\"]))\n total_dict[\"all\"].append(total_dict[\"all\"][-1] + 1)\n for k in all_experiment_types:\n total_dict[k].append(total_dict[k][-1])\n total_dict[experiment_type][-1] += 1\n db.dadd(\"summary\", (\"total_exp\", total_dict))\n\n # Add new experiment to daily dict\n day_dict = data[\"day_exp\"]\n try:\n idx = data[\"day\"].index(start_day)\n day_dict[experiment_type][idx] += 1\n db.dadd(\"summary\", (\"day_exp\", day_dict))\n except Exception:\n data[\"day\"].append(start_day)\n db.dadd(\"summary\", (\"day\", data[\"day\"]))\n for k in all_experiment_types:\n day_dict[k].append(0)\n day_dict[experiment_type][-1] += 1\n db.dadd(\"summary\", (\"day_exp\", day_dict))\n return\n"
},
{
"path": "mle_monitor/protocol/gcs_sync.py",
"content": "import logging\nimport os\nfrom os.path import expanduser\nfrom ..utils import setup_logger\n\n\ndef set_gcp_credentials(credentials_path: str = \"\"):\n if credentials_path != \"\":\n os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = os.path.expanduser(\n credentials_path\n )\n\n\ndef get_gcloud_db(\n project_name: str,\n bucket_name: str,\n gcs_protocol_fname: str,\n local_protocol_fname: str,\n number_of_connect_tries: int = 5,\n) -> int:\n \"\"\"Pull latest experiment database from gcloud storage.\"\"\"\n try:\n from google.cloud import storage\n\n except ImportError:\n raise ImportError(\n \"You need to install `google-cloud-storage` to use GCP buckets.\"\n )\n logger = setup_logger()\n for i in range(number_of_connect_tries):\n try:\n # Connect to project and bucket\n client = storage.Client(project_name)\n bucket = client.get_bucket(bucket_name, timeout=20)\n # Download blob to db file\n blob = bucket.blob(gcs_protocol_fname)\n with open(expanduser(local_protocol_fname), \"wb\") as file_obj:\n blob.download_to_file(file_obj)\n logger.info(f\"Pulled from GCloud Storage - {gcs_protocol_fname}\")\n return 1\n except Exception as ex:\n # Remove empty file - causes error otherwise when trying to load\n try:\n os.remove(expanduser(local_protocol_fname))\n except Exception:\n pass\n if type(ex).__name__ == \"NotFound\":\n logger.info(\n f\"No DB found in GCloud Storage - {gcs_protocol_fname}\"\n )\n logger.info(\n f\"New DB will be created - {project_name}/{bucket_name}\"\n )\n return 1\n else:\n logger.info(\n f\"Attempt {i+1}/{number_of_connect_tries}\"\n \" - Failed pulling from GCloud Storage\"\n f\" - {type(ex).__name__}\"\n )\n # If after 5 pulls no successful connection established - return failure\n return 0\n\n\ndef send_gcloud_db(\n project_name: str,\n bucket_name: str,\n gcs_protocol_fname: str,\n local_protocol_fname: str,\n number_of_connect_tries: int = 5,\n):\n \"\"\"Send updated database back to gcloud storage.\"\"\"\n try:\n from google.cloud import storage\n\n except ImportError:\n raise ImportError(\n \"You need to install `google-cloud-storage` to use GCP buckets.\"\n )\n logger = setup_logger()\n for i in range(number_of_connect_tries):\n try:\n # Connect to project and bucket\n client = storage.Client(project_name)\n bucket = client.get_bucket(bucket_name, timeout=20)\n blob = bucket.blob(gcs_protocol_fname)\n blob.upload_from_filename(filename=expanduser(local_protocol_fname))\n logger.info(f\"Send to GCloud Storage - {gcs_protocol_fname}\")\n return 1\n except Exception:\n logger.info(\n f\"Attempt {i+1}/{number_of_connect_tries}\"\n \" - Failed sending to GCloud Storage\"\n )\n # If after 5 pulls no successful connection established - return failure\n return 0\n"
},
{
"path": "mle_monitor/protocol/load.py",
"content": "import re\nimport pickledb\n\n\ndef load_protocol_db(protocol_fname):\n \"\"\"Load local database from config name & reconstruct experiment id.\"\"\"\n # Attempt loading local protocol database - otherwise return clean one\n db = pickledb.load(protocol_fname, False, sig=False)\n # Get the most recent experiment id\n all_experiment_ids = list(db.getall())\n try:\n all_experiment_ids.remove(\"summary\")\n except:\n pass\n\n def natural_keys(text: str):\n \"\"\"Helper function for sorting alpha-numeric strings.\"\"\"\n\n def atoi(text):\n return int(text) if text.isdigit() else text\n\n return [atoi(c) for c in re.split(r\"(\\d+)\", text)]\n\n # Sort experiment ids & get the last one\n if len(all_experiment_ids) > 0:\n all_experiment_ids.sort(key=natural_keys)\n last_experiment_id = int(all_experiment_ids[-1])\n else:\n last_experiment_id = 0\n return db, all_experiment_ids, last_experiment_id\n"
},
{
"path": "mle_monitor/protocol/summary.py",
"content": "def get_monitor_db_data(db):\n \"\"\"Helper to get all data from pickledb database.\"\"\"\n if len(db.experiment_ids) > 0:\n total_data = get_total_experiments(db, db.experiment_ids)\n last_data = get_last_experiment(db, db.experiment_ids[-1])\n time_data = get_time_experiment(db, db.experiment_ids[-1])\n else:\n total_data = {\n \"total\": \"0\",\n \"run\": \"0\",\n \"done\": \"0\",\n \"aborted\": \"0\",\n \"sge\": \"0\",\n \"slurm\": \"0\",\n \"gcp\": \"0\",\n \"local\": \"0\",\n \"report_gen\": \"0\",\n \"gcs_stored\": \"0\",\n \"retrieved\": \"0\",\n }\n last_data = {\n \"e_id\": \"-\",\n \"e_dir\": \"-\",\n \"e_type\": \"-\",\n \"e_script\": \"-\",\n \"e_config\": \"-\",\n \"report_gen\": \"-\",\n \"job_status\": \"-\",\n \"resource\": \"-\",\n }\n time_data = {\n \"job_status\": \"-\",\n \"total_jobs\": 1,\n \"total_batches\": 1,\n \"completed_jobs\": 0,\n \"jobs_per_batch\": \"-\",\n \"time_per_batch\": \"-\",\n \"start_time\": \"-\",\n \"stop_time\": \"-\",\n \"duration\": \"-\",\n \"num_seeds\": 1,\n }\n return total_data, last_data, time_data\n\n\ndef get_total_experiments(db, all_experiment_ids):\n \"\"\"Get data from db to show in 'total_experiments' panel.\"\"\"\n run, done, aborted, sge, slurm, gcp, local = 0, 0, 0, 0, 0, 0, 0\n report_gen, gcs_stored, retrieved = 0, 0, 0\n for e_id in all_experiment_ids:\n status = db.get(e_id, \"job_status\")\n # Job status\n run += status == \"running\"\n done += status == \"completed\"\n aborted += status not in [\"running\", \"completed\"]\n # Execution resource data\n resource = db.get(e_id, \"exec_resource\")\n sge += resource == \"sge-cluster\"\n slurm += resource == \"slurm-cluster\"\n gcp += resource == \"gcp-cloud\"\n local += resource not in [\"sge-cluster\", \"slurm-cluster\", \"gcp-cloud\"]\n # Additional data: Report generated, GCS stored, Results retrieved\n try:\n report_gen += db.get(e_id, \"report_generated\")\n gcs_stored += db.get(e_id, \"stored_in_gcloud\")\n retrieved += db.get(e_id, \"retrieved_results\")\n except Exception:\n pass\n # Return results dictionary\n results = {\n \"total\": str(len(all_experiment_ids)),\n \"run\": str(run),\n \"done\": str(done),\n \"aborted\": str(aborted),\n \"sge\": str(sge),\n \"slurm\": str(slurm),\n \"gcp\": str(gcp),\n \"local\": str(local),\n \"report_gen\": str(report_gen),\n \"gcs_stored\": str(gcs_stored),\n \"retrieved\": str(retrieved),\n }\n return results\n\n\ndef get_time_experiment(db, last_experiment_id):\n \"\"\"Get data from db to show in 'time_experiment' panel.\"\"\"\n last_experiment = db.get(last_experiment_id)\n results = {\n \"num_seeds\": last_experiment[\"num_seeds\"],\n \"total_jobs\": last_experiment[\"num_total_jobs\"],\n \"completed_jobs\": last_experiment[\"completed_jobs\"],\n \"total_batches\": last_experiment[\"num_job_batches\"],\n \"jobs_per_batch\": last_experiment[\"num_jobs_per_batch\"],\n \"time_per_batch\": last_experiment[\"time_per_job\"],\n \"start_time\": last_experiment[\"start_time\"],\n \"stop_time\": last_experiment[\"stop_time\"],\n \"duration\": last_experiment[\"duration\"],\n \"job_status\": last_experiment[\"job_status\"],\n }\n return results\n\n\ndef get_last_experiment(db, last_experiment_id):\n \"\"\"Get data from db to show in 'last_experiments' panel.\"\"\"\n last_experiment = db.get(last_experiment_id)\n results = {\n \"e_id\": str(last_experiment_id),\n \"job_status\": last_experiment[\"job_status\"],\n \"e_dir\": last_experiment[\"experiment_dir\"],\n \"e_type\": last_experiment[\"experiment_type\"],\n \"e_script\": last_experiment[\"base_fname\"],\n \"e_config\": last_experiment[\"config_fname\"],\n \"report_gen\": last_experiment[\"report_generated\"],\n \"resource\": last_experiment[\"exec_resource\"],\n }\n return results\n"
},
{
"path": "mle_monitor/protocol/tables.py",
"content": "import pandas as pd\nfrom datetime import datetime\n\nfrom rich import box\nfrom rich.table import Table\nfrom rich.spinner import Spinner\nfrom rich.console import Console\nfrom rich.align import Align\nfrom rich.progress import (\n BarColumn,\n Progress,\n TextColumn,\n)\n\n\ndef protocol_summary(\n db, all_experiment_ids, tail: int = 5, verbose: bool = True, full: bool = False\n):\n \"\"\"Construct a summary dataframe of previous experiments.\"\"\"\n # Set pandas df format option to print\n pd.set_option(\"display.max_columns\", 5)\n pd.set_option(\"max_colwidth\", 30)\n if len(all_experiment_ids) > 0:\n purposes, project_names, exp_paths = [], [], []\n num_seeds, statuses, start_times, experiment_types = [], [], [], []\n resource, num_cpus, num_gpus, total_jobs, completed_jobs = [], [], [], [], []\n if tail is None:\n tail = len(all_experiment_ids)\n\n # Loop over experiment ids and extract data to retrieve\n for int_e_id in all_experiment_ids[-tail:]:\n e_id = str(int_e_id)\n purposes.append(db.dget(e_id, \"purpose\"))\n project_names.append(db.dget(e_id, \"project_name\"))\n exp_paths.append(db.dget(e_id, \"experiment_dir\"))\n statuses.append(db.dget(e_id, \"job_status\"))\n start_times.append(db.dget(e_id, \"start_time\"))\n resource.append(db.dget(e_id, \"exec_resource\"))\n num_seeds.append(db.dget(e_id, \"num_seeds\"))\n num_cpus.append(db.dget(e_id, \"num_cpus\"))\n num_gpus.append(db.dget(e_id, \"num_gpus\"))\n experiment_types.append(db.dget(e_id, \"experiment_type\"))\n total_jobs.append(db.dget(e_id, \"num_total_jobs\"))\n completed_jobs.append(db.dget(e_id, \"completed_jobs\"))\n\n d = {\n \"ID\": [str(e_id) for e_id in all_experiment_ids[-tail:]],\n \"Date\": start_times,\n \"Project\": project_names,\n \"Purpose\": purposes,\n \"Experiment Dir\": exp_paths,\n \"Status\": statuses,\n \"Seeds\": num_seeds,\n \"Resource\": resource,\n \"CPUs\": num_cpus,\n \"GPUs\": num_gpus,\n \"Type\": experiment_types,\n \"Jobs\": total_jobs,\n \"Completed Jobs\": completed_jobs,\n }\n df = pd.DataFrame(d)\n df[\"Date\"] = df[\"Date\"].map(\"{:.5}\".format)\n df[\"Purpose\"] = df[\"Purpose\"].map(\"{:.30}\".format)\n\n # Print a nice table overview (no job resources)\n if verbose:\n Console().print(Align.left(protocol_table(df, full)))\n return df\n else:\n if verbose:\n time_t = datetime.now().strftime(\"%m/%d/%Y %I:%M:%S %p\")\n print(time_t, \"No previously recorded experiments\")\n return None\n\n\ndef get_progress_bar(total_jobs: int, completed_jobs: int):\n progress = Progress(\n TextColumn(\n \"{task.completed:^3.0f}/{task.total:^3.0f}\", justify=\"left\", style=\"white\"\n ),\n BarColumn(bar_width=10, style=\"red\"),\n TextColumn(\"[progress.percentage]{task.percentage:>3.0f}%\", style=\"white\"),\n auto_refresh=False,\n )\n\n task = progress.add_task(\"queue\", total=total_jobs)\n progress.update(task, completed=completed_jobs, refresh=True)\n return progress\n\n\ndef protocol_table(df, full: bool = True):\n \"\"\"Generate pretty table of experiment protocol db - preselected db.\"\"\"\n table = Table(show_header=True, show_footer=False, header_style=\"bold blue\")\n table.add_column(\":bookmark:\", justify=\"center\")\n table.add_column(\":id:\", justify=\"center\")\n table.add_column(\":spiral_calendar:\", justify=\"center\")\n table.add_column(\"Project\")\n table.add_column(\"Purpose\")\n table.add_column(\"Type\")\n table.add_column(\"[yellow]:arrow_forward:\", justify=\"center\")\n table.add_column(\"[yellow]:recycle:\", justify=\"center\")\n table.add_column(\"CPU\", justify=\"center\")\n table.add_column(\"GPU\", justify=\"center\")\n # Full option prints also resource requirements of jobs\n if full:\n table.add_column(\n \":hourglass_flowing_sand: Completed Jobs [yellow]:heavy_check_mark:\",\n justify=\"center\",\n )\n\n # Add rows of info if dataframe exists (previously recorded experiments)\n if df is not None:\n for index in reversed(df.index):\n row = df.iloc[index]\n if row[\"Resource\"] == \"sge-cluster\":\n resource = \"SGE\"\n elif row[\"Resource\"] == \"slurm-cluster\":\n resource = \"Slurm\"\n elif row[\"Resource\"] == \"gcp-cloud\":\n resource = \"GCP\"\n else:\n resource = \"Local\"\n\n if row[\"Type\"] == \"hyperparameter-search\":\n exp_type = \"search\"\n elif row[\"Type\"] == \"multiple-configs\":\n exp_type = \"config\"\n elif row[\"Type\"] == \"single-config\":\n exp_type = \"single\"\n else:\n exp_type = row[\"Type\"]\n\n if row[\"Status\"] == \"running\":\n status = Spinner(\"dots\", style=\"magenta\")\n elif row[\"Status\"] == \"completed\":\n status = \"[green]:heavy_check_mark:\"\n else:\n status = \"[red]:heavy_multiplication_x:\"\n\n if full:\n bar = get_progress_bar(int(row[\"Jobs\"]), int(row[\"Completed Jobs\"]))\n table.add_row(\n status,\n row[\"ID\"],\n row[\"Date\"],\n row[\"Project\"][:10],\n row[\"Purpose\"][:15],\n exp_type,\n resource,\n str(row[\"Seeds\"]),\n str(row[\"CPUs\"]),\n str(row[\"GPUs\"]),\n bar,\n )\n else:\n table.add_row(\n status,\n row[\"ID\"],\n row[\"Date\"],\n row[\"Project\"][:10],\n row[\"Purpose\"][:25],\n exp_type,\n resource,\n str(row[\"Seeds\"]),\n str(row[\"CPUs\"]),\n str(row[\"GPUs\"]),\n )\n\n table.border_style = \"blue\"\n table.box = box.SIMPLE_HEAD\n return table\n"
},
{
"path": "mle_monitor/resource/__init__.py",
"content": "from .sge import SGEResource\nfrom .slurm import SlurmResource\nfrom .gcp import GCPResource\nfrom .local import LocalResource\n\n\n__all__ = [\"SGEResource\", \"SlurmResource\", \"GCPResource\", \"LocalResource\"]\n"
},
{
"path": "mle_monitor/resource/gcp.py",
"content": "import time\nimport subprocess as sp\nimport pandas as pd\nfrom typing import Union\n\n\nclass GCPResource(object):\n def __init__(self, monitor_config: Union[dict, None]):\n self.resource_name = \"gcp-cloud\"\n self.monitor_config = monitor_config\n\n def monitor(self):\n \"\"\"Helper to get all utilisation data for resource.\"\"\"\n return self.get_data()\n\n def get_data(self):\n \"\"\"Helper to get all utilisation data for GCP resource.\"\"\"\n while True:\n try:\n check_cmd = [\n \"gcloud\",\n \"compute\",\n \"instances\",\n \"list\",\n \"--verbosity\",\n \"error\",\n ]\n out = sp.check_output(check_cmd)\n break\n except sp.CalledProcessError:\n time.sleep(1)\n\n # Clean up and check if vm_name is in list of all jobs\n job_info = out.split(b\"\\n\")[1:-1]\n df_gcp = {\"experiment_type\": [], \"status\": []}\n for i in range(len(job_info)):\n decoded_job_info = job_info[i].decode(\"utf-8\").split()\n df_gcp[\"experiment_type\"].append(decoded_job_info[2])\n df_gcp[\"status\"].append(decoded_job_info[-1])\n df_gcp = pd.DataFrame(df_gcp)\n\n gcp_data = {\"experiment_type\": [], \"run\": [], \"stop\": [], \"stage\": []}\n for jt in df_gcp.experiment_type.unique():\n sub_df = df_gcp[df_gcp.experiment_type == jt]\n run = (sub_df[\"status\"] == \"RUNNING\").sum()\n stop = (sub_df[\"status\"] == \"STOPPING\").sum()\n stage = (sub_df[\"status\"] == \"STAGING\").sum()\n gcp_data[\"experiment_type\"].append(jt)\n gcp_data[\"run\"].append(run)\n gcp_data[\"stop\"].append(stop)\n gcp_data[\"stage\"].append(stage)\n # Return list of different machine types and their status\n return gcp_data\n"
},
{
"path": "mle_monitor/resource/local.py",
"content": "from datetime import datetime\nimport numpy as np\nfrom typing import Union\n\n\nclass LocalResource(object):\n def __init__(self, monitor_config: Union[dict, None]):\n self.resource_name = \"local\"\n self.monitor_config = monitor_config\n\n def monitor(self):\n \"\"\"Helper to get all utilisation data for local resource.\"\"\"\n proc_data = self.get_process_data()\n device_data = self.get_device_data()\n util_data = self.get_util_data()\n return proc_data, device_data, util_data\n\n def get_process_data(self):\n \"\"\"Get process info running on local machine.\"\"\"\n proc_data = {\n \"pid\": [],\n \"p_name\": [],\n \"mem_util\": [],\n \"cpu_util\": [],\n \"cmdline\": [],\n }\n\n # Get top 5 most memory/cpu demanding processes on local machine\n mem_sort, cpu_sort, total_cpu, total_mem = self.get_cpu_processes()\n\n for p in cpu_sort:\n proc_data[\"pid\"].append(p[\"pid\"])\n proc_data[\"p_name\"].append(p[\"name\"])\n proc_data[\"mem_util\"].append(p[\"vms\"])\n proc_data[\"cpu_util\"].append(p[\"cpu_percent\"])\n proc_data[\"cmdline\"].append(p[\"cmdline\"])\n\n for p in mem_sort:\n proc_data[\"pid\"].append(p[\"pid\"])\n proc_data[\"p_name\"].append(p[\"name\"])\n proc_data[\"mem_util\"].append(p[\"vms\"])\n proc_data[\"cpu_util\"].append(p[\"cpu_percent\"])\n proc_data[\"cmdline\"].append(p[\"cmdline\"])\n\n proc_data[\"total_cpu_util\"] = total_cpu\n proc_data[\"total_mem_util\"] = total_mem\n return proc_data\n\n def get_cpu_processes(self, top_k: int = 6):\n \"\"\"Get list of process sorted by Memory (MB)/CPU Usage (CPU%).\"\"\"\n try:\n import psutil\n\n except ImportError:\n raise ImportError(\n \"You need to install `psutil` to monitor CPU processes.\"\n )\n\n listOfProcObjects = []\n # Iterate over the list\n for proc in psutil.process_iter():\n try:\n # Fetch process details as dict\n pinfo = proc.as_dict(\n attrs=[\"pid\", \"name\", \"username\", \"cmdline\"]\n )\n pinfo[\"vms\"] = proc.memory_info().vms / (1024 * 1024 * 1000)\n pinfo[\"cpu_percent\"] = proc.cpu_percent()\n # Append dict to list\n listOfProcObjects.append(pinfo)\n except (\n psutil.NoSuchProcess,\n psutil.AccessDenied,\n psutil.ZombieProcess,\n ):\n pass\n\n # Sort list of dict by key vms i.e. memory usage\n sorted_by_memory = sorted(\n listOfProcObjects, key=lambda procObj: procObj[\"vms\"], reverse=True\n )\n sorted_by_cpu = sorted(\n listOfProcObjects,\n key=lambda procObj: procObj[\"cpu_percent\"],\n reverse=True,\n )\n\n total_cpu = psutil.cpu_count() * psutil.cpu_percent()\n total_mem = psutil.virtual_memory()._asdict()[\"used\"] / 1000000\n return (\n sorted_by_memory[:top_k],\n sorted_by_cpu[:top_k],\n total_cpu,\n total_mem,\n )\n\n def get_device_data(self):\n \"\"\"Get utilization per core.\"\"\"\n try:\n import psutil\n\n except ImportError:\n raise ImportError(\n \"You need to install `psutil` to monitor CPU processes.\"\n )\n\n device_data = {\n \"core_id\": np.arange(1, psutil.cpu_count() + 1),\n \"percent_util\": [\n round(e, 1) for e in psutil.cpu_percent(interval=1, percpu=True)\n ],\n \"cpu_count\": psutil.cpu_count(logical=True),\n \"logical_count\": psutil.cpu_count(),\n }\n # Add GPU usage data via GPUtil\n try:\n for k, v in self.get_nvidia_gpu_data().items():\n device_data[k] = v\n except Exception:\n pass\n return device_data\n\n def get_nvidia_gpu_data(self):\n \"\"\"Helper function to get NVIDIA GPU utilisation.\"\"\"\n try:\n import GPUtil\n\n except ImportError:\n raise ImportError(\n \"You need to install `gputil` to monitor GPU processes.\"\n )\n gpus = GPUtil.getGPUs()\n gpu_data = {\n \"gpu_id\": [],\n \"gpu_name\": [],\n \"gpu_load\": [],\n \"gpu_mem_util\": [],\n \"gpu_mem_total\": [],\n \"gpu_mem_used\": [],\n }\n\n # Loop over experts and add data\n for gp in gpus:\n gpu_data[\"gpu_id\"].append(gp.id),\n gpu_data[\"gpu_name\"].append(gp.name)\n # Load and memory usage in %\n gpu_data[\"gpu_load\"].append(round(gp.load * 100, 1))\n gpu_data[\"gpu_mem_util\"].append(round(gp.memoryUtil * 100, 1))\n # Memory in GB\n gpu_data[\"gpu_mem_total\"].append(round(gp.memoryTotal / 1000, 1))\n gpu_data[\"gpu_mem_used\"].append(round(gp.memoryUsed / 1000, 1))\n return gpu_data\n\n def get_util_data(self):\n \"\"\"Get memory and CPU utilisation for specific local machine.\"\"\"\n try:\n import psutil\n\n except ImportError:\n raise ImportError(\n \"You need to install `psutil` to monitor CPU processes.\"\n )\n num_cpus = psutil.cpu_count()\n total_mem = psutil.virtual_memory()._asdict()[\"total\"] / 1000000000\n used_mem = psutil.virtual_memory()._asdict()[\"used\"] / 1000000000\n util_data = {\n \"cores\": num_cpus,\n \"cores_util\": num_cpus * psutil.cpu_percent() / 100,\n \"mem\": total_mem,\n \"mem_util\": used_mem,\n \"time_date\": datetime.now().strftime(\"%m/%d/%y\"),\n \"time_hour\": datetime.now().strftime(\"%H:%M:%S\"),\n }\n return util_data\n"
},
{
"path": "mle_monitor/resource/sge.py",
"content": "from datetime import datetime\nimport subprocess as sp\nfrom typing import Union\nimport numpy as np\nimport pandas as pd\nfrom ..utils import natural_keys\n\n\nclass SGEResource(object):\n def __init__(self, monitor_config: Union[dict, None]):\n self.resource_name = \"sge-cluster\"\n self.monitor_config = monitor_config\n\n def monitor(self):\n \"\"\"Helper to get all utilisation data for resource.\"\"\"\n user_data, job_df = self.get_user_data()\n queue_data = self.get_queue_data(job_df)\n node_data = self.get_node_data(job_df)\n util_data = self.get_util_data()\n return user_data, queue_data, util_data, node_data\n\n def get_user_data(self):\n \"\"\"Get jobs scheduled by Slurm cluster users.\n Return dictionary with `users`, `total`, `run`, `wait`, `login`.\n \"\"\"\n user_data = {\"user\": [], \"total\": [], \"run\": [], \"wait\": [], \"login\": []}\n all_users = sp.check_output([\"qconf\", \"-suserl\"]).split(b\"\\n\")[:-1]\n all_users = [u.decode() for u in all_users]\n user_cmd = [[\"-u\", u] for u in all_users]\n user_cmd = [item for sublist in user_cmd for item in sublist]\n queue_cmd = [[\"-q\", q] for q in self.monitor_config[\"queues\"]]\n queue_cmd = [item for sublist in queue_cmd for item in sublist]\n\n # Check all users and queues in one go\n all_job_infos = sp.check_output([\"qstat\", *user_cmd, *queue_cmd]).split(b\"\\n\")[\n 2:-1\n ]\n\n job_df = {\n \"user\": [],\n \"queue\": [],\n \"status\": [],\n \"node\": [],\n }\n\n # Clean all jobs (qlogin, qsub, etc.) + get unique users/hosts\n for job in all_job_infos:\n job_clean = job.decode().split(\" \")\n job_clean = list(filter(None, job_clean))\n host_ip = job_clean[7].split(\"@\")\n job_df[\"user\"].append(job_clean[3])\n if len(host_ip) > 1:\n job_df[\"queue\"].append(host_ip[0])\n job_df[\"node\"].append(host_ip[1][:-1])\n else:\n job_df[\"queue\"].append(None)\n job_df[\"node\"].append(None)\n\n if \"QLOGIN\" in job_clean and job_clean[4] == \"r\":\n job_df[\"status\"].append(\"LOGIN\")\n elif \"QLOGIN\" not in job_clean and job_clean[4] == \"r\":\n job_df[\"status\"].append(\"R\")\n else:\n job_df[\"status\"].append(\"PD\")\n job_df = pd.DataFrame(job_df)\n\n # Loop over unique users and construct data to show\n unique_users = job_df.user.unique().tolist()\n for u_id in unique_users:\n sub_df = job_df.loc[job_df[\"user\"] == u_id]\n user_data[\"user\"].append(u_id)\n user_data[\"total\"].append(sub_df.shape[0])\n sub_run = sub_df.loc[sub_df[\"status\"] == \"R\"]\n user_data[\"run\"].append(sub_run.shape[0])\n sub_wait = sub_df.loc[sub_df[\"status\"] == \"PD\"]\n user_data[\"wait\"].append(sub_wait.shape[0])\n sub_login = sub_df.loc[sub_df[\"status\"] == \"LOGIN\"]\n user_data[\"login\"].append(sub_login.shape[0])\n\n # Sort users based on total jobs in decreasing order\n sort_ids = np.argsort(-np.array(user_data[\"total\"]))\n user_data[\"user\"] = [user_data[\"user\"][i] for i in sort_ids]\n user_data[\"total\"] = [user_data[\"total\"][i] for i in sort_ids]\n user_data[\"run\"] = [user_data[\"run\"][i] for i in sort_ids]\n user_data[\"wait\"] = [user_data[\"wait\"][i] for i in sort_ids]\n user_data[\"login\"] = [user_data[\"login\"][i] for i in sort_ids]\n return user_data, job_df\n\n def get_queue_data(self, job_df: pd.DataFrame):\n \"\"\"Get jobs running on different SGE queues.\"\"\"\n host_data = {\"host_id\": [], \"total\": [], \"run\": [], \"login\": []}\n job_df = job_df[job_df.status != \"PD\"]\n unique_queues = job_df.queue.unique().tolist()\n unique_queues.sort(key=natural_keys)\n\n for h_id in unique_queues:\n sub_df = job_df.loc[job_df[\"queue\"] == h_id]\n host_data[\"host_id\"].append(h_id)\n host_data[\"total\"].append(sub_df.shape[0])\n sub_run = sub_df.loc[sub_df[\"status\"] == \"R\"]\n host_data[\"run\"].append(sub_run.shape[0])\n sub_login = sub_df.loc[sub_df[\"status\"] == \"LOGIN\"]\n host_data[\"login\"].append(sub_login.shape[0])\n return host_data\n\n def get_util_data(self):\n \"\"\"Get memory and CPU utilisation for specific SGE queue.\"\"\"\n processes = sp.check_output([\"qhost\"])\n all_node_infos = processes.split(b\"\\n\")[1:-1]\n all_node_infos = [j.decode() for j in all_node_infos]\n\n total_cores, used_cores, total_mem, used_mem = 0, 0, 0, 0\n for n_info in all_node_infos:\n node_clean = n_info.split()\n try:\n # Cores in threads and memory in GB\n total_cores += int(node_clean[2])\n used_cores += float(node_clean[2]) * float(node_clean[6])\n total_mem += float(node_clean[-4][:-1])\n # Total memory - free memory\n used_mem += float(node_clean[-2][:-1])\n except Exception:\n pass\n util_data = {\n \"cores\": total_cores,\n \"cores_util\": used_cores,\n \"mem\": total_mem,\n \"mem_util\": used_mem,\n \"time_date\": datetime.now().strftime(\"%m/%d/%y\"),\n \"time_hour\": datetime.now().strftime(\"%H:%M:%S\"),\n }\n return util_data\n\n def get_node_data(self, job_df: pd.DataFrame):\n \"\"\"Get jobs running on different SGE cluster nodes.\"\"\"\n host_data = {\"host_id\": [], \"total\": [], \"run\": [], \"login\": []}\n job_df = job_df[job_df.status != \"PD\"]\n unique_nodes = job_df.node.unique().tolist()\n unique_nodes.sort(key=natural_keys)\n\n for h_id in unique_nodes:\n sub_df = job_df.loc[job_df[\"node\"] == h_id]\n host_data[\"host_id\"].append(h_id)\n host_data[\"total\"].append(sub_df.shape[0])\n sub_run = sub_df.loc[sub_df[\"status\"] == \"R\"]\n host_data[\"run\"].append(sub_run.shape[0])\n sub_login = sub_df.loc[sub_df[\"status\"] == \"LOGIN\"]\n host_data[\"login\"].append(sub_login.shape[0])\n return host_data\n"
},
{
"path": "mle_monitor/resource/slurm.py",
"content": "from datetime import datetime\nimport subprocess as sp\nimport pandas as pd\nimport numpy as np\nfrom typing import Union\nfrom ..utils import natural_keys\n\n\nclass SlurmResource(object):\n def __init__(self, monitor_config: Union[dict, None]):\n self.resource_name = \"slurm-cluster\"\n self.monitor_config = monitor_config\n\n def monitor(self):\n \"\"\"Helper to get all utilisation data for resource.\"\"\"\n user_data, job_df = self.get_user_data()\n host_data = self.get_partition_data(job_df)\n node_data = self.get_node_data(job_df)\n util_data = self.get_util_data()\n return user_data, host_data, util_data, node_data\n\n def get_user_data(self):\n \"\"\"Get jobs scheduled by Slurm cluster users.\"\"\"\n user_data = {\"user\": [], \"total\": [], \"run\": [], \"wait\": [], \"login\": []}\n\n # Get squeue output in detailed form\n processes = sp.check_output(\n [\n \"squeue\",\n \"-o\",\n '\"%.20P %.20u %.2t %.10M %.6D %C %m %N\"',\n \"-p\",\n (\",\").join(self.monitor_config[\"partitions\"]),\n ]\n )\n all_job_infos = processes.split(b\"\\n\")[1:-1]\n all_job_infos = [j.decode() for j in all_job_infos]\n\n job_df = {\n \"user\": [],\n \"partition\": [],\n \"status\": [],\n \"node\": [],\n \"run_time\": [],\n \"num_cores\": [],\n \"min_memory\": [],\n }\n # Loop over jobs and extract relevant data into dataframe\n for job in all_job_infos:\n job_clean = job.split()[1:]\n job_df[\"user\"].append(job_clean[1])\n job_df[\"partition\"].append(job_clean[0])\n job_df[\"status\"].append(job_clean[2])\n job_df[\"run_time\"].append(job_clean[3])\n job_df[\"node\"].append(job_clean[7][:-1])\n job_df[\"num_cores\"].append(job_clean[5])\n job_df[\"min_memory\"].append(job_clean[6])\n job_df = pd.DataFrame(job_df)\n\n # Loop over unique users and construct data to show\n unique_users = job_df.user.unique().tolist()\n for u_id in unique_users:\n sub_df = job_df.loc[job_df[\"user\"] == u_id]\n user_data[\"user\"].append(u_id)\n user_data[\"total\"].append(sub_df.shape[0])\n sub_run = sub_df.loc[sub_df[\"status\"] == \"R\"]\n user_data[\"run\"].append(sub_run.shape[0])\n sub_wait = sub_df.loc[sub_df[\"status\"] == \"PD\"]\n # \"CG\" is completing status\n user_data[\"wait\"].append(sub_wait.shape[0])\n user_data[\"login\"].append(0)\n\n # Sort users based on total jobs in decreasing order\n sort_ids = np.argsort(-np.array(user_data[\"total\"]))\n user_data[\"user\"] = [user_data[\"user\"][i] for i in sort_ids]\n user_data[\"total\"] = [user_data[\"total\"][i] for i in sort_ids]\n user_data[\"run\"] = [user_data[\"run\"][i] for i in sort_ids]\n user_data[\"wait\"] = [user_data[\"wait\"][i] for i in sort_ids]\n user_data[\"login\"] = [user_data[\"login\"][i] for i in sort_ids]\n return user_data, job_df\n\n def get_partition_data(self, job_df: pd.DataFrame):\n \"\"\"Get jobs running on different Slurm cluster partitions.\"\"\"\n host_data = {\"host_id\": [], \"total\": [], \"run\": [], \"login\": []}\n\n unique_partitions = job_df.partition.unique().tolist()\n unique_partitions.sort(key=natural_keys)\n for h_id in unique_partitions:\n sub_df = job_df.loc[job_df[\"partition\"] == h_id]\n host_data[\"host_id\"].append(h_id)\n host_data[\"total\"].append(sub_df.shape[0])\n sub_run = sub_df.loc[sub_df[\"status\"] == \"R\"]\n host_data[\"run\"].append(sub_run.shape[0])\n host_data[\"login\"].append(0)\n return host_data\n\n def get_util_data(self):\n \"\"\"Get memory and CPU utilisation for specific slurm partition.\"\"\"\n # Get squeue output in detailed form\n processes = sp.check_output([\"sinfo\", \"--Node\", \"-o\", '\"%.20P %N %c %O %m %e\"'])\n all_node_infos = processes.split(b\"\\n\")[1:-1]\n all_node_infos = [j.decode() for j in all_node_infos]\n\n total_cores, used_cores, total_mem, used_mem = 0, 0, 0, 0\n for n_info in all_node_infos:\n node_clean = n_info.split()[1:]\n try:\n # Cores in threads and memory in GB\n total_cores += int(node_clean[2])\n used_cores += float(node_clean[2]) * float(node_clean[3]) / 100\n total_mem += float(node_clean[4]) / 1000\n # Total memory - free memory\n used_mem += (\n float(node_clean[4]) / 1000 - float(node_clean[5][:-1]) / 1000\n )\n except Exception:\n pass\n\n util_data = {\n \"cores\": total_cores,\n \"cores_util\": used_cores,\n \"mem\": total_mem,\n \"mem_util\": used_mem,\n \"time_date\": datetime.now().strftime(\"%m/%d/%y\"),\n \"time_hour\": datetime.now().strftime(\"%H:%M:%S\"),\n }\n return util_data\n\n def get_node_data(self, job_df: pd.DataFrame):\n \"\"\"Get jobs running on different Slurm cluster nodes.\"\"\"\n host_data = {\"host_id\": [], \"total\": [], \"run\": [], \"login\": []}\n\n unique_nodes = job_df.node.unique().tolist()\n unique_nodes.sort(key=natural_keys)\n for h_id in unique_nodes:\n sub_df = job_df.loc[job_df[\"node\"] == h_id]\n host_data[\"host_id\"].append(h_id)\n host_data[\"total\"].append(sub_df.shape[0])\n sub_run = sub_df.loc[sub_df[\"status\"] == \"R\"]\n host_data[\"run\"].append(sub_run.shape[0])\n host_data[\"login\"].append(0)\n return host_data\n\n\n# squeue -p partition_name\n# sacct -j job_id (get resource!)\n"
},
{
"path": "mle_monitor/utils/__init__.py",
"content": "from .tracker import Tracker\nfrom .helpers import load_json_config, load_yaml_config, natural_keys, setup_logger\nfrom .gcs_zip import send_gcloud_zip, get_gcloud_zip\n\n\n__all__ = [\n \"Tracker\",\n \"load_json_config\",\n \"load_yaml_config\",\n \"natural_keys\",\n \"setup_logger\",\n \"send_gcloud_zip\",\n \"get_gcloud_zip\",\n]\n"
},
{
"path": "mle_monitor/utils/gcs_zip.py",
"content": "import os\nimport glob\nimport zipfile\nfrom typing import Union\nfrom .helpers import setup_logger\n\n\ndef send_dir_gcp(\n cloud_settings: dict,\n local_dir: Union[str, None] = None,\n number_of_connect_tries: int = 5,\n):\n \"\"\"Send entire dir (recursively) to Google Cloud Storage Bucket.\"\"\"\n try:\n from google.cloud import storage\n\n except ImportError:\n raise ImportError(\n \"You need to install `google-cloud-storage` to use GCP buckets.\"\n )\n logger = setup_logger()\n for i in range(number_of_connect_tries):\n try:\n client = storage.Client(cloud_settings[\"project_name\"])\n bucket = client.get_bucket(\n cloud_settings[\"bucket_name\"], timeout=20\n )\n except Exception:\n logger.info(\n f\"Attempt {i+1}/{number_of_connect_tries}\"\n \" - Failed sending to GCloud Storage\"\n )\n\n def upload_single_file(local_path, gcs_path, bucket):\n # Recursively upload the folder structure\n if os.path.isdir(local_path):\n for local_file in glob.glob(os.path.join(local_path, \"**\")):\n if not os.path.isfile(local_file):\n upload_single_file(\n local_file,\n os.path.join(gcs_path, os.path.basename(local_file)),\n bucket,\n )\n else:\n remote_path = os.path.join(\n gcs_path, local_file[1 + len(local_path) :]\n )\n blob = bucket.blob(remote_path)\n blob.upload_from_filename(local_file)\n # Only upload single file - e.g. zip compressed experiment\n else:\n remote_path = gcs_path\n blob = bucket.blob(remote_path)\n blob.upload_from_filename(local_path)\n\n if local_dir is None:\n local_dir = os.getcwd()\n upload_single_file(local_dir, cloud_settings[\"remote_dir\"], bucket)\n\n\ndef copy_dir_gcp(\n cloud_settings: dict,\n remote_dir: str,\n local_dir: Union[str, None] = None,\n number_of_connect_tries: int = 5,\n):\n \"\"\"Download entire dir (recursively) from Google Cloud Storage Bucket.\"\"\"\n try:\n from google.cloud import storage\n\n except ImportError:\n raise ImportError(\n \"You need to install `google-cloud-storage` to use GCP buckets.\"\n )\n logger = setup_logger()\n\n for i in range(number_of_connect_tries):\n try:\n client = storage.Client(cloud_settings[\"project_name\"])\n bucket = client.get_bucket(\n cloud_settings[\"bucket_name\"], timeout=20\n )\n except Exception:\n logger.info(\n f\"Attempt {i+1}/{number_of_connect_tries}\"\n \" - Failed sending to GCloud Storage\"\n )\n\n blobs = bucket.list_blobs(prefix=remote_dir) # Get list of files\n blobs = list(blobs)\n\n if local_dir is None:\n path = os.path.normpath(remote_dir)\n local_dir = path.split(os.sep)[-1]\n\n # Recursively download the folder structure\n if len(blobs) > 1:\n local_path = os.path.expanduser(local_dir)\n for blob in blobs:\n filename = blob.name[len(remote_dir) :]\n local_f_path = os.path.join(local_path, filename)\n dir_path = os.path.dirname(local_f_path)\n\n if not os.path.exists(dir_path):\n try:\n os.makedirs(dir_path)\n except Exception:\n pass\n temp_file = local_path + filename\n temp_dir = os.path.split(temp_file)[0]\n if not os.path.exists(temp_dir):\n os.makedirs(temp_dir)\n blob.download_to_filename(temp_file)\n # Only download single file - e.g. zip compressed experiment\n else:\n for blob in blobs:\n filename = blob.name.split(\"/\")[1]\n blob.download_to_filename(filename)\n\n\ndef zipdir(path: str, zip_fname: str):\n \"\"\"Zip a directory to upload afterwards to GCloud Storage.\"\"\"\n # ziph is zipfile handle\n ziph = zipfile.ZipFile(zip_fname, \"w\", zipfile.ZIP_DEFLATED)\n # Get rid of redundant part of path\n prefix_len = len(path)\n for root, dirs, files in os.walk(path):\n for file in files:\n ziph.write(\n os.path.join(root, file),\n os.path.join(root[prefix_len + 1 :], file),\n )\n ziph.close()\n\n\ndef send_gcloud_zip(\n cloud_settings: dict,\n local_fname: str,\n local_zip_fname: str,\n delete_after_upload: bool = True,\n):\n \"\"\"Zip & upload experiment dir to Gcloud storage.\"\"\"\n # 1. Get experiment hash from the protocol db\n gcloud_hash_fname = \"experiments/\" + local_zip_fname\n\n # 2. Zip compress the experiment\n zipdir(local_fname, local_zip_fname)\n\n # 3. Upload the zip file to the GCS bucket\n cloud_settings[\"remote_dir\"] = gcloud_hash_fname\n send_dir_gcp(cloud_settings, local_zip_fname)\n\n # 4. Delete the .zip file & the folder if desired\n if delete_after_upload:\n os.remove(local_zip_fname)\n\n\ndef get_gcloud_zip(\n cloud_settings: dict,\n hash_to_store: str,\n experiment_id: str,\n local_dir_name: Union[None, str] = None,\n):\n \"\"\"Download zipped experiment from GCS. Unpack & clean up.\"\"\"\n logger = setup_logger()\n # Get unique hash id & download the experiment results folder\n local_hash_fname = hash_to_store + \".zip\"\n gcloud_hash_fname = \"experiments/\" + local_hash_fname\n copy_dir_gcp(cloud_settings, gcloud_hash_fname)\n\n # Unzip the retrieved file\n if local_dir_name is None:\n with zipfile.ZipFile(local_hash_fname, \"r\") as zip_ref:\n zip_ref.extractall(experiment_id)\n else:\n with zipfile.ZipFile(local_hash_fname, \"r\") as zip_ref:\n zip_ref.extractall(local_dir_name)\n\n # Delete the zip file\n os.remove(local_hash_fname)\n\n # Goodbye message if successful\n logger.info(f\"Successfully retrieved {experiment_id} - from GCS\")\n logger.info(f\"Remote Path: {gcloud_hash_fname}\")\n return\n"
},
{
"path": "mle_monitor/utils/helpers.py",
"content": "import yaml\nimport commentjson\nfrom dotmap import DotMap\nimport re\nimport logging\nfrom rich.logging import RichHandler\n\n\ndef load_yaml_config(config_fname: str, return_dotmap: bool = False):\n \"\"\"Load in YAML config file.\"\"\"\n with open(config_fname) as file:\n yaml_config = yaml.load(file, Loader=yaml.FullLoader)\n if not return_dotmap:\n return yaml_config\n else:\n return DotMap(yaml_config)\n\n\ndef load_json_config(config_fname: str, return_dotmap: bool = False):\n \"\"\"Load in JSON config file.\"\"\"\n json_config = commentjson.loads(open(config_fname, \"r\").read())\n if not return_dotmap:\n return json_config\n else:\n return DotMap(json_config)\n\n\ndef atoi(text):\n return int(text) if text.isdigit() else text\n\n\ndef natural_keys(text):\n \"\"\"\n alist.sort(key=natural_keys) sorts in human order\n http://nedbatchelder.com/blog/200712/human_sorting.html\n (See Toothy's implementation in the comments)\n \"\"\"\n return [atoi(c) for c in re.split(r\"(\\d+)\", text)]\n\n\ndef setup_logger(logging_level: int = logging.INFO):\n logging.basicConfig(\n level=logging_level,\n format=\"%(message)s\",\n datefmt=\"[%X]\",\n handlers=[RichHandler()],\n )\n logger = logging.getLogger()\n logger.setLevel(logging_level)\n return logger\n"
},
{
"path": "mle_monitor/utils/tracker.py",
"content": "import os\nimport numpy as np\n\n\nclass Tracker(object):\n def __init__(self, fname=\".mle_tracker.npy\"):\n \"\"\"MLE Tracker for Resource Utilization & Running Experiments.\"\"\"\n self.fname = os.path.join(os.path.expanduser(\"~\"), fname)\n # Storage limit & reload previous stored data\n self.limit = 100000\n self.load()\n\n def update(self, util_data: dict, save: bool = True):\n # Add utilisation data to storage dictionaries\n self.times_date.append(util_data[\"time_date\"])\n self.times_hour.append(util_data[\"time_hour\"])\n self.mem_util.append(util_data[\"mem_util\"] / util_data[\"mem\"])\n self.cpu_util.append(util_data[\"cores_util\"] / util_data[\"cores\"])\n self.moving_window()\n self.save()\n return {\n \"times_date\": self.times_date,\n \"times_hour\": self.times_hour,\n \"rel_mem_util\": self.mem_util,\n \"rel_cpu_util\": self.cpu_util,\n }\n\n def moving_window(self):\n \"\"\"Truncate/Limit memory to approx. last 27 hours.\"\"\"\n self.times_date = self.times_date[: self.limit]\n self.times_hour = self.times_hour[: self.limit]\n self.mem_util = self.mem_util[: self.limit]\n self.cpu_util = self.cpu_util[: self.limit]\n\n def load(self):\n \"\"\"Creates a hidden file storing usage time series data.\"\"\"\n try:\n data = np.load(self.fname)\n self.mem_util = data[:, 0].astype(float).tolist()\n self.cpu_util = data[:, 1].astype(float).tolist()\n self.times_date = data[:, 2].tolist()\n self.times_hour = data[:, 3].tolist()\n except Exception:\n # Start storing utilisation history\n self.mem_util = []\n self.cpu_util = []\n self.times_date = []\n self.times_hour = []\n\n def save(self):\n \"\"\"Save recent usage time series data.\"\"\"\n stacked = np.stack(\n [self.mem_util, self.cpu_util, self.times_date, self.times_hour], axis=1\n )\n np.save(self.fname, stacked)\n"
},
{
"path": "requirements/requirements-examples.txt",
"content": "mle-logging\nmle-hyperopt"
},
{
"path": "requirements/requirements-test.txt",
"content": "psutil\ngputil"
},
{
"path": "requirements/requirements.txt",
"content": "pyyaml\ncommentjson\ndotmap\nnumpy\npandas\nrich\npickledb\nplotext==4.0.0\nPillow\n"
},
{
"path": "setup.py",
"content": "try:\n from setuptools import setup, find_packages\nexcept ImportError:\n from distutils.core import setup, find_packages\n\nimport re\nimport os\nfrom typing import List\n\nCURRENT_DIR = os.path.abspath(os.path.dirname(__file__))\n\nwith open(os.path.join(CURRENT_DIR, \"README.md\"), encoding=\"utf-8\") as f:\n long_description = f.read()\n\n\ndef parse_requirements(path: str) -> List[str]:\n with open(os.path.join(CURRENT_DIR, path)) as f:\n return [\n line.rstrip()\n for line in f\n if not (line.isspace() or line.startswith(\"#\"))\n ]\n\n\nVERSIONFILE = \"mle_monitor/_version.py\"\nverstrline = open(VERSIONFILE, \"rt\").read()\nVSRE = r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\"\nmo = re.search(VSRE, verstrline, re.M)\nif mo:\n verstr = mo.group(1)\nelse:\n raise RuntimeError(\"Unable to find version string in %s.\" % (VERSIONFILE,))\ngit_tar = f\"https://github.com/mle-infrastructure/mle-monitor/archive/v{verstr}.tar.gz\"\n\n\nsetup(\n name=\"mle_monitor\",\n version=verstr,\n author=\"Robert Tjarko Lange\",\n author_email=\"robertlange0@gmail.com\",\n description=\"Machine Learning Experiment Resource Monitoring\",\n long_description=long_description,\n long_description_content_type=\"text/markdown\",\n url=\"https://github.com/mle-infrastructure/mle-monitor\",\n download_url=git_tar,\n classifiers=[\n \"Programming Language :: Python :: 3.6\",\n \"Programming Language :: Python :: 3.7\",\n \"Programming Language :: Python :: 3.8\",\n \"Programming Language :: Python :: 3.9\",\n \"License :: OSI Approved :: MIT License\",\n \"Operating System :: OS Independent\",\n \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n ],\n packages=find_packages(),\n include_package_data=True,\n zip_safe=False,\n platforms=\"any\",\n python_requires=\">=3.6\",\n install_requires=parse_requirements(\n os.path.join(CURRENT_DIR, \"requirements\", \"requirements.txt\")\n ),\n tests_require=parse_requirements(\n os.path.join(CURRENT_DIR, \"requirements\", \"requirements-test.txt\")\n ),\n extras_require={\n \"examples\": parse_requirements(\n os.path.join(\n CURRENT_DIR, \"requirements\", \"requirements-examples.txt\"\n )\n ),\n \"full\": [\"psutil\", \"gputil\", \"google-cloud-storage\", \"GitPython\"],\n },\n)\n"
},
{
"path": "tests/fixtures/base_config.json",
"content": "{\n\"train_config\": {\"lrate\": 0.1},\n\"model_config\": {\"num_layers\": 5},\n\"log_config\": {\"time_to_track\": [\"step_counter\"],\n \"what_to_track\": [\"loss\"],\n \"time_to_print\": [\"step_counter\"],\n \"what_to_print\": [\"loss\"],\n \"print_every_k_updates\": 10,\n \"overwrite_experiment_dir\": 1}\n}\n"
},
{
"path": "tests/test_dashboard.py",
"content": "from mle_monitor import MLEProtocol, MLEResource, MLEDashboard\n\n\ndef test_dashboard():\n # Test data collection and layout generation\n resource = MLEResource(resource_name=\"local\")\n protocol = MLEProtocol(protocol_fname=\"mle_protocol.db\")\n dashboard = MLEDashboard(protocol, resource)\n dashboard.snapshot()\n"
},
{
"path": "tests/test_protocol.py",
"content": "from mle_monitor import MLEProtocol\n\nmeta_data = {\n \"purpose\": \"Test MLEProtocol\",\n \"project_name\": \"MNIST\",\n \"exec_resource\": \"local\",\n \"experiment_dir\": \"log_dir\",\n \"experiment_type\": \"hyperparameter-search\",\n \"base_fname\": \"main.py\",\n \"config_fname\": \"tests/fixtures/base_config.json\",\n \"num_seeds\": 5,\n \"num_total_jobs\": 10,\n \"num_jobs_per_batch\": 5,\n \"num_job_batches\": 2,\n \"time_per_job\": \"00:05:00\", # days-hours-minutes\n \"num_cpus\": 2,\n \"num_gpus\": 1,\n}\n\n\ndef test_add_protocol():\n # Add experiment to new protocol and add data\n protocol = MLEProtocol(protocol_fname=\"mle_protocol.db\")\n e_id = protocol.add(meta_data, save=False)\n proto_data = protocol.get(e_id)\n for k, v in meta_data.items():\n assert proto_data[k] == v\n return\n\n\ndef test_load_protocol():\n # Reload database - assert correctness of data\n protocol = MLEProtocol(protocol_fname=\"tests/fixtures/mle_protocol_test.db\")\n last_data = protocol.get()\n for k, v in meta_data.items():\n if k not in [\"config_fname\", \"purpose\"]:\n assert last_data[k] == v\n\n # Check adding of new data\n e_id = protocol.add(meta_data, save=False)\n proto_data = protocol.get(e_id)\n for k, v in meta_data.items():\n assert proto_data[k] == v\n return\n\n\ndef test_update_delete_abort_protocol():\n # Change some entry of DB store and check it\n protocol = MLEProtocol(protocol_fname=\"mle_protocol.db\")\n e_id = protocol.add(meta_data, save=False)\n # Update some element in the database\n protocol.update(e_id, \"exec_resource\", \"slurm-cluster\", save=False)\n assert protocol.get(e_id, \"exec_resource\") == \"slurm-cluster\"\n # Abort the experiment - changes status\n protocol.abort(e_id, save=False)\n assert protocol.status(e_id) == \"aborted\"\n return\n\n\ndef test_monitor_protocol():\n # Check that all required keys are in collected data\n protocol = MLEProtocol(protocol_fname=\"mle_protocol.db\")\n _ = protocol.add(meta_data, save=False)\n # Get the monitoring data - used later in dashboard\n data = protocol.monitor()\n total_keys = [\n \"total\",\n \"run\",\n \"done\",\n \"aborted\",\n \"sge\",\n \"slurm\",\n \"gcp\",\n \"local\",\n \"report_gen\",\n \"gcs_stored\",\n \"retrieved\",\n ]\n for k in total_keys:\n assert k in data[\"total_data\"].keys()\n last_keys = [\"e_id\", \"e_dir\", \"e_type\", \"e_script\", \"e_config\", \"report_gen\"]\n for k in last_keys:\n assert k in data[\"last_data\"].keys()\n time_keys = [\n \"total_jobs\",\n \"total_batches\",\n \"jobs_per_batch\",\n \"time_per_batch\",\n \"start_time\",\n \"stop_time\",\n \"duration\",\n ]\n for k in time_keys:\n assert k in data[\"time_data\"].keys()\n return\n"
},
{
"path": "tests/test_resource.py",
"content": "from mle_monitor import MLEResource\n\n\ndef test_resource():\n # Instantiate local resource and get usage data\n resource = MLEResource(resource_name=\"local\")\n resource_data = resource.monitor()\n assert \"resource_name\" in resource_data.keys()\n assert \"user_data\" in resource_data.keys()\n assert \"host_data\" in resource_data.keys()\n assert \"util_data\" in resource_data.keys()\n return\n"
}
]![](\"https://github.com/mle-infrastructure/mle-monitor/blob/main/docs/logo_transparent.png?raw=true\"){kind=spacer}
![\"drawing\"](\"https://github.com/mle-infrastructure/mle-monitor/blob/main/docs/monitor-promo-gif.gif?raw=true\")
\n\n## `MLEProtocol`: Keeping Track of Your Experiments 📝\n\n```python\nfrom mle_monitor import MLEProtocol\n\n# Load protocol database or create new one -> print summary\nprotocol_db = MLEProtocol(\"mle_protocol.db\", verbose=False)\nprotocol_db.summary(tail=10, verbose=True)\n\n# Draft data to store in protocol & add it to the protocol\nmeta_data = {\n \"purpose\": \"Grid search\", # Purpose of experiment\n \"project_name\": \"MNIST\", # Project name of experiment\n \"experiment_type\": \"hyperparameter-search\", # Type of experiment\n \"experiment_dir\": \"experiments/logs\", # Experiment directory\n \"num_total_jobs\": 10, # Number of total jobs to run\n ...\n}\nnew_experiment_id = protocol_db.add(meta_data)\n\n# ... train your 10 (pseudo) networks/complete respective jobs\nfor i in range(10):\n protocol_db.update_progress_bar(new_experiment_id)\n\n# Wrap up an experiment (store completion time, etc.)\nprotocol_db.complete(new_experiment_id)\n```\n\nThe meta data can contain the following keys:\n\n| Search Type | Description | Default |\n|----------------------- | ----------- | --------------- |\n| `purpose` | Purpose of experiment | `'None provided'` |\n| `project_name` | Project name of experiment | `'default'` |\n| `exec_resource` | Resource jobs are run on | `'local'` |\n| `experiment_dir` | Experiment log storage directory | `'experiments'` |\n| `experiment_type` | Type of experiment to run | `'single'` |\n| `base_fname` | Main code script to execute | `'main.py'` |\n| `config_fname` | Config file path of experiment | `'base_config.yaml'` |\n| `num_seeds` | Number of evaluations seeds | 1 |\n| `num_total_jobs` | Number of total jobs to run | 1 |\n| `num_job_batches` | Number of jobs in single batch | 1 |\n| `num_jobs_per_batch` | Number of sequential job batches | 1 |\n| `time_per_job` | Expected duration: days-hours-minutes | `'00:01:00'` |\n| `num_cpus` | Number of CPUs used in job | 1 |\n| `num_gpus` | Number of GPUs used in job | 0 |\n\nAdditionally you can synchronize the protocol with a Google Cloud Storage (GCS) bucket by providing `cloud_settings`. In this case also the results stored in `experiment_dir` will be uploaded to the GCS bucket, when you call `protocol.complete()`.\n\n\n```python\n# Define GCS settings - requires 'GOOGLE_APPLICATION_CREDENTIALS' env var.\ncloud_settings = {\n \"project_name\": \"mle-toolbox\", # GCP project name\n \"bucket_name\": \"mle-protocol\", # GCS bucket name\n \"use_protocol_sync\": True, # Whether to sync the protocol to GCS\n \"use_results_storage\": True, # Whether to sync experiment_dir to GCS\n}\nprotocol_db = MLEProtocol(\"mle_protocol.db\", cloud_settings, verbose=True)\n```\n\n## The `MLEResource`: Keeping Track of Your Resources 📉\n\n#### On Your Local Machine\n```python\nfrom mle_monitor import MLEResource\n\n# Instantiate local resource and get usage data\nresource = MLEResource(resource_name=\"local\")\nresource_data = resource.monitor()\n```\n\n#### On a Slurm Cluster\n\n```python\nresource = MLEResource(\n resource_name=\"slurm-cluster\",\n monitor_config={\"partitions\": [\"![\\\"drawing\\\"](\\\"https://github.com/mle-infrastructure/mle-monitor/blob/main/docs/monitor-promo-gif.gif?raw=true\\\")
\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"id\": \"b1ff5d35-b29e-4386-b423-9c83ad1ae169\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"%load_ext autoreload\\n\",\n \"%autoreload 2\\n\",\n \"%config InlineBackend.figure_format = 'retina'\\n\",\n \"\\n\",\n \"try:\\n\",\n \" import mle_monitor\\n\",\n \"except:\\n\",\n \" !pip install -q mle-monitor\\n\",\n \" import mle_monitor\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"9cf5bc30-a187-4936-a57c-3023ff6b3f2a\",\n \"metadata\": {},\n \"source\": [\n \"`mle-monitor` comes with three core functionalities:\\n\",\n \"\\n\",\n \"- **`MLEProtocol`**: A composable protocol database API for ML experiments.\\n\",\n \"- **`MLEResource`**: A tool for obtaining server/cluster usage statistics.\\n\",\n \"- **`MLEDashboard`**: A dashboard visualizing resource usage & experiment protocol.\\n\",\n \"\\n\",\n \"![\\\"drawing\\\"](\\\"https://github.com/mle-infrastructure/mle-monitor/blob/main/docs/mle_monitor_structure.png?raw=true\\\")
\\n\",\n \"\\n\",\n \"Finally, `mle-monitor` is part of the [`mle-infrastructure`](https://github.com/mle-infrastructure) and comes with a set of handy built-in synergies. We will wrap-up by outlining a full workflow of using the protocol together with a random search experiment using [`mle-hyperopt`](https://github.com/mle-infrastructure/mle-hyperopt), [`mle-scheduler`](https://github.com/mle-infrastructure/mle-monitor) and [`mle-logging`](https://github.com/mle-infrastructure/mle-logging).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"id\": \"a66df14c-b920-4926-a8c1-37c42c63e7f2\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Check if code is run in Colab: If so -- download configs from repo\\n\",\n \"try:\\n\",\n \" import google.colab\\n\",\n \" IN_COLAB = True\\n\",\n \" !wget -q https://raw.githubusercontent.com/mle-infrastructure/mle-monitor/main/examples/train.py\\n\",\n \" !wget -q https://raw.githubusercontent.com/mle-infrastructure/mle-monitor/main/examples/base_config.json\\n\",\n \"except:\\n\",\n \" IN_COLAB = False\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"255b0522-6c4b-440d-8dca-345ba9e8a461\",\n \"metadata\": {},\n \"source\": [\n \"# Experiment Management with `MLEProtocol` 📝\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"id\": \"44a221b5-758b-491b-b206-65c3bddeeddb\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from mle_monitor import MLEProtocol\\n\",\n \"\\n\",\n \"# Load the protocol from a local file (create new if it doesn't exist yet)\\n\",\n \"protocol = MLEProtocol(protocol_fname=\\\"mle_protocol.db\\\", verbose=True)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"ef2d1797-0251-495f-8976-47a24756e79c\",\n \"metadata\": {},\n \"source\": [\n \"In order to add a new experiment to the protocol database you have to provide a dictionary containing the experiment meta data:\\n\",\n \"\\n\",\n \"| Search Type | Description | Default |\\n\",\n \"|----------------------- | ----------- | --------------- |\\n\",\n \"| `purpose` | Purpose of experiment | `'None provided'` |\\n\",\n \"| `project_name` | Project name of experiment | `'default'` |\\n\",\n \"| `exec_resource` | Resource jobs are run on | `'local'` |\\n\",\n \"| `experiment_dir` | Experiment log storage directory | `'experiments'` |\\n\",\n \"| `experiment_type` | Type of experiment to run | `'single'` |\\n\",\n \"| `base_fname` | Main code script to execute | `'main.py'` |\\n\",\n \"| `config_fname` | Config file path of experiment | `'base_config.yaml'` |\\n\",\n \"| `num_seeds` | Number of evaluations seeds | 1 |\\n\",\n \"| `num_total_jobs` | Number of total jobs to run | 1 |\\n\",\n \"| `num_job_batches` | Number of jobs in single batch | 1 |\\n\",\n \"| `num_jobs_per_batch` | Number of sequential job batches | 1 |\\n\",\n \"| `time_per_job` | Expected duration: days-hours-minutes | `'00:01:00'` |\\n\",\n \"| `num_cpus` | Number of CPUs used in job | 1 |\\n\",\n \"| `num_gpus` | Number of GPUs used in job | 0 |\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"id\": \"d65bcced-fa6f-4865-939f-b8d35203b05f\",\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"