{
"cells": [
{
"cell_type": "markdown",
"id": "41eb6323",
"metadata": {
"collapsed": true,
"customInput": null,
"originalKey": "ac61b043-8ebf-43b9-9fa5-ed9a42a184ce",
"papermill": {
"duration": 0.007252,
"end_time": "2023-12-09T18:40:47.585361",
"exception": false,
"start_time": "2023-12-09T18:40:47.578109",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"# Tune a CNN on MNIST\n",
"\n",
"This tutorial walks through using Ax to tune two hyperparameters (learning rate and momentum) for a PyTorch CNN on the MNIST dataset trained using SGD with momentum."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "34eac0d3",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:40:47.599198Z",
"iopub.status.busy": "2023-12-09T18:40:47.598712Z",
"iopub.status.idle": "2023-12-09T18:40:50.785735Z",
"shell.execute_reply": "2023-12-09T18:40:50.784772Z"
},
"executionStartTime": 1690415246079,
"executionStopTime": 1690415266324,
"originalKey": "c2b37f0f-3644-4367-912f-f775082f6676",
"papermill": {
"duration": 3.211351,
"end_time": "2023-12-09T18:40:50.802975",
"exception": false,
"start_time": "2023-12-09T18:40:47.591624",
"status": "completed"
},
"requestMsgId": "0b481630-f0f4-436a-a205-a25aa163a364",
"showInput": true,
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:40:50] ax.utils.notebook.plotting: Injecting Plotly library into cell. Do not overwrite or delete cell.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:40:50] ax.utils.notebook.plotting: Please see\n",
" (https://ax.dev/tutorials/visualizations.html#Fix-for-plots-that-are-not-rendering)\n",
" if visualizations are not rendering.\n"
]
},
{
"data": {
"text/html": [
" \n",
" "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import torch\n",
"\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
"from ax.service.ax_client import AxClient, ObjectiveProperties\n",
"from ax.service.utils.report_utils import exp_to_df\n",
"from ax.utils.notebook.plotting import init_notebook_plotting, render\n",
"from ax.utils.tutorials.cnn_utils import evaluate, load_mnist, train\n",
"from torch._tensor import Tensor\n",
"from torch.utils.data import DataLoader\n",
"\n",
"init_notebook_plotting()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "8fa9e641",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:40:50.884491Z",
"iopub.status.busy": "2023-12-09T18:40:50.883919Z",
"iopub.status.idle": "2023-12-09T18:40:50.888780Z",
"shell.execute_reply": "2023-12-09T18:40:50.888131Z"
},
"executionStartTime": 1690415266521,
"executionStopTime": 1690415266529,
"originalKey": "4d0a27c4-a6ce-4b7d-97eb-1c229aabb375",
"papermill": {
"duration": 0.048216,
"end_time": "2023-12-09T18:40:50.890399",
"exception": false,
"start_time": "2023-12-09T18:40:50.842183",
"status": "completed"
},
"requestMsgId": "fd975d25-a185-4b09-a50f-7b2bcd89f93f",
"showInput": true,
"tags": []
},
"outputs": [],
"source": [
"torch.manual_seed(42)\n",
"dtype = torch.float\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
]
},
{
"cell_type": "markdown",
"id": "dfa07683",
"metadata": {
"customInput": null,
"originalKey": "10384e51-444c-4265-b56d-ad078d05d2a1",
"papermill": {
"duration": 0.039062,
"end_time": "2023-12-09T18:40:50.967987",
"exception": false,
"start_time": "2023-12-09T18:40:50.928925",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"## 1. Load MNIST data\n",
"First, we need to load the MNIST data and partition it into training, validation, and test sets.\n",
"\n",
"Note: this will download the dataset if necessary."
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "dfc658bb",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:40:51.047442Z",
"iopub.status.busy": "2023-12-09T18:40:51.046891Z",
"iopub.status.idle": "2023-12-09T18:40:52.030073Z",
"shell.execute_reply": "2023-12-09T18:40:52.029308Z"
},
"executionStartTime": 1690415266733,
"executionStopTime": 1690415266902,
"originalKey": "6f0949e2-1064-44b8-99c0-f6ce23df7c63",
"papermill": {
"duration": 1.024956,
"end_time": "2023-12-09T18:40:52.031834",
"exception": false,
"start_time": "2023-12-09T18:40:51.006878",
"status": "completed"
},
"requestMsgId": "8ce7dd21-9afb-4379-ad11-4112b4d27f8a",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
"Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
" 0%| | 0/9912422 [00:00= p2\" or \"p1 + p2 <= some_bound\".\n",
" # outcome_constraints: Optional, a list of strings of form \"constrained_metric <= some_bound\".\n",
")"
]
},
{
"cell_type": "markdown",
"id": "360d0fe0",
"metadata": {
"customInput": null,
"originalKey": "af441a83-50fd-4385-a380-d8ebc570c0e5",
"papermill": {
"duration": 0.039973,
"end_time": "2023-12-09T18:40:52.449201",
"exception": false,
"start_time": "2023-12-09T18:40:52.409228",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"## 4. Define how to evaluate trials\n"
]
},
{
"cell_type": "markdown",
"id": "a59b6b86",
"metadata": {
"customInput": null,
"originalKey": "c7630dfd-548b-408a-badf-b6abf79275e2",
"papermill": {
"duration": 0.039933,
"end_time": "2023-12-09T18:40:52.529248",
"exception": false,
"start_time": "2023-12-09T18:40:52.489315",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"First we define a simple CNN class to classify the MNIST images"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "f186de08",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:40:52.610668Z",
"iopub.status.busy": "2023-12-09T18:40:52.610383Z",
"iopub.status.idle": "2023-12-09T18:40:52.615845Z",
"shell.execute_reply": "2023-12-09T18:40:52.615225Z"
},
"executionStartTime": 1690415267282,
"executionStopTime": 1690415267286,
"originalKey": "e41fea0a-ae71-4e6f-8c0a-6eb6ae143fb0",
"papermill": {
"duration": 0.048335,
"end_time": "2023-12-09T18:40:52.617485",
"exception": false,
"start_time": "2023-12-09T18:40:52.569150",
"status": "completed"
},
"requestMsgId": "60f14ec9-eb1b-4e88-95c5-15c91f999c90",
"showInput": true,
"tags": []
},
"outputs": [],
"source": [
"class CNN(nn.Module):\n",
" \n",
" def __init__(self) -> None:\n",
" super().__init__()\n",
" self.conv1 = nn.Conv2d(1, 20, kernel_size=5, stride=1)\n",
" self.fc1 = nn.Linear(8 * 8 * 20, 64)\n",
" self.fc2 = nn.Linear(64, 10)\n",
"\n",
" def forward(self, x: Tensor) -> Tensor:\n",
" x = F.relu(self.conv1(x))\n",
" x = F.max_pool2d(x, 3, 3)\n",
" x = x.view(-1, 8 * 8 * 20)\n",
" x = F.relu(self.fc1(x))\n",
" x = self.fc2(x)\n",
" return F.log_softmax(x, dim=-1)"
]
},
{
"cell_type": "markdown",
"id": "a035ccfc",
"metadata": {
"customInput": null,
"originalKey": "8ef6bcb9-c492-4874-b8c7-a07f7e6291ad",
"papermill": {
"duration": 0.039916,
"end_time": "2023-12-09T18:40:52.697129",
"exception": false,
"start_time": "2023-12-09T18:40:52.657213",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"In this tutorial, we want to optimize classification accuracy on the validation set as a function of the learning rate and momentum. The `train_evaluate` function takes in a parameterization (set of parameter values), computes the classification accuracy, and returns that metric. "
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "ac35919b",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:40:52.779188Z",
"iopub.status.busy": "2023-12-09T18:40:52.778568Z",
"iopub.status.idle": "2023-12-09T18:40:52.783252Z",
"shell.execute_reply": "2023-12-09T18:40:52.782607Z"
},
"executionStartTime": 1690415267388,
"executionStopTime": 1690415267395,
"originalKey": "a7e4bcc4-7494-429b-bb93-7ad84d0985af",
"papermill": {
"duration": 0.047372,
"end_time": "2023-12-09T18:40:52.784645",
"exception": false,
"start_time": "2023-12-09T18:40:52.737273",
"status": "completed"
},
"requestMsgId": "5d486dbf-60cb-453d-8f24-8605f974b0a7",
"showInput": true,
"tags": []
},
"outputs": [],
"source": [
"def train_evaluate(parameterization):\n",
" \"\"\"\n",
" Train the model and then compute an evaluation metric.\n",
"\n",
" In this tutorial, the CNN utils package is doing a lot of work\n",
" under the hood:\n",
" - `train` initializes the network, defines the loss function\n",
" and optimizer, performs the training loop, and returns the\n",
" trained model.\n",
" - `evaluate` computes the accuracy of the model on the\n",
" evaluation dataset and returns the metric.\n",
"\n",
" For your use case, you can define training and evaluation functions\n",
" of your choosing.\n",
"\n",
" \"\"\"\n",
" net = CNN()\n",
" net = train(\n",
" net=net,\n",
" train_loader=train_loader,\n",
" parameters=parameterization,\n",
" dtype=dtype,\n",
" device=device,\n",
" )\n",
"\n",
" return evaluate(\n",
" net=net, \n",
" data_loader=valid_loader, \n",
" dtype=dtype, \n",
" device=device,\n",
" )\n"
]
},
{
"cell_type": "markdown",
"id": "b6d14a21",
"metadata": {
"customInput": null,
"originalKey": "9ab127a8-021f-4ec8-9f4e-f4256a2e322a",
"papermill": {
"duration": 0.03985,
"end_time": "2023-12-09T18:40:52.864459",
"exception": false,
"start_time": "2023-12-09T18:40:52.824609",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"## 5. Run optimization loop\n"
]
},
{
"cell_type": "markdown",
"id": "4262c6c0",
"metadata": {
"customInput": null,
"originalKey": "411a2fb4-e8a3-4414-bc17-09f0b5ba3e74",
"papermill": {
"duration": 0.039955,
"end_time": "2023-12-09T18:40:52.944945",
"exception": false,
"start_time": "2023-12-09T18:40:52.904990",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"First we use `attach_trial` to attach a custom trial with manually-chosen parameters. This step is optional, but we include it here to demonstrate adding manual trials and to serve as a baseline model with decent performance. "
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "3ae664f0",
"metadata": {
"customInput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:40:53.026409Z",
"iopub.status.busy": "2023-12-09T18:40:53.025895Z",
"iopub.status.idle": "2023-12-09T18:40:59.795089Z",
"shell.execute_reply": "2023-12-09T18:40:59.794478Z"
},
"executionStartTime": 1690415267533,
"executionStopTime": 1690415287786,
"originalKey": "1388ef55-5642-46ab-b297-c76a73a48aca",
"papermill": {
"duration": 6.811763,
"end_time": "2023-12-09T18:40:59.796571",
"exception": false,
"start_time": "2023-12-09T18:40:52.984808",
"status": "completed"
},
"requestMsgId": "b32a4981-ad59-46e1-b701-fa5a5f118d8b",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:40:53] ax.core.experiment: Attached custom parameterizations [{'lr': 2.6e-05, 'momentum': 0.58}] as trial 0.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:40:59] ax.service.ax_client: Completed trial 0 with data: {'accuracy': (0.841833, None)}.\n"
]
}
],
"source": [
"# Attach the trial\n",
"ax_client.attach_trial(\n",
" parameters={\"lr\": 0.000026, \"momentum\": 0.58}\n",
")\n",
"\n",
"# Get the parameters and run the trial \n",
"baseline_parameters = ax_client.get_trial_parameters(trial_index=0)\n",
"ax_client.complete_trial(trial_index=0, raw_data=train_evaluate(baseline_parameters))"
]
},
{
"cell_type": "markdown",
"id": "eb4f7716",
"metadata": {
"customInput": null,
"originalKey": "f0f886a1-c5c8-44bb-b2fd-9fa3f140357a",
"papermill": {
"duration": 0.040708,
"end_time": "2023-12-09T18:40:59.877673",
"exception": false,
"start_time": "2023-12-09T18:40:59.836965",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"Now we start the optimization loop.\n",
"\n",
"At each step, the user queries the client for a new trial then submits the evaluation of that trial back to the client.\n",
"\n",
"Note that Ax auto-selects an appropriate optimization algorithm based on the search space. For more advanced use cases that require a specific optimization algorithm, pass a `generation_strategy` argument into the `AxClient` constructor. Note that when Bayesian Optimization is used, generating new trials may take a few minutes."
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7efd735d",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:40:59.960236Z",
"iopub.status.busy": "2023-12-09T18:40:59.959925Z",
"iopub.status.idle": "2023-12-09T18:43:59.453467Z",
"shell.execute_reply": "2023-12-09T18:43:59.452755Z"
},
"executionStartTime": 1690415287908,
"executionStopTime": 1690415945107,
"originalKey": "bff5d714-1ab3-43d3-b9b3-8c3a53c81dcb",
"papermill": {
"duration": 179.536717,
"end_time": "2023-12-09T18:43:59.455020",
"exception": false,
"start_time": "2023-12-09T18:40:59.918303",
"status": "completed"
},
"requestMsgId": "a203534f-85dd-4dfa-9fa6-6aa46a0200a3",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:40:59] ax.service.ax_client: Generated new trial 1 with parameters {'lr': 0.009955, 'momentum': 0.633423}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:06] ax.service.ax_client: Completed trial 1 with data: {'accuracy': (0.100333, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:06] ax.service.ax_client: Generated new trial 2 with parameters {'lr': 5e-06, 'momentum': 0.022851}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:12] ax.service.ax_client: Completed trial 2 with data: {'accuracy': (0.318667, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:12] ax.service.ax_client: Generated new trial 3 with parameters {'lr': 7e-06, 'momentum': 0.176948}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:19] ax.service.ax_client: Completed trial 3 with data: {'accuracy': (0.4585, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:19] ax.service.ax_client: Generated new trial 4 with parameters {'lr': 8.2e-05, 'momentum': 0.90883}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:26] ax.service.ax_client: Completed trial 4 with data: {'accuracy': (0.926, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:26] ax.service.ax_client: Generated new trial 5 with parameters {'lr': 0.000302, 'momentum': 0.341904}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:32] ax.service.ax_client: Completed trial 5 with data: {'accuracy': (0.929, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:32] ax.service.ax_client: Generated new trial 6 with parameters {'lr': 0.000137, 'momentum': 0.590917}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:39] ax.service.ax_client: Completed trial 6 with data: {'accuracy': (0.92, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:39] ax.service.ax_client: Generated new trial 7 with parameters {'lr': 1e-05, 'momentum': 1.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:46] ax.service.ax_client: Completed trial 7 with data: {'accuracy': (0.860167, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:46] ax.service.ax_client: Generated new trial 8 with parameters {'lr': 0.000246, 'momentum': 0.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:53] ax.service.ax_client: Completed trial 8 with data: {'accuracy': (0.888833, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:41:53] ax.service.ax_client: Generated new trial 9 with parameters {'lr': 0.000149, 'momentum': 0.286357}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:00] ax.service.ax_client: Completed trial 9 with data: {'accuracy': (0.901667, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:00] ax.service.ax_client: Generated new trial 10 with parameters {'lr': 1e-06, 'momentum': 1.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:07] ax.service.ax_client: Completed trial 10 with data: {'accuracy': (0.560333, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:08] ax.service.ax_client: Generated new trial 11 with parameters {'lr': 3.7e-05, 'momentum': 1.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:14] ax.service.ax_client: Completed trial 11 with data: {'accuracy': (0.759333, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:15] ax.service.ax_client: Generated new trial 12 with parameters {'lr': 0.000261, 'momentum': 0.913716}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:22] ax.service.ax_client: Completed trial 12 with data: {'accuracy': (0.947667, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:22] ax.service.ax_client: Generated new trial 13 with parameters {'lr': 0.000171, 'momentum': 0.802859}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:29] ax.service.ax_client: Completed trial 13 with data: {'accuracy': (0.942667, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:30] ax.service.ax_client: Generated new trial 14 with parameters {'lr': 0.000161, 'momentum': 1.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:37] ax.service.ax_client: Completed trial 14 with data: {'accuracy': (0.843, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:37] ax.service.ax_client: Generated new trial 15 with parameters {'lr': 0.000312, 'momentum': 0.667594}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:44] ax.service.ax_client: Completed trial 15 with data: {'accuracy': (0.9475, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:45] ax.service.ax_client: Generated new trial 16 with parameters {'lr': 0.000834, 'momentum': 0.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:51] ax.service.ax_client: Completed trial 16 with data: {'accuracy': (0.741333, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:52] ax.service.ax_client: Generated new trial 17 with parameters {'lr': 0.000743, 'momentum': 1.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:42:59] ax.service.ax_client: Completed trial 17 with data: {'accuracy': (0.102667, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:00] ax.service.ax_client: Generated new trial 18 with parameters {'lr': 0.000235, 'momentum': 0.66382}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:06] ax.service.ax_client: Completed trial 18 with data: {'accuracy': (0.936667, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:07] ax.modelbridge.base: Untransformed parameter 0.40000000000000013 greater than upper bound 0.4, clamping\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:07] ax.service.ax_client: Generated new trial 19 with parameters {'lr': 0.4, 'momentum': 0.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:13] ax.service.ax_client: Completed trial 19 with data: {'accuracy': (0.101, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:14] ax.service.ax_client: Generated new trial 20 with parameters {'lr': 6.4e-05, 'momentum': 0.453937}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:21] ax.service.ax_client: Completed trial 20 with data: {'accuracy': (0.897, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:22] ax.service.ax_client: Generated new trial 21 with parameters {'lr': 4.7e-05, 'momentum': 0.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:29] ax.service.ax_client: Completed trial 21 with data: {'accuracy': (0.790333, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:29] ax.service.ax_client: Generated new trial 22 with parameters {'lr': 4e-06, 'momentum': 1.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:36] ax.service.ax_client: Completed trial 22 with data: {'accuracy': (0.830667, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:37] ax.service.ax_client: Generated new trial 23 with parameters {'lr': 8.4e-05, 'momentum': 0.692917}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:43] ax.service.ax_client: Completed trial 23 with data: {'accuracy': (0.920333, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:45] ax.service.ax_client: Generated new trial 24 with parameters {'lr': 0.000412, 'momentum': 0.0}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:51] ax.service.ax_client: Completed trial 24 with data: {'accuracy': (0.849, None)}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:52] ax.service.ax_client: Generated new trial 25 with parameters {'lr': 0.000237, 'momentum': 0.507928}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:43:59] ax.service.ax_client: Completed trial 25 with data: {'accuracy': (0.928333, None)}.\n"
]
}
],
"source": [
"for i in range(25):\n",
" parameters, trial_index = ax_client.get_next_trial()\n",
" # Local evaluation here can be replaced with deployment to external system.\n",
" ax_client.complete_trial(trial_index=trial_index, raw_data=train_evaluate(parameters))"
]
},
{
"cell_type": "markdown",
"id": "16bb6fcf",
"metadata": {
"customInput": null,
"originalKey": "ccd16059-db9f-475b-b527-75afb320e0f4",
"papermill": {
"duration": 0.041967,
"end_time": "2023-12-09T18:43:59.539274",
"exception": false,
"start_time": "2023-12-09T18:43:59.497307",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"### How many trials can run in parallel?\n",
"By default, Ax restricts number of trials that can run in parallel for some optimization stages, in order to improve the optimization performance and reduce the number of trials that the optimization will require. To check the maximum parallelism for each optimization stage:"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "0e03ef5e",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:43:59.625446Z",
"iopub.status.busy": "2023-12-09T18:43:59.624758Z",
"iopub.status.idle": "2023-12-09T18:43:59.633503Z",
"shell.execute_reply": "2023-12-09T18:43:59.632802Z"
},
"executionStartTime": 1690415945269,
"executionStopTime": 1690415945336,
"originalKey": "7182d2f9-912c-464c-b5ad-f65ce6f00017",
"papermill": {
"duration": 0.053586,
"end_time": "2023-12-09T18:43:59.634918",
"exception": false,
"start_time": "2023-12-09T18:43:59.581332",
"status": "completed"
},
"requestMsgId": "4cb4ff79-e45b-4c7d-86a1-7f8007eb2c81",
"showInput": true,
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"[(5, 5), (-1, 3)]"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ax_client.get_max_parallelism()"
]
},
{
"cell_type": "markdown",
"id": "acddb454",
"metadata": {
"customInput": null,
"originalKey": "e2f429e6-2ec8-4af2-906b-52a36a53d329",
"papermill": {
"duration": 0.042436,
"end_time": "2023-12-09T18:43:59.720096",
"exception": false,
"start_time": "2023-12-09T18:43:59.677660",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"The output of this function is a list of tuples of form (number of trials, max parallelism), so the example above means \"the max parallelism is 5 for the first 5 trials and 3 for all subsequent trials.\" This is because the first 5 trials are produced quasi-randomly and can all be evaluated at once, and subsequent trials are produced via Bayesian optimization, which converges on optimal point in fewer trials when parallelism is limited. MaxParallelismReachedException indicates that the parallelism limit has been reached –– refer to the 'Service API Exceptions Meaning and Handling' section at the end of the tutorial for handling.\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "7b0a9449",
"metadata": {
"customInput": null,
"originalKey": "86c7aef9-993a-411e-add5-05839b00d3cf",
"papermill": {
"duration": 0.042306,
"end_time": "2023-12-09T18:43:59.805001",
"exception": false,
"start_time": "2023-12-09T18:43:59.762695",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"### How to view all existing trials during optimization?"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "e6a214f4",
"metadata": {
"customInput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:43:59.897362Z",
"iopub.status.busy": "2023-12-09T18:43:59.891884Z",
"iopub.status.idle": "2023-12-09T18:43:59.933833Z",
"shell.execute_reply": "2023-12-09T18:43:59.932237Z"
},
"executionStartTime": 1690415945532,
"executionStopTime": 1690415946199,
"originalKey": "3fbad5dc-863a-494e-b04f-d7dc1e47936c",
"papermill": {
"duration": 0.087844,
"end_time": "2023-12-09T18:43:59.935463",
"exception": false,
"start_time": "2023-12-09T18:43:59.847619",
"status": "completed"
},
"requestMsgId": "905ea8b6-add0-473e-8516-5be6ad7d7658",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[WARNING 12-09 18:43:59] ax.service.utils.report_utils: Column reason missing for all trials. Not appending column.\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" trial_index | \n",
" arm_name | \n",
" trial_status | \n",
" generation_method | \n",
" accuracy | \n",
" lr | \n",
" momentum | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0 | \n",
" 0_0 | \n",
" COMPLETED | \n",
" Manual | \n",
" 0.841833 | \n",
" 0.000026 | \n",
" 0.580000 | \n",
"
\n",
" \n",
" | 1 | \n",
" 1 | \n",
" 1_0 | \n",
" COMPLETED | \n",
" Sobol | \n",
" 0.100333 | \n",
" 0.009955 | \n",
" 0.633423 | \n",
"
\n",
" \n",
" | 2 | \n",
" 2 | \n",
" 2_0 | \n",
" COMPLETED | \n",
" Sobol | \n",
" 0.318667 | \n",
" 0.000005 | \n",
" 0.022851 | \n",
"
\n",
" \n",
" | 3 | \n",
" 3 | \n",
" 3_0 | \n",
" COMPLETED | \n",
" Sobol | \n",
" 0.458500 | \n",
" 0.000007 | \n",
" 0.176948 | \n",
"
\n",
" \n",
" | 4 | \n",
" 4 | \n",
" 4_0 | \n",
" COMPLETED | \n",
" Sobol | \n",
" 0.926000 | \n",
" 0.000082 | \n",
" 0.908830 | \n",
"
\n",
" \n",
" | 5 | \n",
" 5 | \n",
" 5_0 | \n",
" COMPLETED | \n",
" Sobol | \n",
" 0.929000 | \n",
" 0.000302 | \n",
" 0.341904 | \n",
"
\n",
" \n",
" | 6 | \n",
" 6 | \n",
" 6_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.920000 | \n",
" 0.000137 | \n",
" 0.590917 | \n",
"
\n",
" \n",
" | 7 | \n",
" 7 | \n",
" 7_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.860167 | \n",
" 0.000010 | \n",
" 1.000000 | \n",
"
\n",
" \n",
" | 8 | \n",
" 8 | \n",
" 8_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.888833 | \n",
" 0.000246 | \n",
" 0.000000 | \n",
"
\n",
" \n",
" | 9 | \n",
" 9 | \n",
" 9_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.901667 | \n",
" 0.000149 | \n",
" 0.286357 | \n",
"
\n",
" \n",
" | 10 | \n",
" 10 | \n",
" 10_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.560333 | \n",
" 0.000001 | \n",
" 1.000000 | \n",
"
\n",
" \n",
" | 11 | \n",
" 11 | \n",
" 11_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.759333 | \n",
" 0.000037 | \n",
" 1.000000 | \n",
"
\n",
" \n",
" | 12 | \n",
" 12 | \n",
" 12_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.947667 | \n",
" 0.000261 | \n",
" 0.913716 | \n",
"
\n",
" \n",
" | 13 | \n",
" 13 | \n",
" 13_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.942667 | \n",
" 0.000171 | \n",
" 0.802859 | \n",
"
\n",
" \n",
" | 14 | \n",
" 14 | \n",
" 14_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.843000 | \n",
" 0.000161 | \n",
" 1.000000 | \n",
"
\n",
" \n",
" | 15 | \n",
" 15 | \n",
" 15_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.947500 | \n",
" 0.000312 | \n",
" 0.667594 | \n",
"
\n",
" \n",
" | 16 | \n",
" 16 | \n",
" 16_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.741333 | \n",
" 0.000834 | \n",
" 0.000000 | \n",
"
\n",
" \n",
" | 17 | \n",
" 17 | \n",
" 17_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.102667 | \n",
" 0.000743 | \n",
" 1.000000 | \n",
"
\n",
" \n",
" | 18 | \n",
" 18 | \n",
" 18_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.936667 | \n",
" 0.000235 | \n",
" 0.663820 | \n",
"
\n",
" \n",
" | 19 | \n",
" 19 | \n",
" 19_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.101000 | \n",
" 0.400000 | \n",
" 0.000000 | \n",
"
\n",
" \n",
" | 20 | \n",
" 20 | \n",
" 20_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.897000 | \n",
" 0.000064 | \n",
" 0.453937 | \n",
"
\n",
" \n",
" | 21 | \n",
" 21 | \n",
" 21_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.790333 | \n",
" 0.000047 | \n",
" 0.000000 | \n",
"
\n",
" \n",
" | 22 | \n",
" 22 | \n",
" 22_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.830667 | \n",
" 0.000004 | \n",
" 1.000000 | \n",
"
\n",
" \n",
" | 23 | \n",
" 23 | \n",
" 23_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.920333 | \n",
" 0.000084 | \n",
" 0.692917 | \n",
"
\n",
" \n",
" | 24 | \n",
" 24 | \n",
" 24_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.849000 | \n",
" 0.000412 | \n",
" 0.000000 | \n",
"
\n",
" \n",
" | 25 | \n",
" 25 | \n",
" 25_0 | \n",
" COMPLETED | \n",
" BoTorch | \n",
" 0.928333 | \n",
" 0.000237 | \n",
" 0.507928 | \n",
"
\n",
" \n",
"
\n",
"
accuracy: 0.841833 (SEM: None)
lr: 2.6e-05
momentum: 0.58",
"Arm 1_0
accuracy: 0.100333 (SEM: None)
lr: 0.00995509
momentum: 0.633423",
"Arm 2_0
accuracy: 0.318667 (SEM: None)
lr: 5.18336e-06
momentum: 0.0228515",
"Arm 3_0
accuracy: 0.4585 (SEM: None)
lr: 6.63497e-06
momentum: 0.176948",
"Arm 4_0
accuracy: 0.926 (SEM: None)
lr: 8.15482e-05
momentum: 0.90883",
"Arm 5_0
accuracy: 0.929 (SEM: None)
lr: 0.000302077
momentum: 0.341904",
"Arm 6_0
accuracy: 0.92 (SEM: None)
lr: 0.000137311
momentum: 0.590917",
"Arm 7_0
accuracy: 0.860167 (SEM: None)
lr: 1.03262e-05
momentum: 1",
"Arm 8_0
accuracy: 0.888833 (SEM: None)
lr: 0.000246295
momentum: 0",
"Arm 9_0
accuracy: 0.901667 (SEM: None)
lr: 0.000149439
momentum: 0.286357",
"Arm 10_0
accuracy: 0.560333 (SEM: None)
lr: 1e-06
momentum: 1",
"Arm 11_0
accuracy: 0.759333 (SEM: None)
lr: 3.7322e-05
momentum: 1",
"Arm 12_0
accuracy: 0.947667 (SEM: None)
lr: 0.000261032
momentum: 0.913716",
"Arm 13_0
accuracy: 0.942667 (SEM: None)
lr: 0.000171094
momentum: 0.802859",
"Arm 14_0
accuracy: 0.843 (SEM: None)
lr: 0.000160806
momentum: 1",
"Arm 15_0
accuracy: 0.9475 (SEM: None)
lr: 0.000312473
momentum: 0.667594",
"Arm 16_0
accuracy: 0.741333 (SEM: None)
lr: 0.0008343
momentum: 0",
"Arm 17_0
accuracy: 0.102667 (SEM: None)
lr: 0.000742777
momentum: 1",
"Arm 18_0
accuracy: 0.936667 (SEM: None)
lr: 0.000234685
momentum: 0.66382",
"Arm 19_0
accuracy: 0.101 (SEM: None)
lr: 0.4
momentum: 0",
"Arm 20_0
accuracy: 0.897 (SEM: None)
lr: 6.41677e-05
momentum: 0.453937",
"Arm 21_0
accuracy: 0.790333 (SEM: None)
lr: 4.72432e-05
momentum: 0",
"Arm 22_0
accuracy: 0.830667 (SEM: None)
lr: 4.39148e-06
momentum: 1",
"Arm 23_0
accuracy: 0.920333 (SEM: None)
lr: 8.39891e-05
momentum: 0.692917",
"Arm 24_0
accuracy: 0.849 (SEM: None)
lr: 0.00041203
momentum: 0"
],
"type": "scatter",
"x": [
2.6e-05,
0.00995509203921593,
5.183362665905935e-06,
6.6349677287433636e-06,
8.154817047148868e-05,
0.000302076728574865,
0.00013731096878444508,
1.0326238937471567e-05,
0.00024629478836000866,
0.00014943856257187694,
1e-06,
3.732202988785668e-05,
0.00026103181236379136,
0.00017109378370317222,
0.00016080621873396385,
0.00031247292498106337,
0.0008342998174998784,
0.0007427773432953057,
0.00023468478584700203,
0.4,
6.416773780473855e-05,
4.724316442615592e-05,
4.391476425396512e-06,
8.398908131651445e-05,
0.0004120296111885817
],
"xaxis": "x",
"y": [
0.58,
0.6334228515625,
0.022851496934890747,
0.1769482558593154,
0.9088304992765188,
0.34190391190350056,
0.5909173273841994,
1.0,
0.0,
0.2863569283143983,
1.0,
1.0,
0.9137159656656234,
0.8028585219046457,
1.0,
0.6675937225670816,
0.0,
1.0,
0.6638197948979379,
0.0,
0.4539369805349736,
0.0,
1.0,
0.6929170678181066,
0.0
],
"yaxis": "y"
},
{
"hoverinfo": "text",
"legendgroup": "In-sample",
"marker": {
"color": "black",
"opacity": 0.5,
"symbol": 1
},
"mode": "markers",
"name": "In-sample",
"showlegend": false,
"text": [
"Arm 0_0
accuracy: 0.841833 (SEM: None)
lr: 2.6e-05
momentum: 0.58",
"Arm 1_0
accuracy: 0.100333 (SEM: None)
lr: 0.00995509
momentum: 0.633423",
"Arm 2_0
accuracy: 0.318667 (SEM: None)
lr: 5.18336e-06
momentum: 0.0228515",
"Arm 3_0
accuracy: 0.4585 (SEM: None)
lr: 6.63497e-06
momentum: 0.176948",
"Arm 4_0
accuracy: 0.926 (SEM: None)
lr: 8.15482e-05
momentum: 0.90883",
"Arm 5_0
accuracy: 0.929 (SEM: None)
lr: 0.000302077
momentum: 0.341904",
"Arm 6_0
accuracy: 0.92 (SEM: None)
lr: 0.000137311
momentum: 0.590917",
"Arm 7_0
accuracy: 0.860167 (SEM: None)
lr: 1.03262e-05
momentum: 1",
"Arm 8_0
accuracy: 0.888833 (SEM: None)
lr: 0.000246295
momentum: 0",
"Arm 9_0
accuracy: 0.901667 (SEM: None)
lr: 0.000149439
momentum: 0.286357",
"Arm 10_0
accuracy: 0.560333 (SEM: None)
lr: 1e-06
momentum: 1",
"Arm 11_0
accuracy: 0.759333 (SEM: None)
lr: 3.7322e-05
momentum: 1",
"Arm 12_0
accuracy: 0.947667 (SEM: None)
lr: 0.000261032
momentum: 0.913716",
"Arm 13_0
accuracy: 0.942667 (SEM: None)
lr: 0.000171094
momentum: 0.802859",
"Arm 14_0
accuracy: 0.843 (SEM: None)
lr: 0.000160806
momentum: 1",
"Arm 15_0
accuracy: 0.9475 (SEM: None)
lr: 0.000312473
momentum: 0.667594",
"Arm 16_0
accuracy: 0.741333 (SEM: None)
lr: 0.0008343
momentum: 0",
"Arm 17_0
accuracy: 0.102667 (SEM: None)
lr: 0.000742777
momentum: 1",
"Arm 18_0
accuracy: 0.936667 (SEM: None)
lr: 0.000234685
momentum: 0.66382",
"Arm 19_0
accuracy: 0.101 (SEM: None)
lr: 0.4
momentum: 0",
"Arm 20_0
accuracy: 0.897 (SEM: None)
lr: 6.41677e-05
momentum: 0.453937",
"Arm 21_0
accuracy: 0.790333 (SEM: None)
lr: 4.72432e-05
momentum: 0",
"Arm 22_0
accuracy: 0.830667 (SEM: None)
lr: 4.39148e-06
momentum: 1",
"Arm 23_0
accuracy: 0.920333 (SEM: None)
lr: 8.39891e-05
momentum: 0.692917",
"Arm 24_0
accuracy: 0.849 (SEM: None)
lr: 0.00041203
momentum: 0"
],
"type": "scatter",
"x": [
2.6e-05,
0.00995509203921593,
5.183362665905935e-06,
6.6349677287433636e-06,
8.154817047148868e-05,
0.000302076728574865,
0.00013731096878444508,
1.0326238937471567e-05,
0.00024629478836000866,
0.00014943856257187694,
1e-06,
3.732202988785668e-05,
0.00026103181236379136,
0.00017109378370317222,
0.00016080621873396385,
0.00031247292498106337,
0.0008342998174998784,
0.0007427773432953057,
0.00023468478584700203,
0.4,
6.416773780473855e-05,
4.724316442615592e-05,
4.391476425396512e-06,
8.398908131651445e-05,
0.0004120296111885817
],
"xaxis": "x2",
"y": [
0.58,
0.6334228515625,
0.022851496934890747,
0.1769482558593154,
0.9088304992765188,
0.34190391190350056,
0.5909173273841994,
1.0,
0.0,
0.2863569283143983,
1.0,
1.0,
0.9137159656656234,
0.8028585219046457,
1.0,
0.6675937225670816,
0.0,
1.0,
0.6638197948979379,
0.0,
0.4539369805349736,
0.0,
1.0,
0.6929170678181066,
0.0
],
"yaxis": "y2"
}
],
"layout": {
"annotations": [
{
"font": {
"size": 14
},
"showarrow": false,
"text": "Mean",
"x": 0.25,
"xanchor": "center",
"xref": "paper",
"y": 1,
"yanchor": "bottom",
"yref": "paper"
},
{
"font": {
"size": 14
},
"showarrow": false,
"text": "Standard Error",
"x": 0.8,
"xanchor": "center",
"xref": "paper",
"y": 1,
"yanchor": "bottom",
"yref": "paper"
}
],
"autosize": false,
"height": 450,
"hovermode": "closest",
"legend": {
"orientation": "h",
"x": 0,
"y": -0.25
},
"margin": {
"b": 100,
"l": 35,
"pad": 0,
"r": 35,
"t": 35
},
"template": {
"data": {
"bar": [
{
"error_x": {
"color": "#2a3f5f"
},
"error_y": {
"color": "#2a3f5f"
},
"marker": {
"line": {
"color": "#E5ECF6",
"width": 0.5
},
"pattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
}
},
"type": "bar"
}
],
"barpolar": [
{
"marker": {
"line": {
"color": "#E5ECF6",
"width": 0.5
},
"pattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
}
},
"type": "barpolar"
}
],
"carpet": [
{
"aaxis": {
"endlinecolor": "#2a3f5f",
"gridcolor": "white",
"linecolor": "white",
"minorgridcolor": "white",
"startlinecolor": "#2a3f5f"
},
"baxis": {
"endlinecolor": "#2a3f5f",
"gridcolor": "white",
"linecolor": "white",
"minorgridcolor": "white",
"startlinecolor": "#2a3f5f"
},
"type": "carpet"
}
],
"choropleth": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"type": "choropleth"
}
],
"contour": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "contour"
}
],
"contourcarpet": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"type": "contourcarpet"
}
],
"heatmap": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "heatmap"
}
],
"heatmapgl": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "heatmapgl"
}
],
"histogram": [
{
"marker": {
"pattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
}
},
"type": "histogram"
}
],
"histogram2d": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "histogram2d"
}
],
"histogram2dcontour": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "histogram2dcontour"
}
],
"mesh3d": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"type": "mesh3d"
}
],
"parcoords": [
{
"line": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "parcoords"
}
],
"pie": [
{
"automargin": true,
"type": "pie"
}
],
"scatter": [
{
"fillpattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
},
"type": "scatter"
}
],
"scatter3d": [
{
"line": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatter3d"
}
],
"scattercarpet": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattercarpet"
}
],
"scattergeo": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattergeo"
}
],
"scattergl": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattergl"
}
],
"scattermapbox": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattermapbox"
}
],
"scatterpolar": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatterpolar"
}
],
"scatterpolargl": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatterpolargl"
}
],
"scatterternary": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatterternary"
}
],
"surface": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "surface"
}
],
"table": [
{
"cells": {
"fill": {
"color": "#EBF0F8"
},
"line": {
"color": "white"
}
},
"header": {
"fill": {
"color": "#C8D4E3"
},
"line": {
"color": "white"
}
},
"type": "table"
}
]
},
"layout": {
"annotationdefaults": {
"arrowcolor": "#2a3f5f",
"arrowhead": 0,
"arrowwidth": 1
},
"autotypenumbers": "strict",
"coloraxis": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"colorscale": {
"diverging": [
[
0,
"#8e0152"
],
[
0.1,
"#c51b7d"
],
[
0.2,
"#de77ae"
],
[
0.3,
"#f1b6da"
],
[
0.4,
"#fde0ef"
],
[
0.5,
"#f7f7f7"
],
[
0.6,
"#e6f5d0"
],
[
0.7,
"#b8e186"
],
[
0.8,
"#7fbc41"
],
[
0.9,
"#4d9221"
],
[
1,
"#276419"
]
],
"sequential": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"sequentialminus": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
]
},
"colorway": [
"#636efa",
"#EF553B",
"#00cc96",
"#ab63fa",
"#FFA15A",
"#19d3f3",
"#FF6692",
"#B6E880",
"#FF97FF",
"#FECB52"
],
"font": {
"color": "#2a3f5f"
},
"geo": {
"bgcolor": "white",
"lakecolor": "white",
"landcolor": "#E5ECF6",
"showlakes": true,
"showland": true,
"subunitcolor": "white"
},
"hoverlabel": {
"align": "left"
},
"hovermode": "closest",
"mapbox": {
"style": "light"
},
"paper_bgcolor": "white",
"plot_bgcolor": "#E5ECF6",
"polar": {
"angularaxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
},
"bgcolor": "#E5ECF6",
"radialaxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
}
},
"scene": {
"xaxis": {
"backgroundcolor": "#E5ECF6",
"gridcolor": "white",
"gridwidth": 2,
"linecolor": "white",
"showbackground": true,
"ticks": "",
"zerolinecolor": "white"
},
"yaxis": {
"backgroundcolor": "#E5ECF6",
"gridcolor": "white",
"gridwidth": 2,
"linecolor": "white",
"showbackground": true,
"ticks": "",
"zerolinecolor": "white"
},
"zaxis": {
"backgroundcolor": "#E5ECF6",
"gridcolor": "white",
"gridwidth": 2,
"linecolor": "white",
"showbackground": true,
"ticks": "",
"zerolinecolor": "white"
}
},
"shapedefaults": {
"line": {
"color": "#2a3f5f"
}
},
"ternary": {
"aaxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
},
"baxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
},
"bgcolor": "#E5ECF6",
"caxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
}
},
"title": {
"x": 0.05
},
"xaxis": {
"automargin": true,
"gridcolor": "white",
"linecolor": "white",
"ticks": "",
"title": {
"standoff": 15
},
"zerolinecolor": "white",
"zerolinewidth": 2
},
"yaxis": {
"automargin": true,
"gridcolor": "white",
"linecolor": "white",
"ticks": "",
"title": {
"standoff": 15
},
"zerolinecolor": "white",
"zerolinewidth": 2
}
}
},
"title": {
"text": "accuracy"
},
"width": 950,
"xaxis": {
"anchor": "y",
"autorange": false,
"domain": [
0.05,
0.45
],
"exponentformat": "e",
"range": [
-6.0,
-0.3979400086720376
],
"tickfont": {
"size": 11
},
"tickmode": "auto",
"title": {
"text": "lr"
},
"type": "log"
},
"xaxis2": {
"anchor": "y2",
"autorange": false,
"domain": [
0.6,
1
],
"exponentformat": "e",
"range": [
-6.0,
-0.3979400086720376
],
"tickfont": {
"size": 11
},
"tickmode": "auto",
"title": {
"text": "lr"
},
"type": "log"
},
"yaxis": {
"anchor": "x",
"autorange": false,
"domain": [
0,
1
],
"exponentformat": "e",
"range": [
0.0,
1.0
],
"tickfont": {
"size": 11
},
"tickmode": "auto",
"title": {
"text": "momentum"
},
"type": "linear"
},
"yaxis2": {
"anchor": "x2",
"autorange": false,
"domain": [
0,
1
],
"exponentformat": "e",
"range": [
0.0,
1.0
],
"tickfont": {
"size": 11
},
"tickmode": "auto",
"type": "linear"
}
}
},
"text/html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"render(ax_client.get_contour_plot(param_x=\"lr\", param_y=\"momentum\", metric_name=\"accuracy\"))"
]
},
{
"cell_type": "markdown",
"id": "0b9e992b",
"metadata": {
"customInput": null,
"originalKey": "5c91d83a-9a90-4ea0-8df9-9d242d998cb3",
"papermill": {
"duration": 0.057284,
"end_time": "2023-12-09T18:44:01.467579",
"exception": false,
"start_time": "2023-12-09T18:44:01.410295",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"Here we plot the optimization trace, showing the progression of finding the point with the optimal objective:\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "e46404af",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:01.584155Z",
"iopub.status.busy": "2023-12-09T18:44:01.583605Z",
"iopub.status.idle": "2023-12-09T18:44:01.634123Z",
"shell.execute_reply": "2023-12-09T18:44:01.633388Z"
},
"executionStartTime": 1690415953760,
"executionStopTime": 1690415954260,
"originalKey": "3a767bdf-7ef3-48e7-b853-6fae5e9e02ff",
"papermill": {
"duration": 0.114256,
"end_time": "2023-12-09T18:44:01.638931",
"exception": false,
"start_time": "2023-12-09T18:44:01.524675",
"status": "completed"
},
"requestMsgId": "043de459-6a28-4796-b237-808385c9e54c",
"showInput": true,
"tags": []
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"config": {
"linkText": "Export to plot.ly",
"plotlyServerURL": "https://plot.ly",
"showLink": false
},
"data": [
{
"hoverinfo": "none",
"legendgroup": "",
"line": {
"width": 0
},
"mode": "lines",
"showlegend": false,
"type": "scatter",
"x": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26
],
"y": [
0.8418333333333333,
0.8418333333333333,
0.8418333333333333,
0.8418333333333333,
0.926,
0.929,
0.929,
0.929,
0.929,
0.929,
0.929,
0.929,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667
]
},
{
"fill": "tonexty",
"fillcolor": "rgba(128,177,211,0.3)",
"legendgroup": "objective value",
"line": {
"color": "rgba(128,177,211,1)"
},
"mode": "lines",
"name": "objective value",
"text": [
"
Parameterization:
lr: 2.6e-05
momentum: 0.58",
"
Parameterization:
lr: 0.00995509203921593
momentum: 0.6334228515625",
"
Parameterization:
lr: 5.183362665905935e-06
momentum: 0.022851496934890747",
"
Parameterization:
lr: 6.6349677287433636e-06
momentum: 0.1769482558593154",
"
Parameterization:
lr: 8.154817047148868e-05
momentum: 0.9088304992765188",
"
Parameterization:
lr: 0.000302076728574865
momentum: 0.34190391190350056",
"
Parameterization:
lr: 0.00013731096878444508
momentum: 0.5909173273841994",
"
Parameterization:
lr: 1.0326238937471567e-05
momentum: 1.0",
"
Parameterization:
lr: 0.00024629478836000866
momentum: 0.0",
"
Parameterization:
lr: 0.00014943856257187694
momentum: 0.2863569283143983",
"
Parameterization:
lr: 1e-06
momentum: 1.0",
"
Parameterization:
lr: 3.732202988785668e-05
momentum: 1.0",
"
Parameterization:
lr: 0.00026103181236379136
momentum: 0.9137159656656234",
"
Parameterization:
lr: 0.00017109378370317222
momentum: 0.8028585219046457",
"
Parameterization:
lr: 0.00016080621873396385
momentum: 1.0",
"
Parameterization:
lr: 0.00031247292498106337
momentum: 0.6675937225670816",
"
Parameterization:
lr: 0.0008342998174998784
momentum: 0.0",
"
Parameterization:
lr: 0.0007427773432953057
momentum: 1.0",
"
Parameterization:
lr: 0.00023468478584700203
momentum: 0.6638197948979379",
"
Parameterization:
lr: 0.4
momentum: 0.0",
"
Parameterization:
lr: 6.416773780473855e-05
momentum: 0.4539369805349736",
"
Parameterization:
lr: 4.724316442615592e-05
momentum: 0.0",
"
Parameterization:
lr: 4.391476425396512e-06
momentum: 1.0",
"
Parameterization:
lr: 8.398908131651445e-05
momentum: 0.6929170678181066",
"
Parameterization:
lr: 0.0004120296111885817
momentum: 0.0",
"
Parameterization:
lr: 0.00023651776590154824
momentum: 0.5079281703970463"
],
"type": "scatter",
"x": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26
],
"y": [
0.8418333333333333,
0.8418333333333333,
0.8418333333333333,
0.8418333333333333,
0.926,
0.929,
0.929,
0.929,
0.929,
0.929,
0.929,
0.929,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667
]
},
{
"fill": "tonexty",
"fillcolor": "rgba(128,177,211,0.3)",
"hoverinfo": "none",
"legendgroup": "",
"line": {
"width": 0
},
"mode": "lines",
"showlegend": false,
"type": "scatter",
"x": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26
],
"y": [
0.8418333333333333,
0.8418333333333333,
0.8418333333333333,
0.8418333333333333,
0.926,
0.929,
0.929,
0.929,
0.929,
0.929,
0.929,
0.929,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667,
0.9476666666666667
]
},
{
"line": {
"color": "rgba(141,211,199,1)",
"dash": "dash"
},
"mode": "lines",
"name": "model change",
"type": "scatter",
"x": [
5,
5
],
"y": [
0.8418333333333333,
0.9476666666666667
]
}
],
"layout": {
"showlegend": true,
"template": {
"data": {
"bar": [
{
"error_x": {
"color": "#2a3f5f"
},
"error_y": {
"color": "#2a3f5f"
},
"marker": {
"line": {
"color": "#E5ECF6",
"width": 0.5
},
"pattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
}
},
"type": "bar"
}
],
"barpolar": [
{
"marker": {
"line": {
"color": "#E5ECF6",
"width": 0.5
},
"pattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
}
},
"type": "barpolar"
}
],
"carpet": [
{
"aaxis": {
"endlinecolor": "#2a3f5f",
"gridcolor": "white",
"linecolor": "white",
"minorgridcolor": "white",
"startlinecolor": "#2a3f5f"
},
"baxis": {
"endlinecolor": "#2a3f5f",
"gridcolor": "white",
"linecolor": "white",
"minorgridcolor": "white",
"startlinecolor": "#2a3f5f"
},
"type": "carpet"
}
],
"choropleth": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"type": "choropleth"
}
],
"contour": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "contour"
}
],
"contourcarpet": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"type": "contourcarpet"
}
],
"heatmap": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "heatmap"
}
],
"heatmapgl": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "heatmapgl"
}
],
"histogram": [
{
"marker": {
"pattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
}
},
"type": "histogram"
}
],
"histogram2d": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "histogram2d"
}
],
"histogram2dcontour": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "histogram2dcontour"
}
],
"mesh3d": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"type": "mesh3d"
}
],
"parcoords": [
{
"line": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "parcoords"
}
],
"pie": [
{
"automargin": true,
"type": "pie"
}
],
"scatter": [
{
"fillpattern": {
"fillmode": "overlay",
"size": 10,
"solidity": 0.2
},
"type": "scatter"
}
],
"scatter3d": [
{
"line": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatter3d"
}
],
"scattercarpet": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattercarpet"
}
],
"scattergeo": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattergeo"
}
],
"scattergl": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattergl"
}
],
"scattermapbox": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scattermapbox"
}
],
"scatterpolar": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatterpolar"
}
],
"scatterpolargl": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatterpolargl"
}
],
"scatterternary": [
{
"marker": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"type": "scatterternary"
}
],
"surface": [
{
"colorbar": {
"outlinewidth": 0,
"ticks": ""
},
"colorscale": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"type": "surface"
}
],
"table": [
{
"cells": {
"fill": {
"color": "#EBF0F8"
},
"line": {
"color": "white"
}
},
"header": {
"fill": {
"color": "#C8D4E3"
},
"line": {
"color": "white"
}
},
"type": "table"
}
]
},
"layout": {
"annotationdefaults": {
"arrowcolor": "#2a3f5f",
"arrowhead": 0,
"arrowwidth": 1
},
"autotypenumbers": "strict",
"coloraxis": {
"colorbar": {
"outlinewidth": 0,
"ticks": ""
}
},
"colorscale": {
"diverging": [
[
0,
"#8e0152"
],
[
0.1,
"#c51b7d"
],
[
0.2,
"#de77ae"
],
[
0.3,
"#f1b6da"
],
[
0.4,
"#fde0ef"
],
[
0.5,
"#f7f7f7"
],
[
0.6,
"#e6f5d0"
],
[
0.7,
"#b8e186"
],
[
0.8,
"#7fbc41"
],
[
0.9,
"#4d9221"
],
[
1,
"#276419"
]
],
"sequential": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
],
"sequentialminus": [
[
0.0,
"#0d0887"
],
[
0.1111111111111111,
"#46039f"
],
[
0.2222222222222222,
"#7201a8"
],
[
0.3333333333333333,
"#9c179e"
],
[
0.4444444444444444,
"#bd3786"
],
[
0.5555555555555556,
"#d8576b"
],
[
0.6666666666666666,
"#ed7953"
],
[
0.7777777777777778,
"#fb9f3a"
],
[
0.8888888888888888,
"#fdca26"
],
[
1.0,
"#f0f921"
]
]
},
"colorway": [
"#636efa",
"#EF553B",
"#00cc96",
"#ab63fa",
"#FFA15A",
"#19d3f3",
"#FF6692",
"#B6E880",
"#FF97FF",
"#FECB52"
],
"font": {
"color": "#2a3f5f"
},
"geo": {
"bgcolor": "white",
"lakecolor": "white",
"landcolor": "#E5ECF6",
"showlakes": true,
"showland": true,
"subunitcolor": "white"
},
"hoverlabel": {
"align": "left"
},
"hovermode": "closest",
"mapbox": {
"style": "light"
},
"paper_bgcolor": "white",
"plot_bgcolor": "#E5ECF6",
"polar": {
"angularaxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
},
"bgcolor": "#E5ECF6",
"radialaxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
}
},
"scene": {
"xaxis": {
"backgroundcolor": "#E5ECF6",
"gridcolor": "white",
"gridwidth": 2,
"linecolor": "white",
"showbackground": true,
"ticks": "",
"zerolinecolor": "white"
},
"yaxis": {
"backgroundcolor": "#E5ECF6",
"gridcolor": "white",
"gridwidth": 2,
"linecolor": "white",
"showbackground": true,
"ticks": "",
"zerolinecolor": "white"
},
"zaxis": {
"backgroundcolor": "#E5ECF6",
"gridcolor": "white",
"gridwidth": 2,
"linecolor": "white",
"showbackground": true,
"ticks": "",
"zerolinecolor": "white"
}
},
"shapedefaults": {
"line": {
"color": "#2a3f5f"
}
},
"ternary": {
"aaxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
},
"baxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
},
"bgcolor": "#E5ECF6",
"caxis": {
"gridcolor": "white",
"linecolor": "white",
"ticks": ""
}
},
"title": {
"x": 0.05
},
"xaxis": {
"automargin": true,
"gridcolor": "white",
"linecolor": "white",
"ticks": "",
"title": {
"standoff": 15
},
"zerolinecolor": "white",
"zerolinewidth": 2
},
"yaxis": {
"automargin": true,
"gridcolor": "white",
"linecolor": "white",
"ticks": "",
"title": {
"standoff": 15
},
"zerolinecolor": "white",
"zerolinewidth": 2
}
}
},
"title": {
"text": "Model performance vs. # of iterations"
},
"xaxis": {
"title": {
"text": "Iteration"
}
},
"yaxis": {
"title": {
"text": "Accuracy"
}
}
}
},
"text/html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"render(\n",
" ax_client.get_optimization_trace()\n",
") "
]
},
{
"cell_type": "markdown",
"id": "0a39cda1",
"metadata": {
"customInput": null,
"executionStartTime": 1689617061294,
"executionStopTime": 1689617061325,
"originalKey": "09aaec9d-c178-42e2-b549-663cd17f8c3d",
"papermill": {
"duration": 0.060443,
"end_time": "2023-12-09T18:44:01.760670",
"exception": false,
"start_time": "2023-12-09T18:44:01.700227",
"status": "completed"
},
"requestMsgId": "09aaec9d-c178-42e2-b549-663cd17f8c3d",
"showInput": false,
"tags": []
},
"source": [
"## 8. Train CNN with best hyperparameters and evaluate on test set\n",
"Note that the resulting accuracy on the test set generally won't be the same as the maximum accuracy achieved on the evaluation set throughout optimization. "
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "b9804bd9",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:01.884190Z",
"iopub.status.busy": "2023-12-09T18:44:01.883640Z",
"iopub.status.idle": "2023-12-09T18:44:01.909109Z",
"shell.execute_reply": "2023-12-09T18:44:01.907856Z"
},
"executionStartTime": 1690415954397,
"executionStopTime": 1690415954452,
"originalKey": "27f92d16-93c4-43bb-a37f-e7a1aeecd856",
"papermill": {
"duration": 0.089506,
"end_time": "2023-12-09T18:44:01.911417",
"exception": false,
"start_time": "2023-12-09T18:44:01.821911",
"status": "completed"
},
"requestMsgId": "07eba5ce-bebe-4588-8dbb-07553efeb2b0",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[WARNING 12-09 18:44:01] ax.service.utils.report_utils: Column reason missing for all trials. Not appending column.\n"
]
},
{
"data": {
"text/plain": [
"{'lr': 0.00026103181236379136, 'momentum': 0.9137159656656234}"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = ax_client.get_trials_data_frame()\n",
"best_arm_idx = df.trial_index[df[\"accuracy\"] == df[\"accuracy\"].max()].values[0]\n",
"best_arm = ax_client.get_trial_parameters(best_arm_idx)\n",
"best_arm"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "baf43713",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:02.035333Z",
"iopub.status.busy": "2023-12-09T18:44:02.034657Z",
"iopub.status.idle": "2023-12-09T18:44:02.039352Z",
"shell.execute_reply": "2023-12-09T18:44:02.038783Z"
},
"executionStartTime": 1690415954677,
"executionStopTime": 1690415954681,
"originalKey": "d0c7c645-c230-4654-a3b5-a01c61a09393",
"papermill": {
"duration": 0.067912,
"end_time": "2023-12-09T18:44:02.040686",
"exception": false,
"start_time": "2023-12-09T18:44:01.972774",
"status": "completed"
},
"requestMsgId": "0a962cef-65a1-4f95-9410-37a9a8e5c5ac",
"showInput": true,
"tags": []
},
"outputs": [],
"source": [
"combined_train_valid_set = torch.utils.data.ConcatDataset(\n",
" [\n",
" train_loader.dataset.dataset,\n",
" valid_loader.dataset.dataset,\n",
" ]\n",
")\n",
"combined_train_valid_loader = torch.utils.data.DataLoader(\n",
" combined_train_valid_set,\n",
" batch_size=BATCH_SIZE,\n",
" shuffle=True,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "a3837121",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:02.164539Z",
"iopub.status.busy": "2023-12-09T18:44:02.163987Z",
"iopub.status.idle": "2023-12-09T18:44:31.257140Z",
"shell.execute_reply": "2023-12-09T18:44:31.256474Z"
},
"executionStartTime": 1690415954791,
"executionStopTime": 1690416061340,
"originalKey": "5695c78b-4c6e-4d35-ab08-6c60781bd8f1",
"papermill": {
"duration": 29.157211,
"end_time": "2023-12-09T18:44:31.259152",
"exception": false,
"start_time": "2023-12-09T18:44:02.101941",
"status": "completed"
},
"requestMsgId": "e22fa0c7-88cc-4d8a-bb7d-4f96fbae9a42",
"showInput": true,
"tags": []
},
"outputs": [],
"source": [
"net = train(\n",
" net=CNN(),\n",
" train_loader=combined_train_valid_loader,\n",
" parameters=best_arm,\n",
" dtype=dtype,\n",
" device=device,\n",
")\n",
"test_accuracy = evaluate(\n",
" net=net,\n",
" data_loader=test_loader,\n",
" dtype=dtype,\n",
" device=device,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "2e53b8d4",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:31.389704Z",
"iopub.status.busy": "2023-12-09T18:44:31.389152Z",
"iopub.status.idle": "2023-12-09T18:44:31.393555Z",
"shell.execute_reply": "2023-12-09T18:44:31.392831Z"
},
"executionStartTime": 1690416061460,
"executionStopTime": 1690416061467,
"originalKey": "7522e229-9641-4383-a892-12c3f0a8011c",
"papermill": {
"duration": 0.073453,
"end_time": "2023-12-09T18:44:31.394984",
"exception": false,
"start_time": "2023-12-09T18:44:31.321531",
"status": "completed"
},
"requestMsgId": "5552d77d-9c9d-4712-9256-2cb3da836f2c",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Classification Accuracy (test set): 98.28%\n"
]
}
],
"source": [
"print(f\"Classification Accuracy (test set): {round(test_accuracy*100, 2)}%\")"
]
},
{
"cell_type": "markdown",
"id": "001446a5",
"metadata": {
"customInput": null,
"originalKey": "c8232211-4837-4677-b86c-bce730635fff",
"papermill": {
"duration": 0.060997,
"end_time": "2023-12-09T18:44:31.516963",
"exception": false,
"start_time": "2023-12-09T18:44:31.455966",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"## 9. Save / reload optimization to JSON / SQL\n",
"We can serialize the state of optimization to JSON and save it to a `.json` file or save it to the SQL backend. For the former:"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "aabc861d",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:31.640640Z",
"iopub.status.busy": "2023-12-09T18:44:31.639949Z",
"iopub.status.idle": "2023-12-09T18:44:31.668692Z",
"shell.execute_reply": "2023-12-09T18:44:31.667761Z"
},
"executionStartTime": 1690416061571,
"executionStopTime": 1690416061657,
"originalKey": "6afddb45-c980-4b14-b5e9-927747ea98ea",
"papermill": {
"duration": 0.095218,
"end_time": "2023-12-09T18:44:31.673279",
"exception": false,
"start_time": "2023-12-09T18:44:31.578061",
"status": "completed"
},
"requestMsgId": "bab02be8-706c-4422-b97b-c222b5084bba",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:31] ax.service.ax_client: Saved JSON-serialized state of optimization to `ax_client_snapshot.json`.\n"
]
}
],
"source": [
"ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument."
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "2b3c8551",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:31.798069Z",
"iopub.status.busy": "2023-12-09T18:44:31.797427Z",
"iopub.status.idle": "2023-12-09T18:44:31.918548Z",
"shell.execute_reply": "2023-12-09T18:44:31.917913Z"
},
"executionStartTime": 1690416061758,
"executionStopTime": 1690416062132,
"originalKey": "31e6f7b4-cf6b-4967-95ff-f76d03657fb2",
"papermill": {
"duration": 0.185299,
"end_time": "2023-12-09T18:44:31.919993",
"exception": false,
"start_time": "2023-12-09T18:44:31.734694",
"status": "completed"
},
"requestMsgId": "f2d10848-f995-420d-88e7-9036894d7b1b",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"/tmp/tmp.bwZKabbZig/Ax-main/ax/core/data.py:203: FutureWarning:\n",
"\n",
"Passing literal json to 'read_json' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
"\n",
"[INFO 12-09 18:44:31] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n"
]
}
],
"source": [
"restored_ax_client = (\n",
" AxClient.load_from_json_file()\n",
") # For custom filepath, pass `filepath` argument."
]
},
{
"cell_type": "markdown",
"id": "e7eba4fe",
"metadata": {
"customInput": null,
"originalKey": "122510f5-5b9e-4b1c-9f5e-8c8ea2e08848",
"papermill": {
"duration": 0.060498,
"end_time": "2023-12-09T18:44:32.041986",
"exception": false,
"start_time": "2023-12-09T18:44:31.981488",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"To store state of optimization to an SQL backend, first follow [setup instructions](https://ax.dev/docs/storage.html#sql) on Ax website."
]
},
{
"cell_type": "markdown",
"id": "92b10438",
"metadata": {
"customInput": null,
"originalKey": "bd80e639-aa0f-4dc1-8542-0caf0d674fda",
"papermill": {
"duration": 0.060879,
"end_time": "2023-12-09T18:44:32.163739",
"exception": false,
"start_time": "2023-12-09T18:44:32.102860",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"Having set up the SQL backend, pass `DBSettings` to `AxClient` on instantiation (note that `SQLAlchemy` dependency will have to be installed – for installation, refer to [optional dependencies](https://ax.dev/docs/installation.html#optional-dependencies) on Ax website):"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "718e4964",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:32.287744Z",
"iopub.status.busy": "2023-12-09T18:44:32.287095Z",
"iopub.status.idle": "2023-12-09T18:44:32.297769Z",
"shell.execute_reply": "2023-12-09T18:44:32.297221Z"
},
"executionStartTime": 1690416062222,
"executionStopTime": 1690416062314,
"originalKey": "80eb6a2e-6564-405e-b5d4-d448e32dbf60",
"papermill": {
"duration": 0.074386,
"end_time": "2023-12-09T18:44:32.299195",
"exception": false,
"start_time": "2023-12-09T18:44:32.224809",
"status": "completed"
},
"requestMsgId": "65f2307f-b800-4415-b9e7-11734a2a6889",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:32] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n"
]
}
],
"source": [
"from ax.storage.sqa_store.structs import DBSettings\n",
"\n",
"# URL is of the form \"dialect+driver://username:password@host:port/database\".\n",
"db_settings = DBSettings(url=\"sqlite:///foo.db\")\n",
"# Instead of URL, can provide a `creator function`; can specify custom encoders/decoders if necessary.\n",
"new_ax = AxClient(db_settings=db_settings)"
]
},
{
"cell_type": "markdown",
"id": "aebaaa88",
"metadata": {
"customInput": null,
"originalKey": "adafd3aa-b84e-4e86-9694-a29f94c6d5f3",
"papermill": {
"duration": 0.066083,
"end_time": "2023-12-09T18:44:32.427494",
"exception": false,
"start_time": "2023-12-09T18:44:32.361411",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"When valid `DBSettings` are passed into `AxClient`, a unique experiment name is a required argument (`name`) to `ax_client.create_experiment`. The **state of the optimization is auto-saved** any time it changes (i.e. a new trial is added or completed, etc). \n",
"\n",
"To reload an optimization state later, instantiate `AxClient` with the same `DBSettings` and use `ax_client.load_experiment_from_database(experiment_name=\"my_experiment\")`."
]
},
{
"cell_type": "markdown",
"id": "204d27e8",
"metadata": {
"customInput": null,
"originalKey": "2f4a875b-1e18-4352-955d-576d6b01c5ed",
"papermill": {
"duration": 0.062162,
"end_time": "2023-12-09T18:44:32.551381",
"exception": false,
"start_time": "2023-12-09T18:44:32.489219",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"# Special Cases"
]
},
{
"cell_type": "markdown",
"id": "8684a9d0",
"metadata": {
"customInput": null,
"originalKey": "0d49e448-4768-401d-ac1d-810aee633c9a",
"papermill": {
"duration": 0.061155,
"end_time": "2023-12-09T18:44:32.674355",
"exception": false,
"start_time": "2023-12-09T18:44:32.613200",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"**Evaluation failure**: should any optimization iterations fail during evaluation, `log_trial_failure` will ensure that the same trial is not proposed again."
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "f278fde7",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:32.800613Z",
"iopub.status.busy": "2023-12-09T18:44:32.799970Z",
"iopub.status.idle": "2023-12-09T18:44:33.808502Z",
"shell.execute_reply": "2023-12-09T18:44:33.807812Z"
},
"executionStartTime": 1690416062420,
"executionStopTime": 1690416064316,
"originalKey": "faa83f1d-31da-481a-96e4-ccbc12f30b91",
"papermill": {
"duration": 1.074956,
"end_time": "2023-12-09T18:44:33.810008",
"exception": false,
"start_time": "2023-12-09T18:44:32.735052",
"status": "completed"
},
"requestMsgId": "80a40c3a-76ed-4e1d-aa77-3652fadbe69f",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:33] ax.service.ax_client: Generated new trial 26 with parameters {'lr': 0.000246, 'momentum': 0.77497}.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:33] ax.service.ax_client: Registered failure of trial 26.\n"
]
}
],
"source": [
"_, trial_index = ax_client.get_next_trial()\n",
"ax_client.log_trial_failure(trial_index=trial_index)"
]
},
{
"cell_type": "markdown",
"id": "e3ceda78",
"metadata": {
"customInput": null,
"originalKey": "c826a96e-9431-49bd-87d7-62b517537a15",
"papermill": {
"duration": 0.06166,
"end_time": "2023-12-09T18:44:33.933898",
"exception": false,
"start_time": "2023-12-09T18:44:33.872238",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"**Need to run many trials in parallel**: for optimal results and optimization efficiency, we strongly recommend sequential optimization (generating a few trials, then waiting for them to be completed with evaluation data). However, if your use case needs to dispatch many trials in parallel before they are updated with data and you are running into the *\"All trials for current model have been generated, but not enough data has been observed to fit next model\"* error, instantiate `AxClient` as `AxClient(enforce_sequential_optimization=False)`."
]
},
{
"cell_type": "markdown",
"id": "eb410c9f",
"metadata": {
"customInput": null,
"originalKey": "683378e0-893b-49a1-b090-084dc394da1a",
"papermill": {
"duration": 0.061297,
"end_time": "2023-12-09T18:44:34.057165",
"exception": false,
"start_time": "2023-12-09T18:44:33.995868",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"# Service API Exceptions Meaning and Handling\n",
"[**`DataRequiredError`**](https://ax.dev/api/exceptions.html#ax.exceptions.core.DataRequiredError): Ax generation strategy needs to be updated with more data to proceed to the next optimization model. When the optimization moves from initialization stage to the Bayesian optimization stage, the underlying BayesOpt model needs sufficient data to train. For optimal results and optimization efficiency (finding the optimal point in the least number of trials), we recommend sequential optimization (generating a few trials, then waiting for them to be completed with evaluation data). Therefore, the correct way to handle this exception is to wait until more trial evaluations complete and log their data via `ax_client.complete_trial(...)`. \n",
"\n",
"However, if there is strong need to generate more trials before more data is available, instantiate `AxClient` as `AxClient(enforce_sequential_optimization=False)`. With this setting, as many trials will be generated from the initialization stage as requested, and the optimization will move to the BayesOpt stage whenever enough trials are completed."
]
},
{
"cell_type": "markdown",
"id": "c1e87369",
"metadata": {
"customInput": null,
"originalKey": "4602d41d-43aa-46d2-9ca6-392c414d0b5f",
"papermill": {
"duration": 0.061365,
"end_time": "2023-12-09T18:44:34.180173",
"exception": false,
"start_time": "2023-12-09T18:44:34.118808",
"status": "completed"
},
"showInput": false,
"tags": []
},
"source": [
"[**`MaxParallelismReachedException`**](https://ax.dev/api/modelbridge.html#ax.modelbridge.generation_strategy.MaxParallelismReachedException): generation strategy restricts the number of trials that can be run simultaneously (to encourage sequential optimization), and the parallelism limit has been reached. The correct way to handle this exception is the same as `DataRequiredError` – to wait until more trial evluations complete and log their data via `ax_client.complete_trial(...)`.\n",
" \n",
"In some cases higher parallelism is important, so `enforce_sequential_optimization=False` kwarg to AxClient allows the user to suppress limiting of parallelism. It's also possible to override the default parallelism setting for all stages of the optimization by passing `choose_generation_strategy_kwargs` to `ax_client.create_experiment`:"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "5478ae35",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:34.306084Z",
"iopub.status.busy": "2023-12-09T18:44:34.305802Z",
"iopub.status.idle": "2023-12-09T18:44:34.318943Z",
"shell.execute_reply": "2023-12-09T18:44:34.318250Z"
},
"executionStartTime": 1690416064534,
"executionStopTime": 1690416064564,
"originalKey": "d62e6cfd-5127-450e-80b7-d0edcaf97d6c",
"papermill": {
"duration": 0.078098,
"end_time": "2023-12-09T18:44:34.320354",
"exception": false,
"start_time": "2023-12-09T18:44:34.242256",
"status": "completed"
},
"requestMsgId": "cb9a17f9-5734-41c6-9018-c0635c61d8b3",
"showInput": true,
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter x. If that is not the expected value type, you can explicitly specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter y. If that is not the expected value type, you can explicitly specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.service.utils.instantiation: Created search space: SearchSpace(parameters=[RangeParameter(name='x', parameter_type=FLOAT, range=[-5.0, 10.0]), RangeParameter(name='y', parameter_type=FLOAT, range=[0.0, 15.0])], parameter_constraints=[]).\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.modelbridge.dispatch_utils: Using Models.BOTORCH_MODULAR since there are more ordered parameters than there are categories for the unordered categorical parameters.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.modelbridge.dispatch_utils: Calculating the number of remaining initialization trials based on num_initialization_trials=None max_initialization_trials=None num_tunable_parameters=2 num_trials=None use_batch_trials=False\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.modelbridge.dispatch_utils: calculated num_initialization_trials=5\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.modelbridge.dispatch_utils: num_completed_initialization_trials=0 num_remaining_initialization_trials=5\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.modelbridge.dispatch_utils: `verbose`, `disable_progbar`, and `jit_compile` are not yet supported when using `choose_generation_strategy` with ModularBoTorchModel, dropping these arguments.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[INFO 12-09 18:44:34] ax.modelbridge.dispatch_utils: Using Bayesian Optimization generation strategy: GenerationStrategy(name='Sobol+BoTorch', steps=[Sobol for 5 trials, BoTorch for subsequent trials]). Iterations after 5 will take longer to generate due to model-fitting.\n"
]
}
],
"source": [
"ax_client = AxClient()\n",
"ax_client.create_experiment(\n",
" parameters=[\n",
" {\"name\": \"x\", \"type\": \"range\", \"bounds\": [-5.0, 10.0]},\n",
" {\"name\": \"y\", \"type\": \"range\", \"bounds\": [0.0, 15.0]},\n",
" ],\n",
" # Sets max parallelism to 10 for all steps of the generation strategy.\n",
" choose_generation_strategy_kwargs={\"max_parallelism_override\": 10},\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "5565d88e",
"metadata": {
"customInput": null,
"customOutput": null,
"execution": {
"iopub.execute_input": "2023-12-09T18:44:34.446621Z",
"iopub.status.busy": "2023-12-09T18:44:34.445915Z",
"iopub.status.idle": "2023-12-09T18:44:34.451112Z",
"shell.execute_reply": "2023-12-09T18:44:34.450520Z"
},
"executionStartTime": 1690416064679,
"executionStopTime": 1690416064702,
"originalKey": "bc15d2cf-8ddc-4d66-83b6-7469cd15aa4d",
"papermill": {
"duration": 0.069993,
"end_time": "2023-12-09T18:44:34.452417",
"exception": false,
"start_time": "2023-12-09T18:44:34.382424",
"status": "completed"
},
"requestMsgId": "996c4bd3-b296-4cf9-8f95-cbf488639c2f",
"showInput": true,
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"[(5, 10), (-1, 10)]"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ax_client.get_max_parallelism() # Max parallelism is now 10 for all stages of the optimization."
]
}
],
"metadata": {
"fileHeader": "",
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.18"
},
"papermill": {
"default_parameters": {},
"duration": 229.165763,
"end_time": "2023-12-09T18:44:35.988578",
"environment_variables": {},
"exception": null,
"input_path": "/tmp/tmp.bwZKabbZig/Ax-main/tutorials/tune_cnn_service.ipynb",
"output_path": "/tmp/tmp.bwZKabbZig/Ax-main/tutorials/tune_cnn_service.ipynb",
"parameters": {},
"start_time": "2023-12-09T18:40:46.822815",
"version": "2.5.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}