dnvcoijcd

December 1, 2024

[1]: import datetime

print(f"Notebook last updated: {datetime.datetime.now()}")

Notebook last updated: 2024-10-09 09:52:25.428863

[2]: import torch

import torchvision

model = torchvision.models.resnet50()

[3]: model = torchvision.models.resnet50() # note: this could be any model

compiled_model = torch.compile(model)

[4]: device = "cuda" if torch.cuda.is_available() else "cpu"

[5]: model_weights = torchvision.models.ResNet50_Weights.IMAGENET1K_V2 # <- use the␣

↪latest weights (could also use .DEFAULT)

transforms = model_weights.transforms()

# Setup model
model = torchvision.models.resnet50(weights=model_weights)

# Count the number of parameters in the model

total_params = sum(
param.numel() for param in model.parameters() # <- all params
# param.numel() for param in model.parameters() if param.requires_grad␣
↪# <- only trainable params

print(f"Total parameters of model: {total_params} (the more parameters, the␣

↪more GPU memory the model will use, the more *relative* of a speedup you'll␣

↪get)")

print(f"Model transforms:\n{transforms}")

Downloading: "https://download.pytorch.org/models/resnet50-11ad3fa6.pth" to
/root/.cache/torch/hub/checkpoints/resnet50-11ad3fa6.pth
100%|����������| 97.8M/97.8M [00:00<00:00, 143MB/s]

1
 Total parameters of model: 25557032 (the more parameters, the more GPU memory
the model will use, the more *relative* of a speedup you'll get)
Model transforms:
ImageClassification(
crop_size=[224]
resize_size=[232]
mean=[0.485, 0.456, 0.406]
std=[0.229, 0.224, 0.225]
interpolation=InterpolationMode.BILINEAR
)

[6]: def create_model(num_classes=10):

"""
Creates a ResNet50 model with the latest weights and transforms via␣
↪torchvision.

"""
model_weights = torchvision.models.ResNet50_Weights.IMAGENET1K_V2
transforms = model_weights.transforms()
model = torchvision.models.resnet50(weights=model_weights)

# Adjust the number of output features in model to match the number of␣
↪classes in the dataset

model.fc = torch.nn.Linear(in_features=2048,
out_features=num_classes)
return model, transforms

model, transforms = create_model()

[7]: BATCH_SIZE = 32
# IMAGE_SIZE = 128

[8]: # transforms.crop_size = IMAGE_SIZE

# transforms.resize_size = IMAGE_SIZE
# print(f"Updated data transforms:\n{transforms}")

[9]: train_dataset = torchvision.datasets.CIFAR10(root='.',

train=True,
download=True,
transform=transforms)

test_dataset = torchvision.datasets.CIFAR10(root='.',
train=False, # want the test split
download=True,
transform=transforms)

# Get the lengths of the datasets

train_len = len(train_dataset)

2
 test_len = len(test_dataset)

print(f"[INFO] Train dataset length: {train_len}")

print(f"[INFO] Test dataset length: {test_len}")

Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to
./cifar-10-python.tar.gz
100%|����������| 170498071/170498071 [00:05<00:00, 29084790.35it/s]
Extracting ./cifar-10-python.tar.gz to .
Files already downloaded and verified
[INFO] Train dataset length: 50000
[INFO] Test dataset length: 10000

[10]: from torch.utils.data import DataLoader

# Create DataLoaders
import os
NUM_WORKERS = os.cpu_count() # <- use all available CPU cores (this number can␣
↪be tweaked through experimentation but generally more workers means faster␣

↪dataloading from CPU to GPU)

train_dataloader = DataLoader(dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=NUM_WORKERS)

test_dataloader = DataLoader(dataset=test_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=NUM_WORKERS)

# Print details
print(f"Train dataloader length: {len(train_dataloader)} batches of size␣
↪{BATCH_SIZE}")

print(f"Test dataloader length: {len(test_dataloader)} batches of size␣

↪{BATCH_SIZE}")

print(f"Using number of workers: {NUM_WORKERS} (generally more workers means␣

↪faster dataloading from CPU to GPU)")

Train dataloader length: 1563 batches of size 32

Test dataloader length: 313 batches of size 32
Using number of workers: 4 (generally more workers means faster dataloading from
CPU to GPU)

[11]: import time

from tqdm.auto import tqdm
from typing import Dict, List, Tuple

3
def train_step(epoch: int,
model: torch.nn.Module,
dataloader: torch.utils.data.DataLoader,
loss_fn: torch.nn.Module,
optimizer: torch.optim.Optimizer,
device: torch.device,
disable_progress_bar: bool = False) -> Tuple[float, float]:
"""Trains a PyTorch model for a single epoch.

Turns a target PyTorch model to training mode and then

runs through all of the required training steps (forward
pass, loss calculation, optimizer step).

Args:
model: A PyTorch model to be trained.
dataloader: A DataLoader instance for the model to be trained on.
loss_fn: A PyTorch loss function to minimize.
optimizer: A PyTorch optimizer to help minimize the loss function.
device: A target device to compute on (e.g. "cuda" or "cpu").

Returns:
A tuple of training loss and training accuracy metrics.
In the form (train_loss, train_accuracy). For example:

(0.1112, 0.8743)
"""
# Put model in train mode
model.train()

# Setup train loss and train accuracy values

train_loss, train_acc = 0, 0

# Loop through data loader data batches

progress_bar = tqdm(
enumerate(dataloader),
desc=f"Training Epoch {epoch}",
total=len(dataloader),
disable=disable_progress_bar
)

for batch, (X, y) in progress_bar:

# Send data to target device
X, y = X.to(device), y.to(device)

# 1. Forward pass
y_pred = model(X)

4
 # 2. Calculate and accumulate loss
loss = loss_fn(y_pred, y)
train_loss += loss.item()

# 3. Optimizer zero grad

optimizer.zero_grad()

# 4. Loss backward
loss.backward()

# 5. Optimizer step
optimizer.step()

# Calculate and accumulate accuracy metrics across all batches

y_pred_class = torch.argmax(torch.softmax(y_pred, dim=1), dim=1)
train_acc += (y_pred_class == y).sum().item()/len(y_pred)

# Update progress bar

progress_bar.set_postfix(
{
"train_loss": train_loss / (batch + 1),
"train_acc": train_acc / (batch + 1),
}
)

# Adjust metrics to get average loss and accuracy per batch

train_loss = train_loss / len(dataloader)
train_acc = train_acc / len(dataloader)
return train_loss, train_acc

def test_step(epoch: int,

model: torch.nn.Module,
dataloader: torch.utils.data.DataLoader,
loss_fn: torch.nn.Module,
device: torch.device,
disable_progress_bar: bool = False) -> Tuple[float, float]:
"""Tests a PyTorch model for a single epoch.

Turns a target PyTorch model to "eval" mode and then performs

a forward pass on a testing dataset.

Args:
model: A PyTorch model to be tested.
dataloader: A DataLoader instance for the model to be tested on.
loss_fn: A PyTorch loss function to calculate loss on the test data.

5
 device: A target device to compute on (e.g. "cuda" or "cpu").

Returns:
A tuple of testing loss and testing accuracy metrics.
In the form (test_loss, test_accuracy). For example:

(0.0223, 0.8985)
"""
# Put model in eval mode
model.eval()

# Setup test loss and test accuracy values

test_loss, test_acc = 0, 0

# Loop through data loader data batches

progress_bar = tqdm(
enumerate(dataloader),
desc=f"Testing Epoch {epoch}",
total=len(dataloader),
disable=disable_progress_bar
)

# Turn on inference context manager

with torch.no_grad(): # no_grad() required for PyTorch 2.0, I found some␣
↪errors with `torch.inference_mode()`, please let me know if this is not the␣

↪case

# Loop through DataLoader batches

for batch, (X, y) in progress_bar:
# Send data to target device
X, y = X.to(device), y.to(device)

# 1. Forward pass
test_pred_logits = model(X)

# 2. Calculate and accumulate loss

loss = loss_fn(test_pred_logits, y)
test_loss += loss.item()

# Calculate and accumulate accuracy

test_pred_labels = test_pred_logits.argmax(dim=1)
test_acc += ((test_pred_labels == y).sum().item()/
↪len(test_pred_labels))

# Update progress bar

progress_bar.set_postfix(
{
"test_loss": test_loss / (batch + 1),

6
 "test_acc": test_acc / (batch + 1),
}
)

# Adjust metrics to get average loss and accuracy per batch

test_loss = test_loss / len(dataloader)
test_acc = test_acc / len(dataloader)
return test_loss, test_acc

def train(model: torch.nn.Module,

train_dataloader: torch.utils.data.DataLoader,
test_dataloader: torch.utils.data.DataLoader,
optimizer: torch.optim.Optimizer,
loss_fn: torch.nn.Module,
epochs: int,
device: torch.device,
disable_progress_bar: bool = False) -> Dict[str, List]:
"""Trains and tests a PyTorch model.

Passes a target PyTorch models through train_step() and test_step()

functions for a number of epochs, training and testing the model
in the same epoch loop.

Calculates, prints and stores evaluation metrics throughout.

Args:
model: A PyTorch model to be trained and tested.
train_dataloader: A DataLoader instance for the model to be trained on.
test_dataloader: A DataLoader instance for the model to be tested on.
optimizer: A PyTorch optimizer to help minimize the loss function.
loss_fn: A PyTorch loss function to calculate loss on both datasets.
epochs: An integer indicating how many epochs to train for.
device: A target device to compute on (e.g. "cuda" or "cpu").

Returns:
A dictionary of training and testing loss as well as training and
testing accuracy metrics. Each metric has a value in a list for
each epoch.
In the form: {train_loss: [...],
train_acc: [...],
test_loss: [...],
test_acc: [...]}
For example if training for epochs=2:
{train_loss: [2.0616, 1.0537],
train_acc: [0.3945, 0.3945],
test_loss: [1.2641, 1.5706],
test_acc: [0.3400, 0.2973]}

7
"""
# Create empty results dictionary
results = {"train_loss": [],
"train_acc": [],
"test_loss": [],
"test_acc": [],
"train_epoch_time": [],
"test_epoch_time": []
}

# Loop through training and testing steps for a number of epochs

for epoch in tqdm(range(epochs), disable=disable_progress_bar):

# Perform training step and time it

train_epoch_start_time = time.time()
train_loss, train_acc = train_step(epoch=epoch,
model=model,
dataloader=train_dataloader,
loss_fn=loss_fn,
optimizer=optimizer,
device=device,
␣
↪disable_progress_bar=disable_progress_bar)

train_epoch_end_time = time.time()
train_epoch_time = train_epoch_end_time - train_epoch_start_time

# Perform testing step and time it

test_epoch_start_time = time.time()
test_loss, test_acc = test_step(epoch=epoch,
model=model,
dataloader=test_dataloader,
loss_fn=loss_fn,
device=device,
disable_progress_bar=disable_progress_bar)
test_epoch_end_time = time.time()
test_epoch_time = test_epoch_end_time - test_epoch_start_time

# Print out what's happening

print(
f"Epoch: {epoch+1} | "
f"train_loss: {train_loss:.4f} | "
f"train_acc: {train_acc:.4f} | "
f"test_loss: {test_loss:.4f} | "
f"test_acc: {test_acc:.4f} | "
f"train_epoch_time: {train_epoch_time:.4f} | "
f"test_epoch_time: {test_epoch_time:.4f}"
)

8
 # Update results dictionary
results["train_loss"].append(train_loss)
results["train_acc"].append(train_acc)
results["test_loss"].append(test_loss)
results["test_acc"].append(test_acc)
results["train_epoch_time"].append(train_epoch_time)
results["test_epoch_time"].append(test_epoch_time)

# Return the filled results at the end of the epochs

return results

[12]: # Set the number of epochs as a constant

NUM_EPOCHS = 5

# Set the learning rate as a constant (this can be changed to get better␣
↪results but for now we're just focused on time)

LEARNING_RATE = 0.003

[13]: model, transforms = create_model()

model.to(device)

# Create loss function and optimizer

loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=LEARNING_RATE)

# Train model and track results

single_run_no_compile_results = train(model=model,
train_dataloader=train_dataloader,
test_dataloader=test_dataloader,
loss_fn=loss_fn,
optimizer=optimizer,
epochs=NUM_EPOCHS,
device=device)

0%| | 0/5 [00:00<?, ?it/s]

Training Epoch 0: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 0: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 1 | train_loss: 1.2766 | train_acc: 0.5377 | test_loss: 0.9844 |
test_acc: 0.6513 | train_epoch_time: 279.1513 | test_epoch_time: 19.1076
Training Epoch 1: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 1: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 2 | train_loss: 0.8198 | train_acc: 0.7109 | test_loss: 0.7182 |

9
 test_acc: 0.7462 | train_epoch_time: 277.8545 | test_epoch_time: 19.1254
Training Epoch 2: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 2: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 3 | train_loss: 0.5976 | train_acc: 0.7936 | test_loss: 0.5833 |
test_acc: 0.8000 | train_epoch_time: 278.0484 | test_epoch_time: 19.2897
Training Epoch 3: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 3: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 4 | train_loss: 0.4626 | train_acc: 0.8391 | test_loss: 0.5080 |
test_acc: 0.8276 | train_epoch_time: 278.0358 | test_epoch_time: 19.2620
Training Epoch 4: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 4: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 5 | train_loss: 0.3628 | train_acc: 0.8744 | test_loss: 0.5503 |
test_acc: 0.8149 | train_epoch_time: 277.7094 | test_epoch_time: 19.0795

[14]: import torch._dynamo

torch._dynamo.config.suppress_errors = True
import logging
import torch

# Set logging level to WARNING to suppress lower-level logs

logging.basicConfig(level=logging.WARNING)

# Your model and training code here

[15]: # Create model and transforms

model, transforms = create_model()
model.to(device)

# Create loss function and optimizer

loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=LEARNING_RATE)

# Compile the model and time how long it takes

compile_start_time = time.time()

### New in PyTorch 2.x ###

compiled_model = torch.compile(model)
##########################

compile_end_time = time.time()
compile_time = compile_end_time - compile_start_time

10
print(f"Time to compile: {compile_time} | Note: The first time you compile your␣
↪model, the first few epochs will be slower than subsequent runs.")

# Train the compiled model

single_run_compile_results = train(model=compiled_model,
train_dataloader=train_dataloader,
test_dataloader=test_dataloader,
loss_fn=loss_fn,
optimizer=optimizer,
epochs=NUM_EPOCHS,
device=device)

Time to compile: 0.008243560791015625 | Note: The first time you compile your
model, the first few epochs will be slower than subsequent runs.
0%| | 0/5 [00:00<?, ?it/s]
Training Epoch 0: 0%| | 0/1563 [00:00<?, ?it/s]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] WON'T
CONVERT forward /opt/conda/lib/python3.10/site-
packages/torchvision/models/resnet.py line 284
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] due
to:
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
Traceback (most recent call last):
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 948, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
result = self._inner_convert(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 472, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return _compile(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_utils_internal.py", line
84, in wrapper_function
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return StrobelightCompileTimeProfiler.profile_compile_time(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_strobelight/compile_time_profiler.py", line 129, in
profile_compile_time
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner

11
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 817, in _compile
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
guarded_code = compile_inner(code, one_graph, hooks, transform)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 636, in compile_inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
out_code = transform_code_object(code, transform)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/bytecode_transformation.py", line 1185, in
transform_code_object
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
transformations(instructions, code_options)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 178, in _fn
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 582, in transform
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
tracer.run()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2451, in run
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
super().run()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 893, in run
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
while self.step():
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 805, in step
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.dispatch_table[inst.opcode](self, inst)

12
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2642, in RETURN_VALUE
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._return(inst)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2627, in _return
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.output.compile_subgraph(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1098, in compile_subgraph
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1318, in compile_and_call_fx_graph
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = self.call_user_compiler(gm)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1409, in call_user_compiler
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1390, in call_user_compiler
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = compiler_fn(gm, self.example_inputs())
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/repro/after_dynamo.py", line 129, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_gm = compiler_fn(gm, example_inputs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/__init__.py", line 1951, in
__call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]

13
return compile_fx(model_, inputs_, config_patches=self.config)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1505, in compile_fx
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return aot_autograd(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/backends/common.py",
line 69, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 954, in aot_module_simplified
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, _ = create_aot_dispatcher_function(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 687, in create_aot_dispatcher_function
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, fw_metadata = compiler_fn(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line
461, in aot_dispatch_autograd
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fw_func = aot_config.fw_compiler(fw_module, adjusted_flat_args)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1410, in fw_compiler_base
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return inner_compile(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py",

14
line 84, in debug_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
inner_compiled_fn = compiler_fn(gm, example_inputs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/debug.py", line
304, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 527, in compile_fx_inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_graph = fx_codegen_and_compile(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 831, in fx_codegen_and_compile
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = graph.compile_to_fn()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1749, in compile_to_fn
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return self.compile_to_module().call
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1678, in compile_to_module

15
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1634, in codegen
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.scheduler = Scheduler(self.buffers)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in __init__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in <listcomp>
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1462, in create_scheduler_node
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return SchedulerNode(self, node)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 731, in __init__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._compute_attrs()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 742, in _compute_attrs
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
group_fn = self.scheduler.get_backend(self.node.get_device()).group_fn
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2663, in get_backend
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.backends[device] = self.create_backend(device)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2651, in create_backend
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise RuntimeError(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]

16
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
RuntimeError: Found Tesla P100-PCIE-16GB which is too old to be supported by the
triton GPU compiler, which is used as the backend. Triton only supports devices
of CUDA Capability >= 7.0, but your device is of CUDA capability 6.0
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] Set
TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
Traceback (most recent call last):
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 948, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
result = self._inner_convert(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 472, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return _compile(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_utils_internal.py", line
84, in wrapper_function
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return StrobelightCompileTimeProfiler.profile_compile_time(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_strobelight/compile_time_profiler.py", line 129, in
profile_compile_time
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 817, in _compile
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
guarded_code = compile_inner(code, one_graph, hooks, transform)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",

17
line 636, in compile_inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
out_code = transform_code_object(code, transform)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/bytecode_transformation.py", line 1185, in
transform_code_object
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
transformations(instructions, code_options)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 178, in _fn
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 582, in transform
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
tracer.run()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2451, in run
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
super().run()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 893, in run
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
while self.step():
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 805, in step
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.dispatch_table[inst.opcode](self, inst)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2642, in RETURN_VALUE
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._return(inst)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2627, in _return
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.output.compile_subgraph(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1098, in compile_subgraph
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]

18
self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1318, in compile_and_call_fx_graph
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = self.call_user_compiler(gm)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1409, in call_user_compiler
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1390, in call_user_compiler
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = compiler_fn(gm, self.example_inputs())
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/repro/after_dynamo.py", line 129, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_gm = compiler_fn(gm, example_inputs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/__init__.py", line 1951, in
__call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return compile_fx(model_, inputs_, config_patches=self.config)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1505, in compile_fx
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return aot_autograd(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/backends/common.py",
line 69, in __call__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]

19
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 954, in aot_module_simplified
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, _ = create_aot_dispatcher_function(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 687, in create_aot_dispatcher_function
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, fw_metadata = compiler_fn(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line
461, in aot_dispatch_autograd
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fw_func = aot_config.fw_compiler(fw_module, adjusted_flat_args)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1410, in fw_compiler_base
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return inner_compile(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py",
line 84, in debug_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
inner_compiled_fn = compiler_fn(gm, example_inputs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/debug.py", line
304, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner

20
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 527, in compile_fx_inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_graph = fx_codegen_and_compile(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 831, in fx_codegen_and_compile
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = graph.compile_to_fn()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1749, in compile_to_fn
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return self.compile_to_module().call
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1678, in compile_to_module
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1634, in codegen
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.scheduler = Scheduler(self.buffers)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",

21
line 1364, in __init__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in <listcomp>
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1462, in create_scheduler_node
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return SchedulerNode(self, node)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 731, in __init__
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._compute_attrs()
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 742, in _compute_attrs
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
group_fn = self.scheduler.get_backend(self.node.get_device()).group_fn
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2663, in get_backend
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.backends[device] = self.create_backend(device)
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2651, in create_backend
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise RuntimeError(
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
RuntimeError: Found Tesla P100-PCIE-16GB which is too old to be supported by the
triton GPU compiler, which is used as the backend. Triton only supports devices
of CUDA Capability >= 7.0, but your device is of CUDA capability 6.0
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009] Set
TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
W1009 10:17:50.416000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] WON'T
CONVERT _forward_impl /opt/conda/lib/python3.10/site-
packages/torchvision/models/resnet.py line 266
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] due
to:

22
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
Traceback (most recent call last):
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 948, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
result = self._inner_convert(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 472, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return _compile(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_utils_internal.py", line
84, in wrapper_function
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return StrobelightCompileTimeProfiler.profile_compile_time(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_strobelight/compile_time_profiler.py", line 129, in
profile_compile_time
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 817, in _compile
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
guarded_code = compile_inner(code, one_graph, hooks, transform)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 636, in compile_inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
out_code = transform_code_object(code, transform)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/bytecode_transformation.py", line 1185, in
transform_code_object
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
transformations(instructions, code_options)

23
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 178, in _fn
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 582, in transform
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
tracer.run()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2451, in run
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
super().run()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 893, in run
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
while self.step():
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 805, in step
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.dispatch_table[inst.opcode](self, inst)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2642, in RETURN_VALUE
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._return(inst)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2627, in _return
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.output.compile_subgraph(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1098, in compile_subgraph
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1318, in compile_and_call_fx_graph
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]

24
compiled_fn = self.call_user_compiler(gm)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1409, in call_user_compiler
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1390, in call_user_compiler
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = compiler_fn(gm, self.example_inputs())
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/repro/after_dynamo.py", line 129, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_gm = compiler_fn(gm, example_inputs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/__init__.py", line 1951, in
__call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return compile_fx(model_, inputs_, config_patches=self.config)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1505, in compile_fx
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return aot_autograd(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/backends/common.py",
line 69, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 954, in aot_module_simplified
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, _ = create_aot_dispatcher_function(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper

25
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 687, in create_aot_dispatcher_function
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, fw_metadata = compiler_fn(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line
461, in aot_dispatch_autograd
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fw_func = aot_config.fw_compiler(fw_module, adjusted_flat_args)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1410, in fw_compiler_base
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return inner_compile(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py",
line 84, in debug_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
inner_compiled_fn = compiler_fn(gm, example_inputs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/debug.py", line
304, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",

26
line 527, in compile_fx_inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_graph = fx_codegen_and_compile(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 831, in fx_codegen_and_compile
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = graph.compile_to_fn()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1749, in compile_to_fn
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return self.compile_to_module().call
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1678, in compile_to_module
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1634, in codegen
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.scheduler = Scheduler(self.buffers)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in __init__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in <listcomp>
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]

27
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1462, in create_scheduler_node
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return SchedulerNode(self, node)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 731, in __init__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._compute_attrs()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 742, in _compute_attrs
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
group_fn = self.scheduler.get_backend(self.node.get_device()).group_fn
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2663, in get_backend
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.backends[device] = self.create_backend(device)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2651, in create_backend
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise RuntimeError(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
RuntimeError: Found Tesla P100-PCIE-16GB which is too old to be supported by the
triton GPU compiler, which is used as the backend. Triton only supports devices
of CUDA Capability >= 7.0, but your device is of CUDA capability 6.0
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] Set
TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
Traceback (most recent call last):
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 948, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
result = self._inner_convert(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 472, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return _compile(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_utils_internal.py", line

28
84, in wrapper_function
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return StrobelightCompileTimeProfiler.profile_compile_time(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_strobelight/compile_time_profiler.py", line 129, in
profile_compile_time
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 817, in _compile
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
guarded_code = compile_inner(code, one_graph, hooks, transform)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 636, in compile_inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
out_code = transform_code_object(code, transform)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/bytecode_transformation.py", line 1185, in
transform_code_object
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
transformations(instructions, code_options)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 178, in _fn
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 582, in transform
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
tracer.run()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2451, in run
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]

29
super().run()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 893, in run
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
while self.step():
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 805, in step
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.dispatch_table[inst.opcode](self, inst)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2642, in RETURN_VALUE
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._return(inst)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2627, in _return
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.output.compile_subgraph(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1098, in compile_subgraph
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1318, in compile_and_call_fx_graph
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = self.call_user_compiler(gm)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1409, in call_user_compiler
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1390, in call_user_compiler

30
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = compiler_fn(gm, self.example_inputs())
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/repro/after_dynamo.py", line 129, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_gm = compiler_fn(gm, example_inputs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/__init__.py", line 1951, in
__call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return compile_fx(model_, inputs_, config_patches=self.config)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1505, in compile_fx
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return aot_autograd(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/backends/common.py",
line 69, in __call__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 954, in aot_module_simplified
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, _ = create_aot_dispatcher_function(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 687, in create_aot_dispatcher_function
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, fw_metadata = compiler_fn(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line
461, in aot_dispatch_autograd
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fw_func = aot_config.fw_compiler(fw_module, adjusted_flat_args)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]

31
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1410, in fw_compiler_base
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return inner_compile(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py",
line 84, in debug_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
inner_compiled_fn = compiler_fn(gm, example_inputs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/debug.py", line
304, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 527, in compile_fx_inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_graph = fx_codegen_and_compile(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 831, in fx_codegen_and_compile
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = graph.compile_to_fn()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line

32
1749, in compile_to_fn
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return self.compile_to_module().call
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1678, in compile_to_module
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1634, in codegen
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.scheduler = Scheduler(self.buffers)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in __init__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in <listcomp>
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1462, in create_scheduler_node
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return SchedulerNode(self, node)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 731, in __init__
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._compute_attrs()
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 742, in _compute_attrs
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
group_fn = self.scheduler.get_backend(self.node.get_device()).group_fn

33
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2663, in get_backend
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.backends[device] = self.create_backend(device)
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2651, in create_backend
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise RuntimeError(
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
RuntimeError: Found Tesla P100-PCIE-16GB which is too old to be supported by the
triton GPU compiler, which is used as the backend. Triton only supports devices
of CUDA Capability >= 7.0, but your device is of CUDA capability 6.0
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009] Set
TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
W1009 10:18:06.387000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] WON'T
CONVERT forward /opt/conda/lib/python3.10/site-
packages/torchvision/models/resnet.py line 143
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] due
to:
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
Traceback (most recent call last):
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 948, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
result = self._inner_convert(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 472, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return _compile(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_utils_internal.py", line
84, in wrapper_function
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return StrobelightCompileTimeProfiler.profile_compile_time(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_strobelight/compile_time_profiler.py", line 129, in
profile_compile_time
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwargs)

34
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 817, in _compile
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
guarded_code = compile_inner(code, one_graph, hooks, transform)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 636, in compile_inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
out_code = transform_code_object(code, transform)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/bytecode_transformation.py", line 1185, in
transform_code_object
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
transformations(instructions, code_options)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 178, in _fn
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 582, in transform
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
tracer.run()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2451, in run
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
super().run()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 893, in run
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
while self.step():
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 805, in step

35
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.dispatch_table[inst.opcode](self, inst)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2642, in RETURN_VALUE
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._return(inst)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2627, in _return
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.output.compile_subgraph(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1098, in compile_subgraph
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1318, in compile_and_call_fx_graph
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = self.call_user_compiler(gm)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1409, in call_user_compiler
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1390, in call_user_compiler
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = compiler_fn(gm, self.example_inputs())
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/repro/after_dynamo.py", line 129, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_gm = compiler_fn(gm, example_inputs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/__init__.py", line 1951, in

36
__call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return compile_fx(model_, inputs_, config_patches=self.config)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1505, in compile_fx
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return aot_autograd(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/backends/common.py",
line 69, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 954, in aot_module_simplified
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, _ = create_aot_dispatcher_function(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 687, in create_aot_dispatcher_function
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, fw_metadata = compiler_fn(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line
461, in aot_dispatch_autograd
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fw_func = aot_config.fw_compiler(fw_module, adjusted_flat_args)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1410, in fw_compiler_base
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return inner_compile(

37
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py",
line 84, in debug_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
inner_compiled_fn = compiler_fn(gm, example_inputs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/debug.py", line
304, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 527, in compile_fx_inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_graph = fx_codegen_and_compile(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 831, in fx_codegen_and_compile
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = graph.compile_to_fn()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1749, in compile_to_fn
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return self.compile_to_module().call
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]

38
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1678, in compile_to_module
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1634, in codegen
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.scheduler = Scheduler(self.buffers)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in __init__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in <listcomp>
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1462, in create_scheduler_node
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return SchedulerNode(self, node)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 731, in __init__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._compute_attrs()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 742, in _compute_attrs
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
group_fn = self.scheduler.get_backend(self.node.get_device()).group_fn
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2663, in get_backend
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.backends[device] = self.create_backend(device)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2651, in create_backend
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]

39
raise RuntimeError(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
RuntimeError: Found Tesla P100-PCIE-16GB which is too old to be supported by the
triton GPU compiler, which is used as the backend. Triton only supports devices
of CUDA Capability >= 7.0, but your device is of CUDA capability 6.0
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] Set
TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
Traceback (most recent call last):
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 948, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
result = self._inner_convert(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 472, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return _compile(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_utils_internal.py", line
84, in wrapper_function
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return StrobelightCompileTimeProfiler.profile_compile_time(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_strobelight/compile_time_profiler.py", line 129, in
profile_compile_time
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 817, in _compile
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
guarded_code = compile_inner(code, one_graph, hooks, transform)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)

40
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 636, in compile_inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
out_code = transform_code_object(code, transform)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/bytecode_transformation.py", line 1185, in
transform_code_object
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
transformations(instructions, code_options)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 178, in _fn
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/convert_frame.py",
line 582, in transform
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
tracer.run()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2451, in run
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
super().run()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 893, in run
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
while self.step():
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 805, in step
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.dispatch_table[inst.opcode](self, inst)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2642, in RETURN_VALUE
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._return(inst)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/symbolic_convert.py", line 2627, in _return
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.output.compile_subgraph(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",

41
line 1098, in compile_subgraph
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1318, in compile_and_call_fx_graph
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = self.call_user_compiler(gm)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1409, in call_user_compiler
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/output_graph.py",
line 1390, in call_user_compiler
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = compiler_fn(gm, self.example_inputs())
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_dynamo/repro/after_dynamo.py", line 129, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_gm = compiler_fn(gm, example_inputs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/__init__.py", line 1951, in
__call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return compile_fx(model_, inputs_, config_patches=self.config)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1505, in compile_fx
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return aot_autograd(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/backends/common.py",

42
line 69, in __call__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 954, in aot_module_simplified
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, _ = create_aot_dispatcher_function(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py",
line 687, in create_aot_dispatcher_function
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn, fw_metadata = compiler_fn(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-
packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line
461, in aot_dispatch_autograd
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fw_func = aot_config.fw_compiler(fw_module, adjusted_flat_args)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 1410, in fw_compiler_base
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return inner_compile(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/repro/after_aot.py",
line 84, in debug_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
inner_compiled_fn = compiler_fn(gm, example_inputs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/debug.py", line
304, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return fn(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)

43
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 527, in compile_fx_inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_graph = fx_codegen_and_compile(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/contextlib.py", line 79, in inner
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return func(*args, **kwds)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/compile_fx.py",
line 831, in fx_codegen_and_compile
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
compiled_fn = graph.compile_to_fn()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1749, in compile_to_fn
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return self.compile_to_module().call
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1678, in compile_to_module
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/graph.py", line
1634, in codegen
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.scheduler = Scheduler(self.buffers)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_dynamo/utils.py", line 231,
in time_wrapper
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] r
= func(*args, **kwargs)

44
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in __init__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1364, in <listcomp>
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.nodes = [self.create_scheduler_node(n) for n in nodes]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 1462, in create_scheduler_node
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
return SchedulerNode(self, node)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 731, in __init__
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self._compute_attrs()
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 742, in _compute_attrs
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
group_fn = self.scheduler.get_backend(self.node.get_device()).group_fn
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2663, in get_backend
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
self.backends[device] = self.create_backend(device)
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
File "/opt/conda/lib/python3.10/site-packages/torch/_inductor/scheduler.py",
line 2651, in create_backend
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
raise RuntimeError(
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
RuntimeError: Found Tesla P100-PCIE-16GB which is too old to be supported by the
triton GPU compiler, which is used as the backend. Triton only supports devices
of CUDA Capability >= 7.0, but your device is of CUDA capability 6.0
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009] Set
TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
W1009 10:18:07.560000 133648656836416 torch/_dynamo/convert_frame.py:1009]
Testing Epoch 0: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 1 | train_loss: 1.2649 | train_acc: 0.5410 | test_loss: 0.9524 |

45
 test_acc: 0.6595 | train_epoch_time: 312.0904 | test_epoch_time: 19.2846
Training Epoch 1: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 1: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 2 | train_loss: 0.7949 | train_acc: 0.7165 | test_loss: 0.8069 |
test_acc: 0.7214 | train_epoch_time: 278.1916 | test_epoch_time: 19.1748
Training Epoch 2: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 2: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 3 | train_loss: 0.5798 | train_acc: 0.7983 | test_loss: 0.5997 |
test_acc: 0.7912 | train_epoch_time: 278.1900 | test_epoch_time: 19.2733
Training Epoch 3: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 3: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 4 | train_loss: 0.4472 | train_acc: 0.8447 | test_loss: 0.5496 |
test_acc: 0.8170 | train_epoch_time: 278.1507 | test_epoch_time: 19.3345
Training Epoch 4: 0%| | 0/1563 [00:00<?, ?it/s]
Testing Epoch 4: 0%| | 0/313 [00:00<?, ?it/s]
Epoch: 5 | train_loss: 0.3482 | train_acc: 0.8785 | test_loss: 0.4666 |
test_acc: 0.8416 | train_epoch_time: 278.4868 | test_epoch_time: 19.2365

[16]: # Turn experiment results into dataframes

import pandas as pd
single_run_no_compile_results_df = pd.DataFrame(single_run_no_compile_results)
single_run_compile_results_df = pd.DataFrame(single_run_compile_results)

[17]: # Check out the head of one of the results dataframes

single_run_no_compile_results_df.head()

[17]: train_loss train_acc test_loss test_acc train_epoch_time \

0 1.276583 0.537728 0.984366 0.651258 279.151285
1 0.819843 0.710913 0.718186 0.746206 277.854511
2 0.597562 0.793606 0.583254 0.800020 278.048392
3 0.462560 0.839091 0.507981 0.827576 278.035839
4 0.362793 0.874420 0.550340 0.814896 277.709445

test_epoch_time
0 19.107635
1 19.125370
2 19.289747
3 19.262027
4 19.079486

46
[18]: # Create filename to save the results
DATASET_NAME = "CIFAR10"
MODEL_NAME = "ResNet50"
IMAGE_SIZE = 128

[19]: import matplotlib.pyplot as plt

import numpy as np

def plot_mean_epoch_times(non_compiled_results: pd.DataFrame,

compiled_results: pd.DataFrame,
multi_runs: bool=False,
num_runs: int=0,
save: bool=False,
save_path: str="",
dataset_name: str=DATASET_NAME,
model_name: str=MODEL_NAME,
num_epochs: int=NUM_EPOCHS,
image_size: int=IMAGE_SIZE,
batch_size: int=BATCH_SIZE) -> plt.figure:

# Get the mean epoch times from the non-compiled models

mean_train_epoch_time = non_compiled_results.train_epoch_time.mean()
mean_test_epoch_time = non_compiled_results.test_epoch_time.mean()
mean_results = [mean_train_epoch_time, mean_test_epoch_time]

# Get the mean epoch times from the compiled models

mean_compile_train_epoch_time = compiled_results.train_epoch_time.mean()
mean_compile_test_epoch_time = compiled_results.test_epoch_time.mean()
mean_compile_results = [mean_compile_train_epoch_time,␣
↪mean_compile_test_epoch_time]

# Calculate the percentage difference between the mean compile and␣

↪non-compile train epoch times
train_epoch_time_diff = mean_compile_train_epoch_time -␣
↪mean_train_epoch_time

train_epoch_time_diff_percent = (train_epoch_time_diff /␣
↪mean_train_epoch_time) * 100

# Calculate the percentage difference between the mean compile and␣

↪non-compile test epoch times

test_epoch_time_diff = mean_compile_test_epoch_time - mean_test_epoch_time

test_epoch_time_diff_percent = (test_epoch_time_diff /␣
↪mean_test_epoch_time) * 100

# Print the mean difference percentages

print(f"Mean train epoch time difference:␣
↪{round(train_epoch_time_diff_percent, 3)}% (negative means faster)")

47
 print(f"Mean test epoch time difference:␣
↪{round(test_epoch_time_diff_percent, 3)}% (negative means faster)")

# Create a bar plot of the mean train and test epoch time for both compiled␣
↪and non-compiled models
plt.figure(figsize=(10, 7))
width = 0.3
x_indicies = np.arange(len(mean_results))

plt.bar(x=x_indicies, height=mean_results, width=width,␣

↪label="non_compiled_results")
plt.bar(x=x_indicies + width, height=mean_compile_results, width=width,␣
↪label="compiled_results")

plt.xticks(x_indicies + width / 2, ("Train Epoch", "Test Epoch"))

plt.ylabel("Mean epoch time (seconds, lower is better)")

# Create the title based on the parameters passed to the function

if multi_runs:
plt.suptitle("Multiple run results")
plt.title(f"Epochs: {num_epochs} ({num_runs} runs) | Data:␣
↪{dataset_name} | Model: {model_name} | Image size: {image_size} | Batch size:

↪ {batch_size}")

else:
plt.suptitle("Single run results")
plt.title(f"Epochs: {num_epochs} | Data: {dataset_name} | Model:␣
↪{model_name} | Image size: {image_size} | Batch size: {batch_size}")

plt.legend()

# Save the figure

if save:
assert save_path != "", "Please specify a save path to save the model␣
↪figure to via the save_path parameter."

plt.savefig(save_path)
print(f"[INFO] Plot saved to {save_path}")

[20]: plot_mean_epoch_times(non_compiled_results=single_run_no_compile_results_df,
compiled_results=single_run_compile_results_df,
multi_runs=False,
save=True,
save_path="epoch_times_comparison.png")

Mean train epoch time difference: 2.467% (negative means faster)

Mean test epoch time difference: 0.458% (negative means faster)
[INFO] Plot saved to epoch_times_comparison.png

48
[21]: def create_and_train_non_compiled_model(epochs=NUM_EPOCHS,
learning_rate=LEARNING_RATE,
disable_progress_bar=False):
"""
Create and train a non-compiled PyTorch model.
"""
model, _ = create_model()
model.to(device)

loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=learning_rate)

results = train(model=model,
train_dataloader=train_dataloader,
test_dataloader=test_dataloader,
loss_fn=loss_fn,
optimizer=optimizer,
epochs=epochs,
device=device,

49
 disable_progress_bar=disable_progress_bar)
return results

def create_compiled_model():
"""
Create a compiled PyTorch model and return it.
"""
model, _ = create_model()
model.to(device)

compile_start_time = time.time()
### New in PyTorch 2.x ###
compiled_model = torch.compile(model)
##########################
compile_end_time = time.time()

compile_time = compile_end_time - compile_start_time

print(f"Time to compile: {compile_time} | Note: The first time you compile␣

↪your model, the first few epochs will be slower than subsequent runs.")
return compiled_model

def train_compiled_model(model=compiled_model,
epochs=NUM_EPOCHS,
learning_rate=LEARNING_RATE,
disable_progress_bar=False):
"""
Train a compiled model and return the results.
"""
loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(compiled_model.parameters(),
lr=learning_rate)

compile_results = train(model=model,
train_dataloader=train_dataloader,
test_dataloader=test_dataloader,
loss_fn=loss_fn,
optimizer=optimizer,
epochs=epochs,
device=device,
disable_progress_bar=disable_progress_bar)

return compile_results

[22]: # Run non-compiled model for multiple runs

NUM_RUNS = 3
NUM_EPOCHS = 5

50
# Create an empty list to store multiple run results
non_compile_results_multiple_runs = []

# Run non-compiled model for multiple runs

for i in tqdm(range(NUM_RUNS)):
print(f"[INFO] Run {i+1} of {NUM_RUNS} for non-compiled model")
results = create_and_train_non_compiled_model(epochs=NUM_EPOCHS,␣
↪disable_progress_bar=True)

non_compile_results_multiple_runs.append(results)

0%| | 0/3 [00:00<?, ?it/s]

[INFO] Run 1 of 3 for non-compiled model
Epoch: 1 | train_loss: 1.2556 | train_acc: 0.5457 | test_loss: 1.0357 |
test_acc: 0.6371 | train_epoch_time: 275.4204 | test_epoch_time: 18.7562
Epoch: 2 | train_loss: 0.7797 | train_acc: 0.7268 | test_loss: 0.8390 |
test_acc: 0.7165 | train_epoch_time: 275.3036 | test_epoch_time: 18.7747
Epoch: 3 | train_loss: 0.5707 | train_acc: 0.8007 | test_loss: 0.6054 |
test_acc: 0.7900 | train_epoch_time: 275.1358 | test_epoch_time: 18.6967
Epoch: 4 | train_loss: 0.4467 | train_acc: 0.8454 | test_loss: 0.5556 |
test_acc: 0.8143 | train_epoch_time: 275.3187 | test_epoch_time: 18.7088
Epoch: 5 | train_loss: 0.3449 | train_acc: 0.8795 | test_loss: 0.5036 |
test_acc: 0.8281 | train_epoch_time: 275.7380 | test_epoch_time: 18.7181
[INFO] Run 2 of 3 for non-compiled model
Epoch: 1 | train_loss: 1.1722 | train_acc: 0.5774 | test_loss: 0.9447 |
test_acc: 0.6719 | train_epoch_time: 276.4018 | test_epoch_time: 18.8533
Epoch: 2 | train_loss: 0.7235 | train_acc: 0.7491 | test_loss: 0.6738 |
test_acc: 0.7642 | train_epoch_time: 275.7931 | test_epoch_time: 18.6741
Epoch: 3 | train_loss: 0.5347 | train_acc: 0.8148 | test_loss: 0.6051 |
test_acc: 0.7902 | train_epoch_time: 275.4977 | test_epoch_time: 18.6606
Epoch: 4 | train_loss: 0.4226 | train_acc: 0.8544 | test_loss: 0.5377 |
test_acc: 0.8188 | train_epoch_time: 275.8198 | test_epoch_time: 18.8487
Epoch: 5 | train_loss: 0.3219 | train_acc: 0.8887 | test_loss: 0.4748 |
test_acc: 0.8401 | train_epoch_time: 275.7677 | test_epoch_time: 18.5576
[INFO] Run 3 of 3 for non-compiled model
Epoch: 1 | train_loss: 1.2070 | train_acc: 0.5657 | test_loss: 1.0751 |
test_acc: 0.6214 | train_epoch_time: 275.0426 | test_epoch_time: 18.6480
Epoch: 2 | train_loss: 0.7635 | train_acc: 0.7330 | test_loss: 0.6799 |
test_acc: 0.7697 | train_epoch_time: 275.4310 | test_epoch_time: 18.6913
Epoch: 3 | train_loss: 0.5625 | train_acc: 0.8046 | test_loss: 0.5800 |
test_acc: 0.8008 | train_epoch_time: 275.7100 | test_epoch_time: 18.8039
Epoch: 4 | train_loss: 0.4346 | train_acc: 0.8505 | test_loss: 0.4878 |
test_acc: 0.8352 | train_epoch_time: 275.5290 | test_epoch_time: 18.6628
Epoch: 5 | train_loss: 0.3325 | train_acc: 0.8853 | test_loss: 0.4832 |
test_acc: 0.8402 | train_epoch_time: 275.4497 | test_epoch_time: 18.7118

51
[23]: # Go through non_compile_results_multiple_runs and create a dataframe for each␣
↪run then concatenate them together

non_compile_results_dfs = []
for result in non_compile_results_multiple_runs:
result_df = pd.DataFrame(result)
non_compile_results_dfs.append(result_df)
non_compile_results_multiple_runs_df = pd.concat(non_compile_results_dfs)

# Get the averages across the multiple runs

non_compile_results_multiple_runs_df = non_compile_results_multiple_runs_df.
↪groupby(non_compile_results_multiple_runs_df.index).mean()

non_compile_results_multiple_runs_df

[23]: train_loss train_acc test_loss test_acc train_epoch_time \

0 1.211576 0.562947 1.018488 0.643470 275.621620
1 0.755534 0.736278 0.730930 0.750100 275.509225
2 0.555968 0.806702 0.596793 0.793697 275.447823
3 0.434654 0.850121 0.527037 0.822750 275.555811
4 0.333092 0.884510 0.487204 0.836096 275.651826

test_epoch_time
0 18.752528
1 18.713391
2 18.720411
3 18.740115
4 18.662506

[24]: # Create compiled model

compiled_model = create_compiled_model()

# Create an empty list to store compiled model results

compiled_results_multiple_runs = []

# Run compiled model for multiple runs

for i in tqdm(range(NUM_RUNS)):
print(f"[INFO] Run {i+1} of {NUM_RUNS} for compiled model")
# Train the compiled model (note: the model will only be compiled once and␣
↪then re-used for subsequent runs)

results = train_compiled_model(model=compiled_model, epochs=NUM_EPOCHS,␣

↪disable_progress_bar=True)

compiled_results_multiple_runs.append(results)

Time to compile: 0.0017609596252441406 | Note: The first time you compile your
model, the first few epochs will be slower than subsequent runs.
0%| | 0/3 [00:00<?, ?it/s]
[INFO] Run 1 of 3 for compiled model

52
 Epoch: 1 | train_loss: 1.1586 | train_acc: 0.5819 | test_loss: 0.9144 |
test_acc: 0.6745 | train_epoch_time: 275.1540 | test_epoch_time: 18.8432
Epoch: 2 | train_loss: 0.7052 | train_acc: 0.7541 | test_loss: 0.7374 |
test_acc: 0.7412 | train_epoch_time: 275.5065 | test_epoch_time: 18.8171
Epoch: 3 | train_loss: 0.5345 | train_acc: 0.8161 | test_loss: 0.5602 |
test_acc: 0.8060 | train_epoch_time: 275.5504 | test_epoch_time: 18.6937
Epoch: 4 | train_loss: 0.4158 | train_acc: 0.8557 | test_loss: 0.4743 |
test_acc: 0.8402 | train_epoch_time: 275.2414 | test_epoch_time: 18.7959
Epoch: 5 | train_loss: 0.3202 | train_acc: 0.8889 | test_loss: 0.4516 |
test_acc: 0.8506 | train_epoch_time: 275.4432 | test_epoch_time: 18.7335
[INFO] Run 2 of 3 for compiled model
Epoch: 1 | train_loss: 0.2525 | train_acc: 0.9108 | test_loss: 0.4858 |
test_acc: 0.8447 | train_epoch_time: 275.5786 | test_epoch_time: 18.6815
Epoch: 2 | train_loss: 0.1728 | train_acc: 0.9408 | test_loss: 0.4373 |
test_acc: 0.8633 | train_epoch_time: 275.7465 | test_epoch_time: 18.7751
Epoch: 3 | train_loss: 0.1230 | train_acc: 0.9580 | test_loss: 0.4815 |
test_acc: 0.8610 | train_epoch_time: 275.4566 | test_epoch_time: 18.6592
Epoch: 4 | train_loss: 0.0984 | train_acc: 0.9656 | test_loss: 0.4519 |
test_acc: 0.8682 | train_epoch_time: 275.3873 | test_epoch_time: 18.7133
Epoch: 5 | train_loss: 0.0784 | train_acc: 0.9727 | test_loss: 0.5117 |
test_acc: 0.8648 | train_epoch_time: 275.7526 | test_epoch_time: 18.6351
[INFO] Run 3 of 3 for compiled model
Epoch: 1 | train_loss: 0.0763 | train_acc: 0.9730 | test_loss: 0.5565 |
test_acc: 0.8579 | train_epoch_time: 275.8387 | test_epoch_time: 18.7854
Epoch: 2 | train_loss: 0.0653 | train_acc: 0.9779 | test_loss: 0.5248 |
test_acc: 0.8709 | train_epoch_time: 276.3325 | test_epoch_time: 18.7161
Epoch: 3 | train_loss: 0.0523 | train_acc: 0.9819 | test_loss: 0.5601 |
test_acc: 0.8627 | train_epoch_time: 276.0606 | test_epoch_time: 18.6342
Epoch: 4 | train_loss: 0.0494 | train_acc: 0.9832 | test_loss: 0.5629 |
test_acc: 0.8719 | train_epoch_time: 275.8713 | test_epoch_time: 18.6312
Epoch: 5 | train_loss: 0.0455 | train_acc: 0.9846 | test_loss: 0.6024 |
test_acc: 0.8647 | train_epoch_time: 275.7839 | test_epoch_time: 18.7164

[25]: # Go through compile_results_multiple_runs and create a dataframe for each run␣

↪then concatenate them together

compile_results_dfs = []
for result in compiled_results_multiple_runs:
result_df = pd.DataFrame(result)
compile_results_dfs.append(result_df)
compile_results_multiple_runs_df = pd.concat(compile_results_dfs)

# Get the averages across the multiple runs

compile_results_multiple_runs_df = compile_results_multiple_runs_df.
↪groupby(compile_results_multiple_runs_df.index).mean() # .index = groupby␣

↪the epoch number

compile_results_multiple_runs_df

53
[25]: train_loss train_acc test_loss test_acc train_epoch_time \
0 0.495794 0.821870 0.652214 0.792399 275.523742
1 0.314464 0.890962 0.566510 0.825146 275.861858
2 0.236588 0.918646 0.533929 0.843251 275.689193
3 0.187875 0.934828 0.496378 0.860091 275.500034
4 0.148041 0.948723 0.521926 0.860057 275.659885

test_epoch_time
0 18.770048
1 18.769439
2 18.662356
3 18.713465
4 18.695021

[29]: # Plot the mean epoch times for experiment 3 and 4

plot_mean_epoch_times(non_compiled_results=non_compile_results_multiple_runs_df,
compiled_results=compile_results_multiple_runs_df,
multi_runs=True,
num_runs=NUM_RUNS,
save=True,
save_path="/kaggle/working/") # Specify your desired␣
↪save path here

Mean train epoch time difference: 0.033% (negative means faster)

Mean test epoch time difference: 0.023% (negative means faster)
[INFO] Plot saved to /kaggle/working/

54
[ ]:

55