Skip to content
import warnings

from rich import print
from rich.pretty import pprint
from sklearn.model_selection import train_test_split

from pytorch_tabular.utils import make_mixed_dataset
data, cat_col_names, num_col_names = make_mixed_dataset(
    task="classification", n_samples=10000, n_features=20, n_categories=4
)

Importing the Library

from pytorch_tabular import TabularModel
from pytorch_tabular.models import (
    CategoryEmbeddingModelConfig,
)
from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig, ExperimentConfig
from pytorch_tabular.models.common.heads import LinearHeadConfig

Hyperparameter Tuning

train, test = train_test_split(data, random_state=42)
train, val = train_test_split(train, random_state=42)
data_config = DataConfig(
    target=[
        "target"
    ],  # target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
    continuous_cols=num_col_names,
    categorical_cols=cat_col_names,
)
trainer_config = TrainerConfig(
    batch_size=1024,
    max_epochs=100,
    early_stopping="valid_loss",  # Monitor valid_loss for early stopping
    early_stopping_mode="min",  # Set the mode as min because for val_loss, lower is better
    early_stopping_patience=5,  # No. of epochs of degradation training will wait before terminating
    checkpoints="valid_loss",  # Save best checkpoint monitoring val_loss
    load_best=True,  # After training, load the best checkpoint
    progress_bar="none",  # Turning off Progress bar
    trainer_kwargs=dict(enable_model_summary=False),  # Turning off model summary
)
optimizer_config = OptimizerConfig()

head_config = LinearHeadConfig(
    layers="", dropout=0.1, initialization="kaiming"  # No additional layer in head, just a mapping layer to output_dim
).__dict__  # Convert to dict to pass to the model config (OmegaConf doesn't accept objects)

model_config = CategoryEmbeddingModelConfig(
    task="classification",
    layers="1024-512-512",  # Number of nodes in each layer
    activation="LeakyReLU",  # Activation between each layers
    learning_rate=1e-3,
    head="LinearHead",  # Linear Head
    head_config=head_config,  # Linear Head Config
)

Note: For demonstration we are using the test split for tuning, but in real problems, please use a separate validation set for tuning purposes. Otherwise, you will be overfitting to the test set and have falsely high performance estimates.

Define the Hyperparameter Space

The hyperparameter space is defined as a dictionary. The keys are the hyperparameter names and the values are the list of values to be tried. The hyparameter names follow the below convention: - model_config__<hyperparameter_name> for model hyperparameters - model_config.head_config__<hyperparameter_name> for head hyperparameters - optimizer_config__<hyperparameter_name> for optimizer hyperparameters - We can't use data module hyperparameters for tuning as the datamodule is already fitted and we can't change it's hyperparameters.

search_space = {
    "model_config__layers": ["1024-512-512", "1024-512-256", "1024-512-128"],
    "model_config.head_config__dropout": [0.1, 0.2, 0.3],
    "optimizer_config__optimizer": ["RAdam", "AdamW"],
}
# Any other parameter which is not part of the search_space, will be kept constant during the search
from pytorch_tabular.tabular_model_tuner import TabularModelTuner
tuner = TabularModelTuner(
    data_config=data_config,
    model_config=model_config,
    optimizer_config=optimizer_config,
    trainer_config=trainer_config
)
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    result = tuner.tune(
        train=train,
        validation=test,
        search_space=search_space,
        strategy="grid_search",
        # cv=5, # Uncomment this to do a 5 fold cross validation
        metric="accuracy",
        mode="max",
        progress_bar=True,
        verbose=False # Make True if you want to log metrics and params each iteration
    )
Output()


LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]



Result is a namedtuple with trials_df, best_params, best_score and best_model\

  • trials_df: A dataframe with all the hyperparameter combinations and their corresponding scores
  • best_params: The best hyperparameter combination
  • best_score: The best score
  • best_model: If return_best_model is True, return best_model otherwise return None
result.trials_df.head()
trial_id model_config.head_config__dropout model_config__layers optimizer_config__optimizer loss accuracy
0 0 0.1 1024-512-512 RAdam 0.219196 0.9152
1 1 0.1 1024-512-512 AdamW 0.197452 0.9288
2 2 0.1 1024-512-256 RAdam 0.210085 0.9180
3 3 0.1 1024-512-256 AdamW 0.202238 0.9272
4 4 0.1 1024-512-128 RAdam 0.220526 0.9132
print("Best Score: ", result.best_score)
pprint(result.best_params)
Best Score:  0.9308000206947327
{
'model_config.head_config__dropout': 0.3,
'model_config__layers': '1024-512-512',
'optimizer_config__optimizer': 'AdamW',
'loss': 0.19081245362758636
}

Note: For demonstration we are using the test split for tuning, but in real problems, please use a separate validation set for tuning purposes. Otherwise, you will be overfitting to the test set and have falsely high performance estimates.

Define the Hyperparameter Space

The hyperparameter space is defined as a dictionary. The keys are the hyperparameter names and the values are the list of values for categorical and distributions for continuous. The hyparameter names follow the below convention: - model_config__<hyperparameter_name> for model hyperparameters - model_config.head_config__<hyperparameter_name> for head hyperparameters - optimizer_config__<hyperparameter_name> for optimizer hyperparameters - We can't use data module hyperparameters for tuning as the datamodule is already fitted and we can't change it's hyperparameters.

from scipy.stats import uniform, randint, loguniform
search_space = {
    "model_config__layers": ["1024-512-512", "1024-512-256", "1024-512-128"],
    "model_config.head_config__dropout": uniform(0, 0.5),
    "optimizer_config__optimizer": ["RAdam", "AdamW"],
}
# Any other parameter which is not part of the search_space, will be kept constant during the search
from pytorch_tabular.tabular_model_tuner import TabularModelTuner
tuner = TabularModelTuner(
    data_config=data_config,
    model_config=model_config,
    optimizer_config=optimizer_config,
    trainer_config=trainer_config
)
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    result = tuner.tune(
        train=train,
        validation=test, # Need not give validation is we use CV
        search_space=search_space,
        n_trials=10,
        strategy="random_search",
        # cv=5, # Uncomment this to do a 5 fold cross validation
        metric="accuracy",
        mode="max",
        progress_bar=True,
        verbose=False # Make True if you want to log metrics and params each iteration
    )
Output()


LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]



Result is a namedtuple with trials_df, best_params, best_score and best_model\

  • trials_df: A dataframe with all the hyperparameter combinations and their corresponding scores
  • best_params: The best hyperparameter combination
  • best_score: The best score
  • best_model: If return_best_model is True, return best_model otherwise return None
result.trials_df.head()
trial_id model_config.head_config__dropout model_config__layers optimizer_config__optimizer loss accuracy
0 0 0.187270 1024-512-512 RAdam 0.218989 0.9184
1 1 0.365997 1024-512-512 RAdam 0.215875 0.9216
2 2 0.078009 1024-512-128 RAdam 0.226962 0.9104
3 3 0.029042 1024-512-512 AdamW 0.206726 0.9224
4 4 0.071433 1024-512-128 AdamW 0.211062 0.9200
print("Best Score: ", result.best_score)
pprint(result.best_params)
Best Score:  0.9247999787330627
{
'model_config.head_config__dropout': 0.028205789513550128,
'model_config__layers': '1024-512-256',
'optimizer_config__optimizer': 'AdamW',
'loss': 0.18974390625953674
}