import category_encoders as ce
import lightgbm as lgb
import numpy as np
import pandas as pd
from sklearn.datasets import fetch_covtype
from sklearn.metrics import accuracy_score, f1_score
from sklearn.model_selection import train_test_split
from pytorch_tabular.utils import print_metrics, load_covertype_dataset
# %load_ext autoreload
# %autoreload 2
def load_classification_data():
data, cat_col_names, num_col_names, target = load_covertype_dataset()
test_idx = data.sample(int(0.2 * len(data)), random_state=42).index
test = data[data.index.isin(test_idx)]
train = data[~data.index.isin(test_idx)]
return (train, test, cat_col_names, num_col_names, target)
Load Forest Cover Data¶
Baseline¶
Let's use the default LightGBM model as a baseline.
clf = lgb.LGBMClassifier(random_state=42, n_jobs=-1, verbose=-1)
clf.fit(
train_transform.drop(columns=target_col),
train_transform[target_col].values.ravel(),
)
val_pred = clf.predict(val_transform.drop(columns=target_col))
val_metrics = print_metrics(
metrics, val_transform[target_col], val_pred, "Validation", return_dict=True
)
test_pred = clf.predict(test_transform.drop(columns=target_col))
holdout_metrics = print_metrics(
metrics, test_transform[target_col], test_pred, "Holdout", return_dict=True
)
CategoryEmbedding Model¶
from pytorch_tabular import TabularModel
from pytorch_tabular.models import CategoryEmbeddingModelConfig
from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig
from pytorch_tabular.categorical_encoders import CategoricalEmbeddingTransformer
from pytorch_tabular.models.common.heads import LinearHeadConfig
data_config = DataConfig(
target=[target_col], # target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
continuous_cols=num_col_names,
categorical_cols=cat_col_names,
continuous_feature_transform="quantile_normal",
normalize_continuous_features=True,
)
trainer_config = TrainerConfig(
auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate
batch_size=1024,
max_epochs=50,
accelerator="auto", # can be 'cpu','gpu', 'tpu', or 'ipu'
devices=-1, # -1 means use all available
)
optimizer_config = OptimizerConfig()
head_config = LinearHeadConfig(
layers="", # No additional layer in head, just a mapping layer to output_dim
dropout=0.1,
initialization="kaiming",
).__dict__ # Convert to dict to pass to the model config (OmegaConf doesn't accept objects)
model_config = CategoryEmbeddingModelConfig(
task="classification",
layers="512-256-16", # Number of nodes in each layer
activation="LeakyReLU", # Activation between each layers
dropout=0.1,
initialization="kaiming",
head="LinearHead", # Linear Head
head_config=head_config, # Linear Head Config
learning_rate=1e-3,
metrics=["accuracy", "f1_score"],
metrics_params=[{}, {"average": "micro"}],
metrics_prob_input=[False, True],
)
tabular_model = TabularModel(
data_config=data_config,
model_config=model_config,
optimizer_config=optimizer_config,
trainer_config=trainer_config,
verbose=False,
)
Extract the Learned Embedding¶
For the models that support (CategoryEmbeddingModel and CategoryEmbeddingNODE), we can extract the learned embeddings into a sci-kit learn style Transformer. You can use this in your Sci-kit Learn pipelines and workflows as a drop in replacement.
val_transform = transformer.transform(val)
val_pred = clf.predict(val_transform.drop(columns=target_col))
val_metrics = print_metrics(
metrics, val_transform[target_col], val_pred, "Validation", return_dict=True
)
test_transform = transformer.transform(test)
test_pred = clf.predict(test_transform.drop(columns=target_col))
holdout_metrics = print_metrics(
metrics, test_transform[target_col], test_pred, "Holdout", return_dict=True
)