Awesome

TabNet for fastai

This is an adaptation of TabNet (Attention-based network for tabular data) for fastai (>=2.0) library. The original paper https://arxiv.org/pdf/1908.07442.pdf.

Install

pip install fast_tabnet

How to use

model = TabNetModel(emb_szs, n_cont, out_sz, embed_p=0., y_range=None, n_d=8, n_a=8, n_steps=3, gamma=1.5, n_independent=2, n_shared=2, epsilon=1e-15, virtual_batch_size=128, momentum=0.02)

Parameters emb_szs, n_cont, out_sz, embed_p, y_range are the same as for fastai TabularModel.

n_d : int Dimension of the prediction layer (usually between 4 and 64)
n_a : int Dimension of the attention layer (usually between 4 and 64)
n_steps: int Number of sucessive steps in the newtork (usually betwenn 3 and 10)
gamma : float Float above 1, scaling factor for attention updates (usually betwenn 1.0 to 2.0)
momentum : float Float value between 0 and 1 which will be used for momentum in all batch norm
n_independent : int Number of independent GLU layer in each GLU block (default 2)
n_shared : int Number of independent GLU layer in each GLU block (default 2)
epsilon: float Avoid log(0), this should be kept very low

Example

Below is an example from fastai library, but the model in use is TabNet

from fastai.basics import *
from fastai.tabular.all import *
from fast_tabnet.core import *

path = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(path/'adult.csv')
df_main,df_test = df.iloc[:-1000].copy(),df.iloc[-1000:].copy()
df_main.head()

<div> <style scoped> .dataframe tbody tr th:only-of-type { vertical-align: middle; }

.dataframe tbody tr th {
    vertical-align: top;
}

.dataframe thead th {
    text-align: right;
}

</style> <table border="1" class="dataframe"> <thead> <tr style="text-align: right;"> <th></th> <th>age</th> <th>workclass</th> <th>fnlwgt</th> <th>education</th> <th>education-num</th> <th>marital-status</th> <th>occupation</th> <th>relationship</th> <th>race</th> <th>sex</th> <th>capital-gain</th> <th>capital-loss</th> <th>hours-per-week</th> <th>native-country</th> <th>salary</th> </tr> </thead> <tbody> <tr> <th>0</th> <td>49</td> <td>Private</td> <td>101320</td> <td>Assoc-acdm</td> <td>12.0</td> <td>Married-civ-spouse</td> <td>NaN</td> <td>Wife</td> <td>White</td> <td>Female</td> <td>0</td> <td>1902</td> <td>40</td> <td>United-States</td> <td>>=50k</td> </tr> <tr> <th>1</th> <td>44</td> <td>Private</td> <td>236746</td> <td>Masters</td> <td>14.0</td> <td>Divorced</td> <td>Exec-managerial</td> <td>Not-in-family</td> <td>White</td> <td>Male</td> <td>10520</td> <td>0</td> <td>45</td> <td>United-States</td> <td>>=50k</td> </tr> <tr> <th>2</th> <td>38</td> <td>Private</td> <td>96185</td> <td>HS-grad</td> <td>NaN</td> <td>Divorced</td> <td>NaN</td> <td>Unmarried</td> <td>Black</td> <td>Female</td> <td>0</td> <td>0</td> <td>32</td> <td>United-States</td> <td><50k</td> </tr> <tr> <th>3</th> <td>38</td> <td>Self-emp-inc</td> <td>112847</td> <td>Prof-school</td> <td>15.0</td> <td>Married-civ-spouse</td> <td>Prof-specialty</td> <td>Husband</td> <td>Asian-Pac-Islander</td> <td>Male</td> <td>0</td> <td>0</td> <td>40</td> <td>United-States</td> <td>>=50k</td> </tr> <tr> <th>4</th> <td>42</td> <td>Self-emp-not-inc</td> <td>82297</td> <td>7th-8th</td> <td>NaN</td> <td>Married-civ-spouse</td> <td>Other-service</td> <td>Wife</td> <td>Black</td> <td>Female</td> <td>0</td> <td>0</td> <td>50</td> <td>United-States</td> <td><50k</td> </tr> </tbody> </table> </div>

cat_names = ['workclass', 'education', 'marital-status', 'occupation', 
             'relationship', 'race', 'native-country', 'sex']
cont_names = ['age', 'fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
splits = RandomSplitter()(range_of(df_main))

to = TabularPandas(df_main, procs, cat_names, cont_names, y_names="salary", 
                   y_block = CategoryBlock(), splits=splits)

dls = to.dataloaders(bs=32)

dls.valid.show_batch()

<table border="1" class="dataframe"> <thead> <tr style="text-align: right;"> <th></th> <th>workclass</th> <th>education</th> <th>marital-status</th> <th>occupation</th> <th>relationship</th> <th>race</th> <th>native-country</th> <th>sex</th> <th>education-num_na</th> <th>age</th> <th>fnlwgt</th> <th>education-num</th> <th>salary</th> </tr> </thead> <tbody> <tr> <th>0</th> <td>Private</td> <td>HS-grad</td> <td>Married-civ-spouse</td> <td>Other-service</td> <td>Wife</td> <td>White</td> <td>United-States</td> <td>Female</td> <td>False</td> <td>39.000000</td> <td>196673.000115</td> <td>9.0</td> <td><50k</td> </tr> <tr> <th>1</th> <td>Private</td> <td>HS-grad</td> <td>Married-civ-spouse</td> <td>Craft-repair</td> <td>Husband</td> <td>White</td> <td>United-States</td> <td>Male</td> <td>False</td> <td>32.000000</td> <td>198067.999771</td> <td>9.0</td> <td><50k</td> </tr> <tr> <th>2</th> <td>State-gov</td> <td>HS-grad</td> <td>Never-married</td> <td>Adm-clerical</td> <td>Own-child</td> <td>White</td> <td>United-States</td> <td>Female</td> <td>False</td> <td>18.999999</td> <td>176633.999931</td> <td>9.0</td> <td><50k</td> </tr> <tr> <th>3</th> <td>Private</td> <td>Some-college</td> <td>Married-civ-spouse</td> <td>Prof-specialty</td> <td>Husband</td> <td>White</td> <td>United-States</td> <td>Male</td> <td>False</td> <td>67.999999</td> <td>107626.998490</td> <td>10.0</td> <td><50k</td> </tr> <tr> <th>4</th> <td>Private</td> <td>Masters</td> <td>Never-married</td> <td>Exec-managerial</td> <td>Not-in-family</td> <td>Black</td> <td>United-States</td> <td>Male</td> <td>False</td> <td>29.000000</td> <td>214925.000260</td> <td>14.0</td> <td><50k</td> </tr> <tr> <th>5</th> <td>Private</td> <td>HS-grad</td> <td>Married-civ-spouse</td> <td>Priv-house-serv</td> <td>Wife</td> <td>White</td> <td>United-States</td> <td>Female</td> <td>False</td> <td>22.000000</td> <td>200109.000126</td> <td>9.0</td> <td><50k</td> </tr> <tr> <th>6</th> <td>Private</td> <td>Some-college</td> <td>Never-married</td> <td>Sales</td> <td>Own-child</td> <td>White</td> <td>United-States</td> <td>Female</td> <td>False</td> <td>18.000000</td> <td>60980.998429</td> <td>10.0</td> <td><50k</td> </tr> <tr> <th>7</th> <td>Private</td> <td>Some-college</td> <td>Separated</td> <td>Adm-clerical</td> <td>Not-in-family</td> <td>White</td> <td>United-States</td> <td>Female</td> <td>False</td> <td>28.000000</td> <td>334367.998199</td> <td>10.0</td> <td><50k</td> </tr> <tr> <th>8</th> <td>Private</td> <td>11th</td> <td>Married-civ-spouse</td> <td>Transport-moving</td> <td>Husband</td> <td>White</td> <td>United-States</td> <td>Male</td> <td>False</td> <td>49.000000</td> <td>123584.001097</td> <td>7.0</td> <td><50k</td> </tr> <tr> <th>9</th> <td>Private</td> <td>Masters</td> <td>Never-married</td> <td>Prof-specialty</td> <td>Not-in-family</td> <td>White</td> <td>United-States</td> <td>Female</td> <td>False</td> <td>26.000000</td> <td>397316.999922</td> <td>14.0</td> <td><50k</td> </tr> </tbody> </table>

to_tst = to.new(df_test)
to_tst.process()
to_tst.all_cols.head()

<div> <style scoped> .dataframe tbody tr th:only-of-type { vertical-align: middle; }

.dataframe tbody tr th {
    vertical-align: top;
}

.dataframe thead th {
    text-align: right;
}

</style> <table border="1" class="dataframe"> <thead> <tr style="text-align: right;"> <th></th> <th>workclass</th> <th>education</th> <th>marital-status</th> <th>occupation</th> <th>relationship</th> <th>race</th> <th>native-country</th> <th>sex</th> <th>education-num_na</th> <th>age</th> <th>fnlwgt</th> <th>education-num</th> <th>salary</th> </tr> </thead> <tbody> <tr> <th>31561</th> <td>5</td> <td>2</td> <td>5</td> <td>9</td> <td>3</td> <td>3</td> <td>40</td> <td>2</td> <td>1</td> <td>-1.505833</td> <td>-0.559418</td> <td>-1.202170</td> <td>0</td> </tr> <tr> <th>31562</th> <td>2</td> <td>12</td> <td>5</td> <td>2</td> <td>5</td> <td>3</td> <td>40</td> <td>1</td> <td>1</td> <td>-1.432653</td> <td>0.421241</td> <td>-0.418032</td> <td>0</td> </tr> <tr> <th>31563</th> <td>5</td> <td>7</td> <td>3</td> <td>4</td> <td>1</td> <td>5</td> <td>40</td> <td>2</td> <td>1</td> <td>-0.115406</td> <td>0.132868</td> <td>-1.986307</td> <td>0</td> </tr> <tr> <th>31564</th> <td>8</td> <td>12</td> <td>3</td> <td>9</td> <td>1</td> <td>5</td> <td>40</td> <td>2</td> <td>1</td> <td>1.494561</td> <td>0.749805</td> <td>-0.418032</td> <td>0</td> </tr> <tr> <th>31565</th> <td>1</td> <td>12</td> <td>1</td> <td>1</td> <td>5</td> <td>3</td> <td>40</td> <td>2</td> <td>1</td> <td>-0.481308</td> <td>7.529798</td> <td>-0.418032</td> <td>0</td> </tr> </tbody> </table> </div>

emb_szs = get_emb_sz(to)

That's the use of the model

model = TabNetModel(emb_szs, len(to.cont_names), dls.c, n_d=8, n_a=8, n_steps=5, mask_type='entmax');

learn = Learner(dls, model, CrossEntropyLossFlat(), opt_func=Adam, lr=3e-2, metrics=[accuracy])

learn.lr_find()

SuggestedLRs(lr_min=0.2754228591918945, lr_steep=1.9054607491852948e-06)

png

learn.fit_one_cycle(5)

Tabnet interpretability

# feature importance for 2k rows
dl = learn.dls.test_dl(df.iloc[:2000], bs=256)
feature_importances = tabnet_feature_importances(learn.model, dl)

# per sample interpretation
dl = learn.dls.test_dl(df.iloc[:20], bs=20)
res_explain, res_masks = tabnet_explain(learn.model, dl)

plt.xticks(rotation='vertical')
plt.bar(dl.x_names, feature_importances, color='g')
plt.show()

png

def plot_explain(masks, lbls, figsize=(12,12)):
    "Plots masks with `lbls` (`dls.x_names`)"
    fig = plt.figure(figsize=figsize)
    ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
    plt.yticks(np.arange(0, len(masks), 1.0))
    plt.xticks(np.arange(0, len(masks[0]), 1.0))
    ax.set_xticklabels(lbls, rotation=90)
    plt.ylabel('Sample Number')
    plt.xlabel('Variable')
    plt.imshow(masks)

plot_explain(res_explain, dl.x_names)

png

Hyperparameter search with Bayesian Optimization

If your dataset isn't huge you can tune hyperparameters for tabular models with Bayesian Optimization. You can optimize directly your metric using this approach if the metric is sensitive enough (in our example it is not and we use validation loss instead). Also, you should create the second validation set, because you will use the first as a training set for Bayesian Optimization.

You may need to install the optimizer pip install bayesian-optimization

from functools import lru_cache

# The function we'll optimize
@lru_cache(1000)
def get_accuracy(n_d:Int, n_a:Int, n_steps:Int):
    model = TabNetModel(emb_szs, len(to.cont_names), dls.c, n_d=n_d, n_a=n_a, n_steps=n_steps, gamma=1.5)
    learn = Learner(dls, model, CrossEntropyLossFlat(), opt_func=opt_func, lr=3e-2, metrics=[accuracy])
    learn.fit_one_cycle(5)
    return float(learn.validate(dl=learn.dls.valid)[1])

This implementation of Bayesian Optimization doesn't work naturally with descreet values. That's why we use wrapper with lru_cache.

def fit_accuracy(pow_n_d, pow_n_a, pow_n_steps):
    n_d, n_a, n_steps = map(lambda x: 2**int(x), (pow_n_d, pow_n_a, pow_n_steps))
    return get_accuracy(n_d, n_a, n_steps)

from bayes_opt import BayesianOptimization

# Bounded region of parameter space
pbounds = {'pow_n_d': (0, 8), 'pow_n_a': (0, 8), 'pow_n_steps': (0, 4)}

optimizer = BayesianOptimization(
    f=fit_accuracy,
    pbounds=pbounds,
)

optimizer.maximize(
    init_points=15,
    n_iter=100,
)

|   iter    |  target   |  pow_n_a  |  pow_n_d  | pow_n_... |
-------------------------------------------------------------

| [0m 1       [0m | [0m 0.8235  [0m | [0m 0.9408  [0m | [0m 1.898   [0m | [0m 1.652   [0m |

| [95m 2       [0m | [95m 0.8273  [0m | [95m 4.262   [0m | [95m 5.604   [0m | [95m 0.2437  [0m |

| [0m 3       [0m | [0m 0.8172  [0m | [0m 7.233   [0m | [0m 6.471   [0m | [0m 2.508   [0m |

| [95m 4       [0m | [95m 0.8311  [0m | [95m 5.935   [0m | [95m 1.241   [0m | [95m 0.3809  [0m |

| [0m 5       [0m | [0m 0.7586  [0m | [0m 2.554   [0m | [0m 0.4992  [0m | [0m 3.111   [0m |

| [0m 6       [0m | [0m 0.8189  [0m | [0m 4.592   [0m | [0m 2.138   [0m | [0m 2.824   [0m |

| [0m 7       [0m | [0m 0.8058  [0m | [0m 6.186   [0m | [0m 7.016   [0m | [0m 3.316   [0m |

| [0m 8       [0m | [0m 0.7616  [0m | [0m 2.018   [0m | [0m 1.316   [0m | [0m 3.675   [0m |

| [0m 9       [0m | [0m 0.801   [0m | [0m 2.051   [0m | [0m 1.958   [0m | [0m 2.332   [0m |

| [0m 10      [0m | [0m 0.7983  [0m | [0m 5.203   [0m | [0m 7.719   [0m | [0m 3.407   [0m |

| [0m 11      [0m | [0m 0.8308  [0m | [0m 6.048   [0m | [0m 4.376   [0m | [0m 0.08141 [0m |

| [0m 12      [0m | [0m 0.8161  [0m | [0m 7.083   [0m | [0m 1.385   [0m | [0m 2.806   [0m |

| [0m 13      [0m | [0m 0.8238  [0m | [0m 4.812   [0m | [0m 3.785   [0m | [0m 1.396   [0m |
| [0m 14      [0m | [0m 0.8172  [0m | [0m 7.672   [0m | [0m 6.719   [0m | [0m 2.72    [0m |

| [0m 15      [0m | [0m 0.8215  [0m | [0m 6.464   [0m | [0m 7.954   [0m | [0m 2.647   [0m |

| [0m 16      [0m | [0m 0.8259  [0m | [0m 0.1229  [0m | [0m 7.83    [0m | [0m 0.3708  [0m |

| [0m 17      [0m | [0m 0.8259  [0m | [0m 0.03098 [0m | [0m 3.326   [0m | [0m 0.007025[0m |

| [0m 18      [0m | [0m 0.8294  [0m | [0m 7.81    [0m | [0m 7.976   [0m | [0m 0.0194  [0m |

| [0m 19      [0m | [0m 0.2489  [0m | [0m 0.4499  [0m | [0m 0.138   [0m | [0m 0.001101[0m |

| [0m 20      [0m | [0m 0.8221  [0m | [0m 0.0     [0m | [0m 6.575   [0m | [0m 4.0     [0m |

| [0m 21      [0m | [0m 0.2489  [0m | [0m 8.0     [0m | [0m 0.0     [0m | [0m 0.0     [0m |

| [0m 22      [0m | [0m 0.8251  [0m | [0m 0.0     [0m | [0m 4.502   [0m | [0m 2.193   [0m |

| [0m 23      [0m | [0m 0.789   [0m | [0m 8.0     [0m | [0m 3.702   [0m | [0m 4.0     [0m |

| [0m 24      [0m | [0m 0.7549  [0m | [0m 6.009   [0m | [0m 0.0     [0m | [0m 4.0     [0m |

| [0m 25      [0m | [0m 0.8292  [0m | [0m 3.522   [0m | [0m 8.0     [0m | [0m 0.0     [0m |

| [0m 26      [0m | [0m 0.817   [0m | [0m 0.0     [0m | [0m 8.0     [0m | [0m 4.0     [0m |

| [0m 27      [0m | [0m 0.7983  [0m | [0m 0.0     [0m | [0m 0.0     [0m | [0m 4.0     [0m |

| [0m 28      [0m | [0m 0.8289  [0m | [0m 3.379   [0m | [0m 2.848   [0m | [0m 0.0     [0m |

| [0m 29      [0m | [0m 0.8072  [0m | [0m 0.0     [0m | [0m 2.588   [0m | [0m 4.0     [0m |

| [0m 30      [0m | [0m 0.8262  [0m | [0m 1.217   [0m | [0m 5.622   [0m | [0m 0.0     [0m |

| [0m 31      [0m | [0m 0.8016  [0m | [0m 8.0     [0m | [0m 8.0     [0m | [0m 4.0     [0m |

| [0m 32      [0m | [0m 0.8294  [0m | [0m 5.864   [0m | [0m 8.0     [0m | [0m 0.0     [0m |

| [0m 33      [0m | [0m 0.8208  [0m | [0m 1.776   [0m | [0m 8.0     [0m | [0m 2.212   [0m |

| [0m 34      [0m | [0m 0.8289  [0m | [0m 5.777   [0m | [0m 2.2     [0m | [0m 1.31    [0m |

| [0m 35      [0m | [0m 0.8016  [0m | [0m 2.748   [0m | [0m 5.915   [0m | [0m 4.0     [0m |

| [0m 36      [0m | [0m 0.7719  [0m | [0m 6.251   [0m | [0m 2.532   [0m | [0m 4.0     [0m |

| [0m 37      [0m | [0m 0.821   [0m | [0m 5.093   [0m | [0m 0.172   [0m | [0m 1.64    [0m |

| [0m 38      [0m | [0m 0.8243  [0m | [0m 8.0     [0m | [0m 5.799   [0m | [0m 0.0     [0m |

| [0m 39      [0m | [0m 0.8278  [0m | [0m 1.62    [0m | [0m 3.832   [0m | [0m 1.151   [0m |

| [0m 40      [0m | [0m 0.7966  [0m | [0m 2.198   [0m | [0m 8.0     [0m | [0m 4.0     [0m |

| [0m 41      [0m | [0m 0.7641  [0m | [0m 8.0     [0m | [0m 1.03    [0m | [0m 4.0     [0m |

| [0m 42      [0m | [0m 0.8232  [0m | [0m 0.0     [0m | [0m 2.504   [0m | [0m 2.135   [0m |

| [0m 43      [0m | [0m 0.8294  [0m | [0m 0.0     [0m | [0m 5.441   [0m | [0m 0.0     [0m |

| [0m 44      [0m | [0m 0.8276  [0m | [0m 4.636   [0m | [0m 1.476   [0m | [0m 0.0     [0m |

| [0m 45      [0m | [0m 0.8262  [0m | [0m 0.0     [0m | [0m 7.071   [0m | [0m 2.071   [0m |

| [0m 46      [0m | [0m 0.8211  [0m | [0m 4.26    [0m | [0m 6.934   [0m | [0m 1.79    [0m |

| [0m 47      [0m | [0m 0.8279  [0m | [0m 6.579   [0m | [0m 6.485   [0m | [0m 0.0     [0m |

| [0m 48      [0m | [0m 0.8265  [0m | [0m 8.0     [0m | [0m 4.293   [0m | [0m 1.74    [0m |

| [0m 49      [0m | [0m 0.8308  [0m | [0m 7.909   [0m | [0m 7.827   [0m | [0m 1.323   [0m |

| [0m 50      [0m | [0m 0.8303  [0m | [0m 4.946   [0m | [0m 1.246   [0m | [0m 1.589   [0m |

| [95m 51      [0m | [95m 0.8314  [0m | [95m 5.664   [0m | [95m 2.626   [0m | [95m 0.003048[0m |

| [0m 52      [0m | [0m 0.8289  [0m | [0m 1.287   [0m | [0m 3.505   [0m | [0m 0.06804 [0m |

| [0m 53      [0m | [0m 0.8272  [0m | [0m 1.606   [0m | [0m 7.998   [0m | [0m 0.2009  [0m |

| [0m 54      [0m | [0m 0.82    [0m | [0m 4.579   [0m | [0m 5.017   [0m | [0m 2.928   [0m |
| [0m 55      [0m | [0m 0.8259  [0m | [0m 0.02565 [0m | [0m 3.699   [0m | [0m 0.9808  [0m |

| [0m 56      [0m | [0m 0.8015  [0m | [0m 1.927   [0m | [0m 5.92    [0m | [0m 2.53    [0m |

| [0m 57      [0m | [0m 0.8197  [0m | [0m 0.7796  [0m | [0m 4.576   [0m | [0m 3.952   [0m |

| [0m 58      [0m | [0m 0.8297  [0m | [0m 3.525   [0m | [0m 4.198   [0m | [0m 0.02314 [0m |

| [0m 59      [0m | [0m 0.8194  [0m | [0m 6.711   [0m | [0m 3.848   [0m | [0m 2.395   [0m |

| [0m 60      [0m | [0m 0.8308  [0m | [0m 4.914   [0m | [0m 7.944   [0m | [0m 0.9998  [0m |

| [0m 61      [0m | [0m 0.8279  [0m | [0m 7.962   [0m | [0m 6.151   [0m | [0m 1.119   [0m |

| [0m 62      [0m | [0m 0.8292  [0m | [0m 2.583   [0m | [0m 6.996   [0m | [0m 0.008348[0m |

| [0m 63      [0m | [0m 0.8311  [0m | [0m 0.2249  [0m | [0m 8.0     [0m | [0m 2.823   [0m |

| [0m 64      [0m | [0m 0.8238  [0m | [0m 5.764   [0m | [0m 5.509   [0m | [0m 1.482   [0m |

| [0m 65      [0m | [0m 0.8292  [0m | [0m 1.048   [0m | [0m 2.939   [0m | [0m 1.922   [0m |

| [0m 66      [0m | [0m 0.7836  [0m | [0m 3.68    [0m | [0m 3.977   [0m | [0m 3.919   [0m |

| [0m 67      [0m | [0m 0.8131  [0m | [0m 5.907   [0m | [0m 0.9452  [0m | [0m 2.168   [0m |

| [0m 68      [0m | [0m 0.814   [0m | [0m 7.981   [0m | [0m 2.796   [0m | [0m 2.78    [0m |

| [0m 69      [0m | [0m 0.8268  [0m | [0m 7.223   [0m | [0m 3.762   [0m | [0m 0.6961  [0m |

| [0m 70      [0m | [0m 0.8254  [0m | [0m 0.009375[0m | [0m 5.081   [0m | [0m 3.79    [0m |

| [0m 71      [0m | [0m 0.8251  [0m | [0m 3.293   [0m | [0m 2.76    [0m | [0m 1.061   [0m |
| [0m 72      [0m | [0m 0.8308  [0m | [0m 4.589   [0m | [0m 7.13    [0m | [0m 0.003179[0m |

| [0m 73      [0m | [0m 0.8286  [0m | [0m 7.118   [0m | [0m 5.204   [0m | [0m 0.5939  [0m |

| [0m 74      [0m | [0m 0.8254  [0m | [0m 6.837   [0m | [0m 7.988   [0m | [0m 1.055   [0m |
| [0m 75      [0m | [0m 0.8259  [0m | [0m 0.6629  [0m | [0m 7.012   [0m | [0m 0.03222 [0m |
| [0m 76      [0m | [0m 0.8311  [0m | [0m 5.177   [0m | [0m 1.457   [0m | [0m 0.5857  [0m |

| [0m 77      [0m | [0m 0.8188  [0m | [0m 3.308   [0m | [0m 4.533   [0m | [0m 2.048   [0m |

| [0m 78      [0m | [0m 0.8227  [0m | [0m 0.4036  [0m | [0m 7.997   [0m | [0m 1.439   [0m |

| [0m 79      [0m | [0m 0.8264  [0m | [0m 3.438   [0m | [0m 7.982   [0m | [0m 1.829   [0m |

| [0m 80      [0m | [0m 0.8297  [0m | [0m 6.88    [0m | [0m 2.404   [0m | [0m 1.666   [0m |

| [0m 81      [0m | [0m 0.8238  [0m | [0m 0.03221 [0m | [0m 1.306   [0m | [0m 3.909   [0m |

| [0m 82      [0m | [0m 0.8306  [0m | [0m 1.575   [0m | [0m 2.689   [0m | [0m 0.8684  [0m |

| [0m 83      [0m | [0m 0.831   [0m | [0m 2.765   [0m | [0m 5.439   [0m | [0m 0.04047 [0m |

| [0m 84      [0m | [0m 0.8253  [0m | [0m 0.1961  [0m | [0m 4.123   [0m | [0m 0.02039 [0m |

| [0m 85      [0m | [0m 0.8213  [0m | [0m 7.937   [0m | [0m 7.939   [0m | [0m 2.895   [0m |

| [0m 86      [0m | [0m 0.8235  [0m | [0m 0.06921 [0m | [0m 5.7     [0m | [0m 2.778   [0m |

| [0m 87      [0m | [0m 0.8184  [0m | [0m 7.965   [0m | [0m 5.261   [0m | [0m 2.661   [0m |

| [0m 88      [0m | [0m 0.8305  [0m | [0m 2.792   [0m | [0m 7.917   [0m | [0m 0.761   [0m |
| [0m 89      [0m | [0m 0.8294  [0m | [0m 7.995   [0m | [0m 7.186   [0m | [0m 0.1199  [0m |

| [0m 90      [0m | [0m 0.8305  [0m | [0m 2.526   [0m | [0m 3.741   [0m | [0m 0.1186  [0m |
| [0m 91      [0m | [0m 0.8294  [0m | [0m 0.03285 [0m | [0m 5.742   [0m | [0m 0.9747  [0m |

| [0m 92      [0m | [0m 0.8298  [0m | [0m 4.112e-0[0m | [0m 8.0     [0m | [0m 0.0     [0m |

| [0m 93      [0m | [0m 0.8047  [0m | [0m 4.563   [0m | [0m 0.6868  [0m | [0m 2.461   [0m |

| [0m 94      [0m | [0m 0.8146  [0m | [0m 4.697   [0m | [0m 3.412   [0m | [0m 2.514   [0m |

| [0m 95      [0m | [0m 0.8275  [0m | [0m 5.04    [0m | [0m 0.4492  [0m | [0m 0.5899  [0m |

| [0m 96      [0m | [0m 0.8237  [0m | [0m 0.6025  [0m | [0m 2.712   [0m | [0m 1.166   [0m |

| [95m 97      [0m | [95m 0.8321  [0m | [95m 6.389   [0m | [95m 3.648   [0m | [95m 1.016   [0m |

| [0m 98      [0m | [0m 0.8273  [0m | [0m 0.5927  [0m | [0m 1.715   [0m | [0m 2.847   [0m |

| [0m 99      [0m | [0m 0.8184  [0m | [0m 6.602   [0m | [0m 2.266   [0m | [0m 2.535   [0m |

| [0m 100     [0m | [0m 0.8302  [0m | [0m 3.099   [0m | [0m 6.058   [0m | [0m 1.058   [0m |

| [0m 101     [0m | [0m 0.8302  [0m | [0m 4.268   [0m | [0m 5.28    [0m | [0m 1.397   [0m |

| [0m 102     [0m | [0m 0.8275  [0m | [0m 4.735   [0m | [0m 3.398   [0m | [0m 0.01904 [0m |

| [0m 103     [0m | [0m 0.8245  [0m | [0m 0.0     [0m | [0m 6.371   [0m | [0m 9.721e-0[0m |

| [0m 104     [0m | [0m 0.8305  [0m | [0m 1.683   [0m | [0m 6.381   [0m | [0m 0.8564  [0m |

| [0m 105     [0m | [0m 0.8305  [0m | [0m 6.387   [0m | [0m 2.333   [0m | [0m 0.4877  [0m |

| [0m 106     [0m | [0m 0.8245  [0m | [0m 0.9452  [0m | [0m 6.902   [0m | [0m 3.991   [0m |

| [0m 107     [0m | [0m 0.831   [0m | [0m 5.475   [0m | [0m 5.721   [0m | [0m 0.02659 [0m |
| [0m 108     [0m | [0m 0.8308  [0m | [0m 4.597   [0m | [0m 7.994   [0m | [0m 0.1318  [0m |

| [0m 109     [0m | [0m 0.8291  [0m | [0m 2.391   [0m | [0m 4.915   [0m | [0m 0.8695  [0m |

| [0m 110     [0m | [0m 0.8286  [0m | [0m 5.711   [0m | [0m 7.031   [0m | [0m 1.157   [0m |

| [0m 111     [0m | [0m 0.83    [0m | [0m 3.867   [0m | [0m 7.351   [0m | [0m 0.7373  [0m |
| [0m 112     [0m | [0m 0.8275  [0m | [0m 5.568   [0m | [0m 0.8565  [0m | [0m 0.9522  [0m |
| [0m 113     [0m | [0m 0.8311  [0m | [0m 5.553   [0m | [0m 1.45    [0m | [0m 0.0     [0m |

| [0m 114     [0m | [0m 0.8256  [0m | [0m 3.949   [0m | [0m 1.125   [0m | [0m 1.109   [0m |

| [0m 115     [0m | [0m 0.8313  [0m | [0m 7.258   [0m | [0m 4.724   [0m | [0m 1.639   [0m |
=============================================================

optimizer.max['target']

0.8428390622138977

{key: 2**int(value)
  for key, value in optimizer.max['params'].items()}

{'pow_n_a': 2, 'pow_n_d': 64, 'pow_n_steps': 1}

Out of memory dataset

If your dataset is so big it doesn't fit in memory, you can load a chunk of it each epoch.

df = pd.read_csv(path/'adult.csv')
df_main,df_valid = df.iloc[:-1000].copy(),df.iloc[-1000:].copy()

# choose size that fit in memory
dataset_size = 1000

# load chunk with your own code
def load_chunk():
    return df_main.sample(dataset_size).copy()

df_small = load_chunk()
cat_names = ['workclass', 'education', 'marital-status', 'occupation', 
             'relationship', 'race', 'native-country', 'sex']
cont_names = ['age', 'fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
splits = RandomSplitter()(range_of(df_small))

to = TabularPandas(df_small, procs, cat_names, cont_names, y_names="salary", y_block = CategoryBlock(), 
                   splits=None, do_setup=True)

# save the validation set
to_valid = to.new(df_valid)
to_valid.process()
val_dl = TabDataLoader(to_valid.train)

len(to.train)

class ReloadCallback(Callback):
    def begin_epoch(self): 
        df_small = load_chunk()
        to_new = to.new(df_small)
        to_new.process()
        trn_dl = TabDataLoader(to_new.train)
        self.learn.dls = DataLoaders(trn_dl, val_dl).cuda()

dls = to.dataloaders()
emb_szs = get_emb_sz(to)
model = TabNetModel(emb_szs, len(to.cont_names), dls.c, n_d=8, n_a=32, n_steps=1);

opt_func = partial(Adam, wd=0.01, eps=1e-5)
learn = Learner(dls, model, CrossEntropyLossFlat(), opt_func=opt_func, lr=3e-2, metrics=[accuracy])
learn.add_cb(ReloadCallback());

learn.fit_one_cycle(10)

<table border="1" class="dataframe"> <thead> <tr style="text-align: left;"> <th>epoch</th> <th>train_loss</th> <th>valid_loss</th> <th>accuracy</th> <th>time</th> </tr> </thead> <tbody> <tr> <td>0</td> <td>0.587740</td> <td>0.550544</td> <td>0.756000</td> <td>00:01</td> </tr> <tr> <td>1</td> <td>0.545411</td> <td>0.515772</td> <td>0.782000</td> <td>00:01</td> </tr> <tr> <td>2</td> <td>0.484289</td> <td>0.468586</td> <td>0.813000</td> <td>00:01</td> </tr> <tr> <td>3</td> <td>0.447111</td> <td>0.435774</td> <td>0.817000</td> <td>00:01</td> </tr> <tr> <td>4</td> <td>0.449050</td> <td>0.394715</td> <td>0.819000</td> <td>00:01</td> </tr> <tr> <td>5</td> <td>0.428863</td> <td>0.382005</td> <td>0.835000</td> <td>00:01</td> </tr> <tr> <td>6</td> <td>0.382100</td> <td>0.404258</td> <td>0.826000</td> <td>00:01</td> </tr> <tr> <td>7</td> <td>0.383915</td> <td>0.376179</td> <td>0.833000</td> <td>00:01</td> </tr> <tr> <td>8</td> <td>0.389460</td> <td>0.367857</td> <td>0.834000</td> <td>00:01</td> </tr> <tr> <td>9</td> <td>0.376486</td> <td>0.367577</td> <td>0.834000</td> <td>00:01</td> </tr> </tbody> </table>