Objective Functions

The objective functions listed here are available for both the XGBoost and LightGBM backends. We follow the naming conventions from both packages. When we provide an objective not available in a package currently (for example, multivariate squared error), we try to extend the naming conventions from either backend to cover this objective.

Univariate Squared Error

XGBoost - 'objective':'reg:squarederror'
LightGBM - 'objective':'regression'

Source code in wideboost/objectives/squareloss.py

def squareloss_gradient_hessian(X,B,Y):
    # Loss = 1/2 (Y - X)^2

    Y = Y.reshape([Y.shape[0],-1])

    assert len(X.shape) == 2
    assert len(B.shape) == 2
    assert len(Y.shape) == 2

    R = Y - X.dot(B)
    dX = -R * B.transpose()
    dX2 = np.ones(Y.shape) * np.square(B).transpose()

    return dX, dX2

Multivariate Squared Error

XGBoost - 'objective':'multi:squarederror'
LightGBM - 'objective':'multiregression'

Source code in wideboost/objectives/squareloss.py

def multi_squareloss_gradient_hessian(X,B,Y):
    # Loss = 1/2 (Y - X)^2
    Y = Y.reshape([Y.shape[0],-1])

    assert len(X.shape) == 2
    assert len(B.shape) == 2
    assert len(Y.shape) == 2

    # Take the gradient and hessian of the usual
    # boosting problem and pass through the general
    # converters in general_gh
    G = X.dot(B) - Y
    H = row_diag(np.ones([Y.shape[0],Y.shape[1]]))

    dX = f_gradient_B(G,B)
    dX2 = f_hessian_B(H,B)

    return dX, dX2

Binary Classification

XGBoost - 'objective':'binary:logistic'
LightGBM - 'objective':'binary'

Source code in wideboost/objectives/binarylogloss.py

def binarylogloss_gradient_hessian(X,B,Y):
    # Loss = log(p) when Y == 1
    # Loss = log(1-p) when Y == 0
    # p = exp(XB)/(1 + exp(XB))
    # 1-p = 1/(1 + exp(XB))

    Y = Y.reshape([Y.shape[0],-1])

    assert len(X.shape) == 2
    assert len(B.shape) == 2
    assert len(Y.shape) == 2

    logits = X.dot(B)
    P = 1 / (1 + np.exp(-logits))
    eps = 1e-16

    dX = (P - Y) * B.transpose()
    dX2 = np.maximum((P * (1-P) * np.square(B).transpose() ), eps)

    return dX, dX2

Multicategory Classification

XGBoost - 'objective':'multi:softmax'
LightGBM - 'objective':'multiclass'

Source code in wideboost/objectives/categoricallogloss.py

def categoricallogloss_gradient_hessian(X,B,Y):
    Y = Y.reshape([Y.shape[0],-1])

    def _onehot(Y):
        b = np.zeros((Y.shape[0], Y.max().astype(int)+1))
        b[np.arange(Y.shape[0]),Y.astype(int).flatten()] = 1
        return b

    assert len(X.shape) == 2
    assert len(B.shape) == 2
    assert len(Y.shape) == 2

    logits = X.dot(B)
    max_logit = np.max(logits,axis=1,keepdims=True)
    logits = logits - max_logit

    sum_exp = np.sum(np.exp(logits),axis=1,keepdims=True)
    P = np.exp(logits) / sum_exp

    eps = 1e-16
    dX = (P - _onehot(Y)).dot(B.transpose())
    dX2 = np.maximum(P.dot(np.square(B).transpose()) - np.square(P.dot(B.transpose())),eps)

    return dX, dX2