diff --git a/lit_nlp/components/ablation_flip_int_test.py b/lit_nlp/components/ablation_flip_int_test.py index 10cf9f9e..c392e0dd 100644 --- a/lit_nlp/components/ablation_flip_int_test.py +++ b/lit_nlp/components/ablation_flip_int_test.py @@ -58,27 +58,33 @@ def predict(self, class ModelBasedAblationFlipTest(absltest.TestCase): - def setUp(self): - super(ModelBasedAblationFlipTest, self).setUp() - self.ablation_flip = ablation_flip.AblationFlip() - + @classmethod + def setUpClass(cls): + super().setUpClass() # Classification model that clasifies a given input sentence. - self.classification_model = glue_models.SST2Model(BERT_TINY_PATH) - self.classification_config = {ablation_flip.PREDICTION_KEY: 'probas'} + cls.classification_model = glue_models.SST2Model(BERT_TINY_PATH) # Classification model with the 'sentence' field marked as # non-required. - self.classification_model_non_required_field = SST2ModelNonRequiredField( - BERT_TINY_PATH) + cls.classification_model_non_required_field = SST2ModelNonRequiredField( + BERT_TINY_PATH + ) # Classification model with a counter to count number of predict calls. # TODO(ataly): Consider setting up a Mock object to count number of # predict calls. - self.classification_model_with_predict_counter = ( - SST2ModelWithPredictCounter(BERT_TINY_PATH)) + cls.classification_model_with_predict_counter = SST2ModelWithPredictCounter( + BERT_TINY_PATH + ) # Regression model determining similarity between two input sentences. - self.regression_model = glue_models.STSBModel(STSB_PATH) + cls.regression_model = glue_models.STSBModel(STSB_PATH) + + def setUp(self): + super().setUp() + self.ablation_flip = ablation_flip.AblationFlip() + self.classification_config = {ablation_flip.PREDICTION_KEY: 'probas'} + self.classification_model_with_predict_counter.predict_counter = 0 self.regression_config = {ablation_flip.PREDICTION_KEY: 'score'} def test_ablation_flip_num_ex(self): diff --git a/lit_nlp/components/hotflip_int_test.py b/lit_nlp/components/hotflip_int_test.py index c5d7758b..7a491a21 100644 --- a/lit_nlp/components/hotflip_int_test.py +++ b/lit_nlp/components/hotflip_int_test.py @@ -50,13 +50,14 @@ class HotflipIntegrationTest(parameterized.TestCase): - def __init__(self, *args, **kwargs): - super(HotflipIntegrationTest, self).__init__(*args, **kwargs) - self.classification_model = glue_models.SST2Model(BERT_TINY_PATH) - self.regression_model = glue_models.STSBModel(STSB_PATH) + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.classification_model = glue_models.SST2Model(BERT_TINY_PATH) + cls.regression_model = glue_models.STSBModel(STSB_PATH) def setUp(self): - super(HotflipIntegrationTest, self).setUp() + super().setUp() self.hotflip = hotflip.HotFlip() @parameterized.named_parameters( diff --git a/lit_nlp/components/tcav_int_test.py b/lit_nlp/components/tcav_int_test.py index 041a6a84..05ede0af 100644 --- a/lit_nlp/components/tcav_int_test.py +++ b/lit_nlp/components/tcav_int_test.py @@ -112,53 +112,164 @@ def test_tcav(self): self.assertLen(result, 1) expected = { - 'p_val': 0.13311, - 'random_mean': 0.56667, + 'p_val': 0.10634, + 'random_mean': 0.54444, 'result': { - 'score': - 0.33333, + 'score': 0.33333, 'cos_sim': [ - 0.088691, -0.12179, 0.16013, 0.24840, -0.09793, 0.05166, - -0.21578, -0.06560, -0.14759 + 0.088691, + -0.12179, + 0.16013, + 0.24840, + -0.09793, + 0.05166, + -0.21578, + -0.06560, + -0.14759, ], 'dot_prods': [ - 189.085096, -266.36317, 344.350498, 547.144949, -211.663965, - 112.502439, -472.72066, -144.529598, -323.31888 + 76.24736, + -107.40964, + 138.85724, + 220.63297, + -85.35256, + 45.365814, + -190.62225, + -58.28084, + -130.37686, ], - 'accuracy': - 0.66667, + 'accuracy': 0.66667, 'cav': [[ - 0.91326976, -13.67988429, -1.33965892, 12.23702997, 4.83590182, - -20.6276055, 19.07518597, 4.9641991, -5.52570226, 10.37834134, - 0.53685622, -5.21721739, 12.00234068, -26.36114057, 6.20088358, - 4.76729567, 26.4359945, 16.30961659, 14.70620605, -13.47771528, - 5.29218365, 0.87290488, 10.19441762, 2.96687215, -0.70745918, - -12.201803, -10.65010904, -5.90814342, -25.17510006, - -5.90629019, 26.53638293, 27.44054429, -11.75430314, - -3.72779609, 10.28197421, -11.58444132, -18.09351946, - -22.09520524, -30.04023056, -16.7551763, -12.34913637, - 30.27897114, -13.98790656, 0.65758253, 3.75770261, 4.81132118, - -11.18005293, 5.85445303, 18.88336475, -7.34885733, - -35.76094848, -3.39495953, 5.52774132, -8.38126488, 10.07413613, - -1.96825956, 17.97041089, -2.47085774, 19.5700424, -49.46295186, - 0.12541183, -6.95592842, -0.33562953, 12.9269603, -13.6288284, - -9.51211543, 21.22778867, -2.81344371, -9.66434107, - -15.41551695, -30.15406401, 14.65903841, 2.51729041, - 17.70711379, 13.21615045, 12.55899318, 11.45749114, - -25.67659992, 13.00876054, -1.52499005, 27.45026658, - -4.36110401, 10.01664277, -11.24470769, -6.79411522, - -1.67106503, -2.59389537, 11.72310319, -0.84126818, 2.03886137, - -11.25047383, -8.60679631, -23.1676229, 22.83532544, - -25.2657054, -15.49795527, -1.62890474, -15.49504251, - 27.26973702, -8.00652979, 6.87541861, -3.61878753, 0.82889822, - -2.88891667, 6.13730358, 12.55884424, -2.24121286, 45.90285087, - 34.43108722, -13.32567113, 19.0988537, 2.16242269, 17.45654791, - 18.17472208, 18.28023357, -20.34869744, -11.21275755, 6.0583063, - -10.38857432, -7.30056744, -8.85395997, -17.19779724, - -8.12087822, 2.46058364, 29.35061395, -6.53820405, 10.3522653, - 8.54478485 - ]] - } + 0.36827186, + -5.516342, + -0.5401955, + 4.934511, + 1.9500555, + -8.317984, + 7.6919622, + 2.0017889, + -2.2282174, + 4.1850123, + 0.21649589, + -2.1038177, + 4.8398824, + -10.6299925, + 2.5004756, + 1.9223839, + 10.660182, + 6.576766, + 5.9301996, + -5.4348207, + 2.1340482, + 0.3519863, + 4.110835, + 1.1963654, + -0.28528506, + -4.9203076, + -4.2945995, + -2.3824337, + -10.151711, + -2.3816726, + 10.700653, + 11.065259, + -4.73986, + -1.5032197, + 4.1461425, + -4.671359, + -7.2961125, + -8.909774, + -12.113558, + -6.7564387, + -4.9797287, + 12.209834, + -5.640538, + 0.26517436, + 1.5152711, + 1.9401419, + -4.5083013, + 2.3607779, + 7.614603, + -2.9633944, + -14.420401, + -1.3690051, + 2.2290342, + -3.3796995, + 4.062332, + -0.7936807, + 7.2464848, + -0.9963622, + 7.8915153, + -19.94567, + 0.05056518, + -2.8049498, + -0.13533497, + 5.2127295, + -5.4957557, + -3.8357103, + 8.559994, + -1.1345136, + -3.8970888, + -6.2162194, + -12.15947, + 5.911185, + 1.015078, + 7.1402836, + 5.329341, + 5.0643563, + 4.6201625, + -10.353953, + 5.2457047, + -0.6149202, + 11.069172, + -1.7585949, + 4.0391507, + -4.534356, + -2.7396822, + -0.6738471, + -1.0459745, + 4.727278, + -0.33924684, + 0.822157, + -4.5366936, + -3.47064, + -9.342223, + 9.208217, + -10.188266, + -6.249456, + -0.65684766, + -6.2482944, + 10.996384, + -3.2285907, + 2.7724774, + -1.459264, + 0.33424366, + -1.1649461, + 2.4748344, + 5.06429, + -0.9037646, + 18.510073, + 13.884155, + -5.3735056, + 7.7014923, + 0.871988, + 7.0392666, + 7.3288665, + 7.37141, + -8.205492, + -4.5214796, + 2.4429858, + -4.1891437, + -2.9439187, + -3.5703194, + -6.9349284, + -3.2747078, + 0.9922186, + 11.835495, + -2.6364844, + 4.1745014, + 3.445649, + ]], + }, } testing_utils.assert_deep_almost_equal(self, expected, result[0]) @@ -193,8 +304,8 @@ def test_tcav_ignores_labels(self): self.assertLen(result, 1) expected = { - 'p_val': 0.13311, - 'random_mean': 0.56667, + 'p_val': 0.10634, + 'random_mean': 0.54444, 'result': { 'score': 0.33333, 'cos_sim': [ @@ -209,146 +320,146 @@ def test_tcav_ignores_labels(self): -0.14759, ], 'dot_prods': [ - 189.085096, - -266.36317, - 344.350498, - 547.144949, - -211.663965, - 112.502439, - -472.72066, - -144.529598, - -323.31888, + 76.24736, + -107.40957, + 138.85725, + 220.63297, + -85.35252, + 45.365868, + -190.62221, + -58.28079, + -130.3768, ], 'accuracy': 0.66667, 'cav': [[ - 0.91326976, - -13.67988429, - -1.33965892, - 12.23702997, - 4.83590182, - -20.6276055, - 19.07518597, - 4.9641991, - -5.52570226, - 10.37834134, - 0.53685622, - -5.21721739, - 12.00234068, - -26.36114057, - 6.20088358, - 4.76729567, - 26.4359945, - 16.30961659, - 14.70620605, - -13.47771528, - 5.29218365, - 0.87290488, - 10.19441762, - 2.96687215, - -0.70745918, - -12.201803, - -10.65010904, - -5.90814342, - -25.17510006, - -5.90629019, - 26.53638293, - 27.44054429, - -11.75430314, - -3.72779609, - 10.28197421, - -11.58444132, - -18.09351946, - -22.09520524, - -30.04023056, - -16.7551763, - -12.34913637, - 30.27897114, - -13.98790656, - 0.65758253, - 3.75770261, - 4.81132118, - -11.18005293, - 5.85445303, - 18.88336475, - -7.34885733, - -35.76094848, - -3.39495953, - 5.52774132, - -8.38126488, - 10.07413613, - -1.96825956, - 17.97041089, - -2.47085774, - 19.5700424, - -49.46295186, - 0.12541183, - -6.95592842, - -0.33562953, - 12.9269603, - -13.6288284, - -9.51211543, - 21.22778867, - -2.81344371, - -9.66434107, - -15.41551695, - -30.15406401, - 14.65903841, - 2.51729041, - 17.70711379, - 13.21615045, - 12.55899318, - 11.45749114, - -25.67659992, - 13.00876054, - -1.52499005, - 27.45026658, - -4.36110401, - 10.01664277, - -11.24470769, - -6.79411522, - -1.67106503, - -2.59389537, - 11.72310319, - -0.84126818, - 2.03886137, - -11.25047383, - -8.60679631, - -23.1676229, - 22.83532544, - -25.2657054, - -15.49795527, - -1.62890474, - -15.49504251, - 27.26973702, - -8.00652979, - 6.87541861, - -3.61878753, - 0.82889822, - -2.88891667, - 6.13730358, - 12.55884424, - -2.24121286, - 45.90285087, - 34.43108722, - -13.32567113, - 19.0988537, - 2.16242269, - 17.45654791, - 18.17472208, - 18.28023357, - -20.34869744, - -11.21275755, - 6.0583063, - -10.38857432, - -7.30056744, - -8.85395997, - -17.19779724, - -8.12087822, - 2.46058364, - 29.35061395, - -6.53820405, - 10.3522653, - 8.54478485, + 0.36827213, + -5.5163417, + -0.5401933, + 4.9345136, + 1.9500562, + -8.317982, + 7.6919675, + 2.0017896, + -2.228223, + 4.185012, + 0.21649882, + -2.1038194, + 4.8398795, + -10.629993, + 2.5004756, + 1.9223818, + 10.660178, + 6.5767703, + 5.930208, + -5.4348187, + 2.134046, + 0.35198915, + 4.110833, + 1.1963683, + -0.2852855, + -4.9203086, + -4.294598, + -2.3824322, + -10.151715, + -2.381672, + 10.700656, + 11.065256, + -4.7398543, + -1.5032191, + 4.146144, + -4.671359, + -7.296113, + -8.909774, + -12.113555, + -6.7564373, + -4.9797287, + 12.209831, + -5.640541, + 0.26517656, + 1.5152706, + 1.9401368, + -4.5083017, + 2.360779, + 7.614604, + -2.963391, + -14.4204, + -1.3690065, + 2.2290342, + -3.379704, + 4.062335, + -0.79368806, + 7.246479, + -0.9963608, + 7.891514, + -19.945663, + 0.05056739, + -2.8049488, + -0.13533422, + 5.2127314, + -5.4957542, + -3.8357036, + 8.559985, + -1.1345087, + -3.8970866, + -6.2162213, + -12.159465, + 5.9111843, + 1.01508, + 7.1402836, + 5.3293405, + 5.0643544, + 4.6201696, + -10.35396, + 5.245707, + -0.6149189, + 11.069167, + -1.7585993, + 4.0391564, + -4.5343585, + -2.7396827, + -0.6738493, + -1.0459726, + 4.7272754, + -0.33924684, + 0.82215333, + -4.5366936, + -3.470642, + -9.342218, + 9.208217, + -10.188267, + -6.2494574, + -0.6568484, + -6.248293, + 10.996386, + -3.2285926, + 2.7724752, + -1.4592619, + 0.3342465, + -1.1649454, + 2.4748354, + 5.0642943, + -0.9037652, + 18.51007, + 13.884152, + -5.3735065, + 7.7014976, + 0.87198716, + 7.039267, + 7.328861, + 7.371412, + -8.205494, + -4.5214834, + 2.4429836, + -4.1891418, + -2.9439118, + -3.5703099, + -6.9349294, + -3.274712, + 0.9922171, + 11.83549, + -2.6364837, + 4.174497, + 3.4456437, ]], }, } @@ -384,60 +495,159 @@ def test_tcav_sample_from_positive(self): 'p_val': 0.80489, 'random_mean': 0.53333, 'result': { - 'score': - 0.8, + 'score': 0.8, 'cos_sim': [ - 0.09527, -0.20442, 0.05141, 0.14985, 0.06750, -0.28244, - -0.11022, -0.14479 + 0.09527, + -0.20442, + 0.05141, + 0.14985, + 0.06750, + -0.28244, + -0.11022, + -0.14479, ], 'dot_prods': [ - 152.48776, -335.64998, 82.99588, 247.80113, 109.53684, - -461.81805, -181.29095, -239.47817 + 79.91107, + -175.89749, + 43.49391, + 129.86026, + 57.40278, + -242.01596, + -95.00559, + -125.49859, ], - 'accuracy': - 1.0, + 'accuracy': 1.0, 'cav': [[ - -9.18092006e+00, -3.25423105e+00, -1.95599351e+01, - 7.90544869e+00, 2.71945760e-01, 9.21655507e+00, -7.10360094e-01, - 1.06792122e+01, -4.22297728e+00, -5.88364071e+00, - -5.90468259e+00, 3.02462186e-01, 2.73947077e+01, - -4.21289488e+00, 4.10535370e+00, 2.90032257e+00, - -1.15293947e+01, -6.05013625e+00, -9.63492785e+00, - -1.24673936e+01, -7.16846202e+00, 1.65560561e+01, - 3.91797282e+00, 9.43190766e+00, 1.13459987e+01, -3.52568932e+00, - 9.13124442e-01, 9.91573080e-01, -3.25439546e+00, - -9.93854182e-01, -1.07992122e+01, 3.11628229e+00, - 6.59499537e+00, -1.36178363e+01, 1.29858952e+01, 1.03866086e+01, - 1.59031300e+01, -5.56392889e+00, -3.77058407e+00, - -9.32599755e+00, -9.47891226e-01, 2.05803580e+01, - -1.37954357e+01, -7.73649342e+00, -9.07151538e+00, - 9.29099926e-01, -1.08088474e+01, -7.80737270e-01, - 2.40018316e+00, 1.09211064e+01, -3.27414948e+01, - -2.34077057e+01, 1.17713587e+01, 1.81635854e+00, 2.31977099e+01, - 1.61744777e+00, 1.17686967e+00, -3.88545806e+00, 2.46277754e+00, - -4.51580108e+01, 9.32061903e+00, -1.73902286e+01, - -3.66235470e+00, 3.26925824e+01, 3.21295843e+00, - -1.79316338e+01, 2.01011182e+01, 3.49235823e+00, 1.52195300e+01, - -5.04835360e+00, 1.45131574e+01, 1.59716750e+01, - -2.96747872e+00, 5.25282201e+00, -1.98576797e+01, - -1.46852052e+00, 1.58219867e+00, -5.74221070e-01, - 1.16208072e+01, 5.11250274e-01, 5.52448443e+00, 1.11949046e+01, - -6.93072443e-02, -7.17318663e+00, -8.48821209e+00, - -8.71016927e+00, -1.05849366e+01, 8.71665351e+00, - 1.24408289e+01, 4.21549502e+00, -2.07208098e+01, 1.56304334e+01, - -4.81732341e+00, 3.20792625e+01, -1.20141017e+01, - -1.73165977e+01, -6.54369967e+00, 3.16826359e+00, - -1.19574116e+01, -5.90525906e+00, 1.08200571e-01, - -9.32849433e+00, -1.29775955e+00, -1.20217051e+01, - 1.30472736e+01, 2.25283409e+01, 7.04978975e-02, 5.09369848e-01, - 1.27449879e+01, -2.16621823e+00, -8.19007901e+00, - -4.16839353e+00, 9.41498786e+00, 6.79635008e+00, 4.88519035e+00, - 2.63718274e+00, -5.28336147e-03, 1.26470921e+00, - -3.35551546e-01, 8.26617183e+00, 2.58916933e+00, - -8.54040920e+00, -1.18897963e+00, -5.09214401e+00, - 7.85278007e+00, -1.26074616e+01, -3.05422845e+00, 3.79508590e+00 - ]] - } + -4.8112683, + -1.7053839, + -10.250393, + 4.1428509, + 0.14250877, + 4.8299484, + -0.37227121, + 5.5964441, + -2.2130635, + -3.0833187, + -3.0943539, + 0.15851586, + 14.356208, + -2.2077665, + 2.1514158, + 1.5198991, + -6.041996, + -3.170579, + -5.0491924, + -6.5335569, + -3.7566402, + 8.6762218, + 2.053216, + 4.9427938, + 5.9458685, + -1.8476292, + 0.47852561, + 0.51963037, + -1.7054616, + -0.52083009, + -5.6593351, + 1.6330978, + 3.4561174, + -7.1364303, + 6.8052559, + 5.4431267, + 8.3340492, + -2.9157789, + -1.9759707, + -4.8873239, + -0.49674106, + 10.785144, + -7.229485, + -4.0543032, + -4.7539325, + 0.48690283, + -5.6643939, + -0.40914419, + 1.25781, + 5.7232237, + -17.158203, + -12.266828, + 6.1687803, + 0.95185655, + 12.156774, + 0.84763551, + 0.61674112, + -2.0361741, + 1.2906191, + -23.665077, + 4.8844857, + -9.113369, + -1.91926, + 17.132561, + 1.6837527, + -9.3970757, + 10.534003, + 1.8301795, + 7.9758077, + -2.6455872, + 7.6056347, + 8.3699665, + -1.5551119, + 2.7527373, + -10.406433, + -0.76957136, + 0.82914901, + -0.30093738, + 6.0898943, + 0.26792961, + 2.8951075, + 5.8666959, + -0.03632164, + -3.7591085, + -4.4482446, + -4.5645723, + -5.5470419, + 4.5679674, + 6.5196161, + 2.2091303, + -10.858754, + 8.1911459, + -2.5245326, + 16.811152, + -6.2959967, + -9.0747662, + -3.429239, + 1.6603287, + -6.2662807, + -3.0946536, + 0.056704286, + -4.8886104, + -0.68009311, + -6.2999897, + 6.8374286, + 11.805986, + 0.036961604, + 0.2669405, + 6.6790109, + -1.135214, + -4.2920089, + -2.1844563, + 4.933918, + 3.5616317, + 2.560091, + 1.382026, + -0.002763835, + 0.66278166, + -0.17584141, + 4.3318915, + 1.3568529, + -4.4756227, + -0.62307918, + -2.6685481, + 4.1152539, + -6.6069341, + -1.6005683, + 1.988831, + ]], + }, } testing_utils.assert_deep_almost_equal(self, expected, result[0]) @@ -479,60 +689,193 @@ def test_relative_tcav(self): 'result': { 'score': 1.0, 'cos_sim': [ - 0.9999999581246426, 0.049332143689572144, 0.8987945047547466, - -0.41858423757857954, 0.6908297036543664, -0.5167857909664919, - 0.8423017503220364, -0.005793079244916016, 0.8334491603894322, - -0.4054645113448612, 0.7616102123736647, -0.4578596155267783, - 0.8366905563807711, -0.27390786544756535, 0.7325538474066896, - 0.5190287630768531, 0.8145227936096425, 0.02005592868363552, - -0.1143256029298114, -0.1221480700842533, 0.6852995739227957, - 0.3984620730733816, 0.5211149530112407, 0.5909723902471223 + 0.9999999581246426, + 0.049332143689572144, + 0.8987945047547466, + -0.41858423757857954, + 0.6908297036543664, + -0.5167857909664919, + 0.8423017503220364, + -0.005793079244916016, + 0.8334491603894322, + -0.4054645113448612, + 0.7616102123736647, + -0.4578596155267783, + 0.8366905563807711, + -0.27390786544756535, + 0.7325538474066896, + 0.5190287630768531, + 0.8145227936096425, + 0.02005592868363552, + -0.1143256029298114, + -0.1221480700842533, + 0.6852995739227957, + 0.3984620730733816, + 0.5211149530112407, + 0.5909723902471223, ], 'dot_prods': [ - 1385.1480610241554, 69.95638452724207, 1239.4947646060161, - -595.253135700978, 971.5880156862692, -725.0749813217176, - 1182.8641913758102, -8.149647641120662, 1146.5803071544124, - -576.4043054391316, 1038.3510704649307, -648.097269442522, - 1154.4720122394317, -378.32103870822493, 1024.066390571124, - 738.6959135414066, 1139.7963358416857, 28.691395032352318, - -167.37808507284706, -176.4474746971391, 959.5159619261449, - 562.8772536987927, 716.7270332848395, 840.7031847912738 + 787.2422, + 39.759335, + 704.46094, + -338.30914, + 552.1974, + -412.09283, + 672.27515, + -4.631728, + 651.6533, + -327.5965, + 590.1418, + -368.34293, + 656.13855, + -215.01688, + 582.0231, + 419.83423, + 647.7977, + 16.306522, + -95.128555, + -100.28313, + 545.3363, + 319.90857, + 407.34833, + 477.8097, ], 'accuracy': 0.5, 'cav': [[ - 2.03451865, -13.17604027, -8.97477873, 14.2175882, 1.75463727, - -0.46571316, 15.63798371, 9.93142109, -16.8851253, 9.11427972, - -11.06386059, 7.19105954, -17.18586585, -4.17196936, - 29.00569093, 4.33784787, 12.33925142, -15.43126877, - -24.48932802, 9.00534947, -2.19966647, -0.43898864, 10.17245833, - -1.75776027, 5.32707314, -10.89541682, -2.57798881, 4.57338285, - -1.9818453, 8.84294916, -5.4750157, 4.17024913, -19.01907373, - -1.23878054, 0.78451806, -11.60109751, 6.45912611, 1.2580972, - -11.36715828, 3.96784638, 5.01414527, -8.80780738, -17.95446165, - 6.28686056, 11.37040026, -9.99612493, -5.1080606, 9.22049438, - -6.30251447, 3.63926025, 13.14918974, 14.03763789, 5.7383655, - -8.20873775, 1.60465614, -17.91190497, -13.33602912, - -4.49996902, 28.75901011, -3.47341192, -3.82988803, 7.55923052, - -8.09407339, -16.11918298, -1.71507548, -1.36150357, - -5.86025439, 15.68441846, -3.51353798, 2.43790252, -2.19348517, - 5.0211597, 2.37646679, 13.41654008, 0.55570348, -6.05668506, - -8.42704919, -5.02851713, 14.84420539, -7.34185174, 4.03091355, - -5.09645307, 8.2008516, 1.08925377, -0.58413633, 9.92903691, - 2.14726806, 13.30179969, 13.57605882, -2.90356308, -5.45706524, - 10.79901632, -7.80617599, 17.51044709, -7.5189762, -16.52918401, - -0.52797854, -2.44961364, 10.13560444, -13.19007029, - -0.93666233, 0.54694589, 1.92814453, -4.7547722, -1.39708064, - 6.24580764, -1.41235513, -2.47243578, 10.31137604, -11.16312467, - -10.42280945, -13.08010871, 17.36711388, 5.96436519, - -11.97783485, 18.54090287, 7.91701915, -17.52516079, - -13.20368127, -26.3674698, -2.45510052, 5.56414756, 10.80934702, - -8.21923153, -11.8954632, -4.44263104, -3.42691711, -4.79007185 - ]] + 1.156309, + -7.4885406, + -5.1007705, + 8.080497, + 0.9972395, + -0.26468632, + 8.88777, + 5.64447, + -9.596578, + 5.1800556, + -6.2880955, + 4.087003, + -9.767502, + -2.3711176, + 16.485243, + 2.465395, + 7.0129533, + -8.770288, + -13.91839, + 5.118149, + -1.2501711, + -0.24950005, + 5.781469, + -0.9990161, + 3.0276148, + -6.192359, + -1.4651864, + 2.5992606, + -1.1263721, + 5.025849, + -3.1116967, + 2.3701432, + -10.809398, + -0.70405656, + 0.4458724, + -6.593426, + 3.6710117, + 0.7150358, + -6.4604683, + 2.2551076, + 2.8497634, + -5.0058727, + -10.204328, + 3.5731046, + 6.462311, + -5.6812506, + -2.9031432, + 5.2404213, + -3.5820036, + 2.0683544, + 7.4732795, + 7.9782243, + 3.261372, + -4.6653953, + 0.9119986, + -10.180145, + -7.579469, + -2.5575323, + 16.345041, + -1.974097, + -2.1766987, + 4.2962556, + -4.6002207, + -9.1612625, + -0.97475797, + -0.7738031, + -3.3306477, + 8.914165, + -1.9969038, + 1.3855696, + -1.2466589, + 2.8537538, + 1.3506539, + 7.6252246, + 0.31582844, + -3.442287, + -4.789475, + -2.8579352, + 8.436631, + -4.1727037, + 2.2909505, + -2.8965473, + 4.6609116, + 0.6190737, + -0.33198914, + 5.6431217, + 1.2203902, + 7.560014, + 7.7158875, + -1.6502286, + -3.101498, + 6.137571, + -4.4366007, + 9.951975, + -4.273374, + -9.394284, + -0.30007574, + -1.3922303, + 5.7605214, + -7.496512, + -0.5323485, + 0.31085476, + 1.0958543, + -2.7023506, + -0.79402196, + 3.5497754, + -0.8027026, + -1.4051963, + 5.8604226, + -6.344507, + -5.9237537, + -7.434017, + 9.870518, + 3.3898156, + -6.8075423, + 10.537633, + 4.4995995, + -9.960338, + -7.5042505, + -14.985823, + -1.3953457, + 3.1623576, + 6.143443, + -4.67136, + -6.7607274, + -2.5249496, + -1.9476708, + -2.722412, + ]], }, 'p_val': None, 'random_mean': 0.9285714285714286, 'split_size': 3, - 'num_runs': 1 + 'num_runs': 1, } testing_utils.assert_deep_almost_equal(self, expected, result[0], places=3) @@ -571,60 +914,193 @@ def test_relative_tcav(self): 'result': { 'score': 0.0, 'cos_sim': [ - 0.2731987606830683, 0.427838045403812, 0.3166440584420665, - -0.1358964965831398, 0.5616614702946262, -0.16511808390168164, - -0.05103355252438478, -0.16945565920473257, 0.28148962348967155, - -0.18169036476392003, 0.33244873698665106, -0.13316476546155087, - 0.15226772288202886, -0.05534469666649352, 0.2886150002073456, - 0.33888135113008555, 0.12875301375254147, 0.046908665182593096, - -0.052445114502024985, 0.088858405172313, 0.219517174438115, - 0.35833013079793435, 0.2291162415605806, 0.3635686086637199 + 0.2731987606830683, + 0.427838045403812, + 0.3166440584420665, + -0.1358964965831398, + 0.5616614702946262, + -0.16511808390168164, + -0.05103355252438478, + -0.16945565920473257, + 0.28148962348967155, + -0.18169036476392003, + 0.33244873698665106, + -0.13316476546155087, + 0.15226772288202886, + -0.05534469666649352, + 0.2886150002073456, + 0.33888135113008555, + 0.12875301375254147, + 0.046908665182593096, + -0.052445114502024985, + 0.088858405172313, + 0.219517174438115, + 0.35833013079793435, + 0.2291162415605806, + 0.3635686086637199, ], 'dot_prods': [ - 452.17220644153525, 724.9460578876271, 521.776546745851, - -230.9170522777958, 943.8754747127095, -276.8190148523963, - -85.63511897570154, -284.8487792023684, 462.71830216201926, - -308.62790255581496, 541.5830529968077, -225.2299308998058, - 251.04716264718752, -91.33998249705493, 482.0991668852444, - 576.3029773313335, 215.28329927312336, 80.18458502795752, - -91.74640483442752, 153.37559992294862, 367.2562273288043, - 604.8378479001944, 376.53473821563625, 618.003311205616 + 228.6362, + 366.5617, + 263.83105, + -116.76074, + 477.26117, + -139.97066, + -43.300613, + -144.03076, + 233.96881, + -156.05446, + 273.84595, + -113.88513, + 126.939316, + -46.185066, + 243.76842, + 291.40173, + 108.85574, + 40.54453, + -46.390568, + 77.552826, + 185.69937, + 305.83014, + 190.39096, + 312.48724, ], 'accuracy': 0.5, 'cav': [[ - 7.55329281, -2.62460237, -9.0024543, 0.59818666, -14.58093322, - -15.42218557, -5.5354822, 12.364417, 21.63543734, -12.10592188, - -28.13137179, 14.31976269, -31.59852801, -9.5688074, - 12.95670217, -22.25037747, 16.35172203, 1.05301289, -31.3269959, - -2.59048782, 19.77513966, 10.23254963, 4.18399631, 4.87700827, - -16.33114267, 2.79754082, 10.54735581, 11.78229815, - -12.25723096, 4.70124727, -11.24257086, -5.43199386, - -7.69344958, 14.56370698, -5.75688441, -5.26137455, 23.44984724, - -6.31077994, 17.08448569, 9.05777822, 1.6895136, -17.90790926, - -14.34180658, 10.53637, 20.52315338, 13.25800162, 10.93816225, - -3.75636305, -7.45849454, -3.97231903, -2.56363646, 7.11917039, - 8.16617295, -7.89912694, 12.99981985, -21.34898036, - -17.63337856, -2.63946913, -4.40439706, 6.52652447, 3.80778434, - -9.19669592, -6.7804855, -1.69738751, 14.07850725, -19.72428768, - -4.39987548, 0.52300402, -1.32919268, -1.17373043, 25.24466652, - 1.10433416, 2.28887964, 2.73226767, 11.38988286, -15.36477422, - 1.72328432, -10.98681852, 4.09940512, 1.35212746, -11.08208377, - 15.02775212, 24.25939889, 2.5819625, -4.82504369, -2.8454034, - 24.2562686, -4.11429068, -1.74594319, -19.37503033, 2.29546159, - 6.19942707, 3.62587758, 17.79947087, 1.53031934, 2.38564874, - -16.59565939, -5.20520209, -4.84197479, -9.78838634, - -12.3678921, 16.95331818, 7.66526629, 20.39393112, 2.87282507, - -17.29492882, -7.67759675, 2.78484882, 7.79825779, -5.36932978, - 12.74328979, -5.22601154, 3.36537017, -16.2269691, 5.08010908, - 8.33516097, 6.93697789, -30.79130942, 8.65766065, 14.05914846, - 1.90387642, 7.06612428, 13.16682386, -7.97255194, -15.33183742, - 2.61200359, -2.05912859, 10.68784471 - ]] + 3.8192496, + -1.3270987, + -4.5519996, + 0.30246577, + -7.372711, + -7.798068, + -2.798964, + 6.25194, + 10.93974, + -6.1212344, + -14.2243395, + 7.240647, + -15.977484, + -4.838373, + 6.551419, + -11.250678, + 8.268084, + 0.53245205, + -15.840163, + -1.3098477, + 9.9991045, + 5.173985, + 2.1155946, + 2.4660072, + -8.257678, + 1.4145501, + 5.333165, + 5.957603, + -6.197754, + 2.3771377, + -5.684692, + -2.7466276, + -3.8901126, + 7.364, + -2.9109194, + -2.6603553, + 11.857175, + -3.1909864, + 8.638606, + 4.5799747, + 0.85427845, + -9.054963, + -7.2517867, + 5.327609, + 10.377333, + 6.7037725, + 5.530765, + -1.899365, + -3.7713125, + -2.0085628, + -1.29627, + 3.599725, + 4.129141, + -3.994111, + 6.573218, + -10.7948885, + -8.916138, + -1.3346167, + -2.2270472, + 3.3000827, + 1.9253712, + -4.6502147, + -3.4284825, + -0.85826504, + 7.118656, + -9.973384, + -2.224755, + 0.26445186, + -0.6721002, + -0.59349084, + 12.764709, + 0.55839765, + 1.1573459, + 1.3815417, + 5.7591877, + -7.769041, + 0.87135667, + -5.555376, + 2.0728283, + 0.6836813, + -5.603544, + 7.598627, + 12.266522, + 1.3055462, + -2.439729, + -1.4387506, + 12.264931, + -2.0803547, + -0.8828149, + -9.796794, + 1.1606747, + 3.1346762, + 1.8334013, + 9.000124, + 0.7737874, + 1.2062835, + -8.391424, + -2.6319501, + -2.448292, + -4.9493966, + -6.2537127, + 8.572276, + 3.875862, + 10.311994, + 1.452615, + -8.745012, + -3.882103, + 1.4081296, + 3.943109, + -2.7149363, + 6.443512, + -2.6424878, + 1.7016634, + -8.205001, + 2.5687048, + 4.2145853, + 3.507609, + -15.569323, + 4.377659, + 7.1088753, + 0.962679, + 3.5729153, + 6.657662, + -4.031242, + -7.7523932, + 1.3207365, + -1.0411787, + 5.4042, + ]], }, 'p_val': 0.42264973081037427, 'random_mean': 0.0, 'split_size': 3, - 'num_runs': 2 + 'num_runs': 2, } testing_utils.assert_deep_almost_equal(self, expected, result[0], places=3) @@ -670,75 +1146,195 @@ def test_relative_tcav(self): self.assertLen(result, 1) expected = [{ 'result': { - 'score': 0.42857142857142855, + 'score': 0.14285714285714285, 'cos_sim': [ - -0.1107393877916321, -0.0993967046974328, -0.2214985917242054, - 0.08132588965575606, -0.3590211572508748, 0.18708109817461333, - 0.000724498781128839, 0.09700473783330398, -0.25015742815240055, - 0.16108236033785076, -0.10283274286140846, 0.0972663321478731, - -0.05924679176256152, -0.048499696342091746, - -0.4357117016074766, -0.593245752003111, -0.3645147796989344, - -0.5507605083253673, -0.27914997949782694, -0.30908550968594417, - -0.5584676299422896, -0.16983339994284577, -0.42587740852240746, - -0.37482298817032594 + 0.065847985, + 0.38948667, + 0.045807727, + 0.279943, + 0.12873492, + 0.15351126, + -0.046762887, + 0.16203871, + -0.06715148, + 0.16850145, + -0.4235319, + -0.116421655, + -0.32443357, + 0.04368962, + 0.17103079, + 0.26918238, + -0.044799704, + 0.17242716, + 0.16247045, + -0.047080424, + -0.009721848, + 0.02543351, + -0.0983404, + 0.37685287, ], 'dot_prods': [ - -261.4389298435066, -240.23776409902007, -520.6275907607769, - 197.11495117497446, -860.6035066083074, 447.3775519523981, - 1.7341104803878409, 232.59170976304426, -586.5576327736542, - 390.2961568516803, -238.95427152619726, 234.6617547723058, - -139.3334215524385, -114.17392512371171, -1038.149036709951, - -1439.0663895591745, -869.3828698612926, -1342.899780229334, - -696.569760699206, -760.9907977738051, -1332.7284530349625, - -408.90435403478875, -998.3360993150825, -908.8111404537224 + 58.66085, + 355.22186, + 40.628685, + 256.03424, + 116.44404, + 138.52313, + -42.23555, + 146.60806, + -59.414257, + 154.05923, + -371.37036, + -105.98669, + -287.9082, + 38.81003, + 153.77032, + 246.39426, + -40.31892, + 158.64423, + 152.9816, + -43.740013, + -8.754478, + 23.106956, + -86.988525, + 344.79193, ], - 'accuracy': 0.75, + 'accuracy': 0.5, 'cav': [[ - -2.01128835e+01, -2.53418963e+01, 3.02074307e+01, - 3.78726319e+00, -6.30370696e+00, 1.74700275e+00, 2.26769890e+01, - -1.06487415e+01, -1.76710295e+00, 1.53911930e+01, - 6.54822833e+00, 2.02141240e+00, 1.16919236e+00, -4.59410912e-01, - 4.54187735e+00, 1.98659207e+01, 1.01712167e-01, 1.18509678e+01, - 3.11218189e+01, 1.60491967e+01, -2.84655620e+01, - -1.08942472e+01, -1.64852881e+01, 4.79670684e+00, - 3.08027345e+01, -2.64932823e+01, 1.14056950e+00, - -1.45201048e+01, -8.41149151e+00, -1.26598740e+01, - -5.87020201e+00, -2.87027220e+00, 2.13724773e+01, - 3.16488669e+01, -1.38671518e+01, 2.25443206e+01, - -1.93658542e+01, 1.47272427e+01, 1.42369662e+01, - -7.98238134e+00, -2.34073887e+01, 3.63425744e+00, - 3.20426643e+00, -1.57109214e+01, -1.87085664e+01, - 2.04525909e-02, 1.75499187e+01, -1.89476242e+00, 1.22602038e+01, - -8.30212717e-01, 6.89138456e+00, -4.76848738e+01, - -1.02832284e+01, 1.25879647e+01, -2.34436812e+01, - 3.46227988e+01, 3.98466501e+00, -4.18899781e+01, 2.31382829e+01, - 9.44899112e+00, -5.55529692e+00, 2.99535098e+01, 1.91656336e+01, - 1.90906964e+01, -1.97222943e+01, -8.81455291e+00, - -4.31148519e-01, 1.35055378e+01, -2.93433908e+01, - 1.41053039e+01, 7.31250277e+00, 1.84754657e+01, -3.59438637e+00, - -1.94830172e+01, 1.82690513e+00, 1.11006786e+01, - -1.98780467e+01, 9.70483509e+00, 3.10464830e+01, - -1.58164540e+00, 3.92983591e+01, -1.94148650e+01, - -1.78218017e+01, 1.68395976e+01, 1.88509800e+01, - -4.67428605e+00, -4.11605219e+01, 4.20027328e+00, - -1.78356055e+01, 2.93033407e+01, 3.67970865e+00, 4.43803533e+00, - -5.86478123e+00, -1.89515809e+01, -4.15883069e+00, - -2.53725359e+01, -2.06732149e+01, 3.51060797e+00, - 7.32087133e+00, 7.98146613e-01, 3.38068707e+00, -2.48928693e+01, - 1.99907796e+00, 2.12458621e+00, -1.33852676e+01, 1.42904487e+01, - 1.33286404e-02, -3.82017252e+00, -1.04635257e+01, - 1.00280360e+01, 8.93273804e+00, 5.26520675e+00, -1.28270015e+01, - 3.54992057e+00, 2.32608393e+00, -1.84115520e+01, - -1.74357882e+00, 2.53797871e+01, -1.75258907e+01, - -4.22843710e+01, -1.49635329e+01, -3.77852872e+00, - -1.25404291e+01, 1.17741455e+01, 3.35462077e+01, 1.65145075e+01, - 4.27775903e-01, 8.26403983e+00 - ]] + -4.1506495, + -7.1951985, + 0.90339786, + -6.1899633, + -7.77931, + 16.267708, + -5.424457, + 11.222493, + -5.5136533, + -9.43756, + -3.5046537, + -3.9068658, + 0.4145894, + -3.6296427, + 0.42548138, + 6.6225533, + -1.3941894, + 3.5685213, + -7.6347156, + -1.7981998, + -3.6240344, + -0.7874432, + -14.922223, + -1.88698, + -12.165323, + -1.5065875, + 3.434664, + 10.669602, + -4.047451, + -7.260671, + -3.773127, + 5.5680423, + -2.8338368, + 4.741398, + -0.7754047, + -5.062835, + -3.7674167, + -4.9375997, + 2.438198, + -6.10431, + 4.24054, + -6.8066883, + -2.091162, + 7.4850273, + -6.7407784, + 5.9368405, + 6.8403163, + -3.2204735, + -3.2611327, + 1.8095824, + 8.339414, + 9.342779, + 4.9996133, + 2.5944111, + -3.047835, + -2.815008, + 6.255702, + 2.0537443, + -7.334257, + 8.804704, + -4.9964232, + -4.3621807, + -5.5490036, + -2.3311555, + 1.6228596, + -16.534676, + 3.4137661, + 3.0507114, + 10.216002, + -0.93795264, + 12.797617, + -8.083842, + 4.501012, + 6.415509, + 1.4108915, + 2.2607276, + 3.442493, + -8.117062, + 2.0640905, + -6.9452996, + 2.1540446, + 13.912465, + 2.6976292, + 0.25915417, + -4.7452064, + 1.5720334, + 3.373914, + -1.896033, + 7.259476, + -12.933163, + -4.5062537, + 1.4334662, + 16.052553, + -7.655357, + 4.448883, + -12.834609, + 3.7579305, + 17.18403, + 2.8025444, + -6.1295238, + -1.7349106, + 0.6179918, + 3.1866891, + -2.8977861, + 7.8064804, + -2.8491228, + 3.823132, + -11.68467, + -1.6850648, + 3.0090592, + 6.856376, + -0.3384791, + 10.425631, + -0.5820264, + 2.605319, + 5.800684, + -3.4050944, + -1.4920071, + 4.2210917, + -0.73371845, + 13.576842, + 12.256167, + 1.5905027, + -2.1719818, + -11.575534, + 2.123623, + 8.175223, + -0.4228358, + ]], }, - 'p_val': 0.04400624968940752, + 'p_val': 0.001885015813683682, 'random_mean': 0.9642857142857143, 'split_size': 5, - 'num_runs': 2 + 'num_runs': 2, }] testing_utils.assert_deep_almost_equal(self, expected, result, places=3) diff --git a/lit_nlp/examples/glue/model_int_test.py b/lit_nlp/examples/glue/model_int_test.py index 366a9428..4907efe6 100644 --- a/lit_nlp/examples/glue/model_int_test.py +++ b/lit_nlp/examples/glue/model_int_test.py @@ -8,7 +8,6 @@ --guitar_detach """ -from typing import Any from absl.testing import absltest from absl.testing import parameterized from lit_nlp.examples.glue import models as glue_models @@ -18,15 +17,16 @@ # TODO(b/254110131): Fix test flakiness. Expand to SST-2, STS-B, and MNLI class GlueModelsIntTest(parameterized.TestCase): - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) + @classmethod + def setUpClass(cls): + super().setUpClass() # Create the SST-2 Model model_path = "https://storage.googleapis.com/what-if-tool-resources/lit-models/sst2_tiny.tar.gz" # pylint: disable=line-too-long if model_path.endswith(".tar.gz"): model_path = file_cache.cached_path( model_path, extract_compressed_file=True ) - self.sst2_model = glue_models.SST2Model(model_path) + cls.sst2_model = glue_models.SST2Model(model_path) @parameterized.named_parameters( dict( @@ -62,16 +62,15 @@ def __init__(self, *args: Any, **kwargs: Any): ) def test_sst2_model_predict(self, config: dict[str, bool]): # Configure model. - if config: - self.sst2_model.config = glue_models.GlueModelConfig( - # Include the SST-2 default config options - text_a_name="sentence", - text_b_name=None, - labels=["0", "1"], - null_label_idx=0, - # Add the output-affecting config options - **config - ) + self.sst2_model.config = glue_models.GlueModelConfig( + # Include the SST-2 default config options + text_a_name="sentence", + text_b_name=None, + labels=["0", "1"], + null_label_idx=0, + # Add the output-affecting config options + **config + ) # Run prediction to ensure no failure. model_in = [{"sentence": "test sentence"}]