diff --git a/scripts/algorithms/StepGLM.dml b/scripts/algorithms/StepGLM.dml deleted file mode 100644 index 213f373b1b9..00000000000 --- a/scripts/algorithms/StepGLM.dml +++ /dev/null @@ -1,1196 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -# -# THIS SCRIPT CHOOSES A GLM REGRESSION MODEL IN A STEPWISE ALGIRITHM USING AIC -# EACH GLM REGRESSION IS SOLVED USING NEWTON/FISHER SCORING WITH TRUST REGIONS -# -# INPUT PARAMETERS: -# --------------------------------------------------------------------------------------------- -# NAME TYPE DEFAULT MEANING -# --------------------------------------------------------------------------------------------- -# X String --- Location to read the matrix X of feature vectors -# Y String --- Location to read response matrix Y with 1 column -# B String --- Location to store estimated regression parameters (the betas) -# S String --- Location to write the selected features ordered as computed by the algorithm -# O String " " Location to write the printed statistics; by default is standard output -# link Int 2 Link function code: 1 = log, 2 = Logit, 3 = Probit, 4 = Cloglog -# yneg Double 0.0 Response value for Bernoulli "No" label, usually 0.0 or -1.0 -# icpt Int 0 Intercept presence, X columns shifting and rescaling: -# 0 = no intercept, no shifting, no rescaling; -# 1 = add intercept, but neither shift nor rescale X; -# 2 = add intercept, shift & rescale X columns to mean = 0, variance = 1 -# tol Double 0.000001 Tolerance (epsilon) -# disp Double 0.0 (Over-)dispersion value, or 0.0 to estimate it from data -# moi Int 200 Maximum number of outer (Newton / Fisher Scoring) iterations -# mii Int 0 Maximum number of inner (Conjugate Gradient) iterations, 0 = no maximum -# thr Double 0.01 Threshold to stop the algorithm: if the decrease in the value of AIC falls below thr -# no further features are being checked and the algorithm stops -# fmt String "text" The betas matrix output format, such as "text" or "csv" -# --------------------------------------------------------------------------------------------- -# OUTPUT: Matrix beta, whose size depends on icpt: -# icpt=0: ncol(X) x 1; icpt=1: (ncol(X) + 1) x 1; icpt=2: (ncol(X) + 1) x 2 -# -# In addition, in the last run of GLM some statistics are provided in CSV format, one comma-separated name-value -# pair per each line, as follows: -# -# NAME MEANING -# ------------------------------------------------------------------------------------------- -# TERMINATION_CODE A positive integer indicating success/failure as follows: -# 1 = Converged successfully; 2 = Maximum number of iterations reached; -# 3 = Input (X, Y) out of range; 4 = Distribution/link is not supported -# BETA_MIN Smallest beta value (regression coefficient), excluding the intercept -# BETA_MIN_INDEX Column index for the smallest beta value -# BETA_MAX Largest beta value (regression coefficient), excluding the intercept -# BETA_MAX_INDEX Column index for the largest beta value -# INTERCEPT Intercept value, or NaN if there is no intercept (if icpt=0) -# DISPERSION Dispersion used to scale deviance, provided as "disp" input parameter -# or estimated (same as DISPERSION_EST) if the "disp" parameter is <= 0 -# DISPERSION_EST Dispersion estimated from the dataset -# DEVIANCE_UNSCALED Deviance from the saturated model, assuming dispersion == 1.0 -# DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value -# ------------------------------------------------------------------------------------------- -# -# HOW TO INVOKE THIS SCRIPT - EXAMPLE: -# hadoop jar SystemDS.jar -f StepGLM.dml -nvargs X=INPUT_DIR/X Y=INPUT_DIR/Y B=OUTPUT_DIR/betas -# S=OUTPUT_DIR_S/selected O=OUTPUT_DIR/stats link=2 yneg=-1.0 icpt=2 tol=0.00000001 -# disp=1.0 moi=100 mii=10 thr=0.01 fmt=csv -# -# THE StepGLM SCRIPT CURRENTLY SUPPORTS BERNOULLI DISTRIBUTION FAMILY AND THE FOLLOWING LINK FUNCTIONS ONLY! -# - LOG -# - LOGIT -# - PROBIT -# - CLOGLOG - -fileX = $X; -fileY = $Y; -fileB = $B; -intercept_status = ifdef ($icpt, 0); -thr = ifdef ($thr, 0.01); -bernoulli_No_label = ifdef ($yneg, 0.0); # $yneg = 0.0; -distribution_type = 2; - -bernoulli_No_label = as.double (bernoulli_No_label); - -# currently only the forward selection strategy in supported: start from one feature and iteratively add -# features until AIC improves -dir = "forward"; - -print("BEGIN STEPWISE GLM SCRIPT"); -print ("Reading X and Y..."); -X_orig = read (fileX); -Y = read (fileY); - -if (distribution_type == 2 & ncol(Y) == 1) { - is_Y_negative = (Y == bernoulli_No_label); - Y = cbind (1 - is_Y_negative, is_Y_negative); - count_Y_negative = sum (is_Y_negative); - if (count_Y_negative == 0) { - stop ("StepGLM Input Error: all Y-values encode Bernoulli YES-label, none encode NO-label"); - } - if (count_Y_negative == nrow(Y)) { - stop ("StepGLM Input Error: all Y-values encode Bernoulli NO-label, none encode YES-label"); - } -} - -num_records = nrow (X_orig); -num_features = ncol (X_orig); - -# BEGIN STEPWISE GENERALIZED LINEAR MODELS - -if (dir == "forward") { - - continue = TRUE; - columns_fixed = matrix (0, rows = 1, cols = num_features); - columns_fixed_ordered = matrix (0, rows = 1, cols = 1); - - # X_global stores the best model found at each step - X_global = matrix (0, rows = num_records, cols = 1); - - if (intercept_status == 0) { - # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered, " "); - } else { - # compute AIC of an empty model with only intercept (all Ys are constant) - all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered, " "); - } - print ("Best AIC without any features: " + AIC_best); - - # First pass to examine single features - AICs = matrix (AIC_best, rows = 1, cols = num_features); - parfor (i in 1:num_features) { - [AIC_1] = glm_fit (X_orig[,i], Y, intercept_status, num_features, columns_fixed_ordered, " "); - AICs[1,i] = AIC_1; - } - - # Determine the best AIC - column_best = 0; - for (k in 1:num_features) { - AIC_cur = as.scalar (AICs[1,k]); - if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) ) { - column_best = k; - AIC_best = as.scalar(AICs[1,k]); - } - } - - if (column_best == 0) { - print ("AIC of an empty model is " + AIC_best + " and adding no feature achieves more than " + (thr * 100) + "% decrease in AIC!"); - if (intercept_status == 0) { - # Compute AIC of an empty model with no features and no intercept (all Ys are zero) - [AIC_best] = glm_fit (X_global, Y, 0, num_features, columns_fixed_ordered, fileB); - } else { - # compute AIC of an empty model with only intercept (all Ys are constant) - ###all_ones = matrix (1, rows = num_records, cols = 1); - [AIC_best] = glm_fit (all_ones, Y, 0, num_features, columns_fixed_ordered, fileB); - } - }; - - print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); - columns_fixed[1,column_best] = 1; - columns_fixed_ordered[1,1] = column_best; - X_global = X_orig[,column_best]; - - while (continue) { - # Subsequent passes over the features - parfor (i in 1:num_features) { - if (as.scalar(columns_fixed[1,i]) == 0) { - - # Construct the feature matrix - X = cbind (X_global, X_orig[,i]); - - [AIC_2] = glm_fit (X, Y, intercept_status, num_features, columns_fixed_ordered, " "); - AICs[1,i] = AIC_2; - } - } - - # Determine the best AIC - for (k in 1:num_features) { - AIC_cur = as.scalar (AICs[1,k]); - if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) & (as.scalar(columns_fixed[1,k]) == 0) ) { - column_best = k; - AIC_best = as.scalar(AICs[1,k]); - } - } - - # cbind best found features (i.e., columns) to X_global - if (as.scalar(columns_fixed[1,column_best]) == 0) { # new best feature found - print ("Best AIC " + AIC_best + " achieved with feature: " + column_best); - columns_fixed[1,column_best] = 1; - columns_fixed_ordered = cbind (columns_fixed_ordered, as.matrix(column_best)); - if (ncol(columns_fixed_ordered) == num_features) { # all features examined - X_global = cbind (X_global, X_orig[,column_best]); - continue = FALSE; - } else { - X_global = cbind (X_global, X_orig[,column_best]); - } - } else { - continue = FALSE; - } - } - - # run GLM with selected set of features - print ("Running GLM with selected features..."); - [AIC] = glm_fit (X_global, Y, intercept_status, num_features, columns_fixed_ordered, fileB); - -} else { - stop ("Currently only forward selection strategy is supported!"); -} - - -################### UDFS USED IN THIS SCRIPT ################## - -glm_fit = function (Matrix[Double] X, Matrix[Double] Y, Int intercept_status, Double num_features_orig, Matrix[Double] Selected, String fileB) return (Double AIC) { - - # distribution family code: 1 = Power, 2 = Bernoulli/Binomial; currently only Bernouli distribution family is supported! - distribution_type = 2; # $dfam = 2; - variance_as_power_of_the_mean = 0.0; # $vpow = 0.0; - # link function code: 0 = canonical (depends on distribution), 1 = Power, 2 = Logit, 3 = Probit, 4 = Cloglog, 5 = Cauchit; - # currently only log (link = 1), logit (link = 2), probit (link = 3), and cloglog (link = 4) are supported! - link_type = ifdef ($link, 2); # $link = 2; - link_as_power_of_the_mean = 0.0; # $lpow = 0.0; - - dispersion = ifdef ($disp, 0.0); # $disp = 0.0; - eps = ifdef ($tol, 0.000001); # $tol = 0.000001; - max_iteration_IRLS = ifdef ($moi, 200); # $moi = 200; - max_iteration_CG = ifdef ($mii, 0); # $mii = 0; - - variance_as_power_of_the_mean = as.double (variance_as_power_of_the_mean); - link_as_power_of_the_mean = as.double (link_as_power_of_the_mean); - - dispersion = as.double (dispersion); - eps = as.double (eps); - - # Default values for output statistics: - regularization = 0.0; - termination_code = 0.0; - min_beta = NaN; - i_min_beta = NaN; - max_beta = NaN; - i_max_beta = NaN; - intercept_value = NaN; - dispersion = NaN; - estimated_dispersion = NaN; - deviance_nodisp = NaN; - deviance = NaN; - - ##### INITIALIZE THE PARAMETERS ##### - - num_records = nrow (X); - num_features = ncol (X); - zeros_r = matrix (0, rows = num_records, cols = 1); - ones_r = 1 + zeros_r; - - # Introduce the intercept, shift and rescale the columns of X if needed - - if (intercept_status == 1 | intercept_status == 2) { # add the intercept column - X = cbind (X, ones_r); - num_features = ncol (X); - } - - scale_lambda = matrix (1, rows = num_features, cols = 1); - if (intercept_status == 1 | intercept_status == 2) { - scale_lambda [num_features, 1] = 0; - } - - if (intercept_status == 2) { # scale-&-shift X columns to mean 0, variance 1 - # Important assumption: X [, num_features] = ones_r - avg_X_cols = t(colSums(X)) / num_records; - var_X_cols = (t(colSums (X ^ 2)) - num_records * (avg_X_cols ^ 2)) / (num_records - 1); - is_unsafe = (var_X_cols <= 0); - scale_X = 1.0 / sqrt (var_X_cols * (1 - is_unsafe) + is_unsafe); - scale_X [num_features, 1] = 1; - shift_X = - avg_X_cols * scale_X; - shift_X [num_features, 1] = 0; - rowSums_X_sq = (X ^ 2) %*% (scale_X ^ 2) + X %*% (2 * scale_X * shift_X) + sum (shift_X ^ 2); - } else { - scale_X = matrix (1, rows = num_features, cols = 1); - shift_X = matrix (0, rows = num_features, cols = 1); - rowSums_X_sq = rowSums (X ^ 2); - } - - # Henceforth we replace "X" with "X %*% (SHIFT/SCALE TRANSFORM)" and rowSums(X ^ 2) - # with "rowSums_X_sq" in order to preserve the sparsity of X under shift and scale. - # The transform is then associatively applied to the other side of the expression, - # and is rewritten via "scale_X" and "shift_X" as follows: - # - # ssX_A = (SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: - # ssX_A = diag (scale_X) %*% A; - # ssX_A [num_features, ] = ssX_A [num_features, ] + t(shift_X) %*% A; - # - # tssX_A = t(SHIFT/SCALE TRANSFORM) %*% A --- is rewritten as: - # tssX_A = diag (scale_X) %*% A + shift_X %*% A [num_features, ]; - - # Initialize other input-dependent parameters - - lambda = scale_lambda * regularization; - if (max_iteration_CG == 0) { - max_iteration_CG = num_features; - } - - # Set up the canonical link, if requested [Then we have: Var(mu) * (d link / d mu) = const] - - if (link_type == 0) { - if (distribution_type == 1) { - link_type = 1; - link_as_power_of_the_mean = 1.0 - variance_as_power_of_the_mean; - } else { - if (distribution_type == 2) { - link_type = 2; - } - } - } - - # For power distributions and/or links, we use two constants, - # "variance as power of the mean" and "link_as_power_of_the_mean", - # to specify the variance and the link as arbitrary powers of the - # mean. However, the variance-powers of 1.0 (Poisson family) and - # 2.0 (Gamma family) have to be treated as special cases, because - # these values integrate into logarithms. The link-power of 0.0 - # is also special as it represents the logarithm link. - - num_response_columns = ncol (Y); - is_supported = 0; - if (num_response_columns == 2 & distribution_type == 2 & link_type >= 1 & link_type <= 4) { # BERNOULLI DISTRIBUTION - is_supported = 1; - } - if (num_response_columns == 1 & distribution_type == 2) { - print ("Error: Bernoulli response matrix has not been converted into two-column format."); - } - - if (is_supported == 1) { - - ##### INITIALIZE THE BETAS ##### - - [beta, saturated_log_l, isNaN] = - glm_initialize (X, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean, intercept_status, max_iteration_CG); - - # print(" --- saturated logLik " + saturated_log_l); - - if (isNaN == 0) { - - ##### START OF THE MAIN PART ##### - - sum_X_sq = sum (rowSums_X_sq); - trust_delta = 0.5 * sqrt (num_features) / max (sqrt (rowSums_X_sq)); - ### max_trust_delta = trust_delta * 10000.0; - log_l = 0.0; - deviance_nodisp = 0.0; - new_deviance_nodisp = 0.0; - isNaN_log_l = 2; - newbeta = beta; - g = matrix (0.0, rows = num_features, cols = 1); - g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); - accept_new_beta = 1; - reached_trust_boundary = 0; - neg_log_l_change_predicted = 0.0; - i_IRLS = 0; - - # print ("BEGIN IRLS ITERATIONS..."); - - ssX_newbeta = diag (scale_X) %*% newbeta; - ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; - all_linear_terms = X %*% ssX_newbeta; - - [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part - (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - if (isNaN_new_log_l == 0) { - new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); - new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); - } - - while (termination_code == 0) { - accept_new_beta = 1; - - if (i_IRLS > 0) { - if (isNaN_log_l == 0) { - accept_new_beta = 0; - } - - # Decide whether to accept a new iteration point and update the trust region - # See Alg. 4.1 on p. 69 of "Numerical Optimization" 2nd ed. by Nocedal and Wright - - rho = (- new_log_l + log_l) / neg_log_l_change_predicted; - if (rho < 0.25 | isNaN_new_log_l == 1) { - trust_delta = 0.25 * trust_delta; - } - if (rho > 0.75 & isNaN_new_log_l == 0 & reached_trust_boundary == 1) { - trust_delta = 2 * trust_delta; - - ### if (trust_delta > max_trust_delta) { - ### trust_delta = max_trust_delta; - ### } - } - if (rho > 0.1 & isNaN_new_log_l == 0) { - accept_new_beta = 1; - } - } - - if (accept_new_beta == 1) { - beta = newbeta; log_l = new_log_l; deviance_nodisp = new_deviance_nodisp; isNaN_log_l = isNaN_new_log_l; - - [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - # We introduced these variables to avoid roundoff errors: - # g_Y = y_residual / (y_var * link_grad); - # w = 1.0 / (y_var * link_grad * link_grad); - - gXY = - t(X) %*% g_Y; - g = diag (scale_X) %*% gXY + shift_X %*% gXY [num_features, ]; - g_norm = sqrt (sum ((g + lambda * beta) ^ 2)); - } - - [z, neg_log_l_change_predicted, num_CG_iters, reached_trust_boundary] = - get_CG_Steihaug_point (X, scale_X, shift_X, w, g, beta, lambda, trust_delta, max_iteration_CG); - - newbeta = beta + z; - - ssX_newbeta = diag (scale_X) %*% newbeta; - ssX_newbeta [num_features, ] = ssX_newbeta [num_features, ] + t(shift_X) %*% newbeta; - all_linear_terms = X %*% ssX_newbeta; - - [new_log_l, isNaN_new_log_l] = glm_log_likelihood_part - (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - if (isNaN_new_log_l == 0) { - new_deviance_nodisp = 2.0 * (saturated_log_l - new_log_l); - new_log_l = new_log_l - 0.5 * sum (lambda * newbeta ^ 2); - } - - log_l_change = new_log_l - log_l; # R's criterion for termination: |dev - devold|/(|dev| + 0.1) < eps - - if (reached_trust_boundary == 0 & isNaN_new_log_l == 0 & - (2.0 * abs (log_l_change) < eps * (deviance_nodisp + 0.1) | abs (log_l_change) < (abs (log_l) + abs (new_log_l)) * 0.00000000000001) ) { - termination_code = 1; - } - rho = - log_l_change / neg_log_l_change_predicted; - z_norm = sqrt (sum (z * z)); - - i_IRLS = i_IRLS + 1; - - if (i_IRLS == max_iteration_IRLS) { - termination_code = 2; - } - } - - beta = newbeta; - log_l = new_log_l; - deviance_nodisp = new_deviance_nodisp; - - #---------------------------- last part - - if (termination_code != 1) { - print ("One of the runs of GLM did not converged in " + i_IRLS + " steps!"); - } - - ##### COMPUTE AIC ##### - - if (distribution_type == 2 & link_type >= 1 & link_type <= 4) { - AIC = -2 * log_l; - if (sum (X) != 0) { - AIC = AIC + 2 * num_features; - } - } else { - stop ("Currently only the Bernoulli distribution family the following link functions are supported: log, logit, probit, and cloglog!"); - } - - if (fileB != " ") { - fileO = ifdef ($O, " "); - fileS = $S; - fmt = ifdef ($fmt, "text"); - - # Output which features give the best AIC and are being used for linear regression - write (Selected, fileS, format=fmt); - - ssX_beta = diag (scale_X) %*% beta; - ssX_beta [num_features, ] = ssX_beta [num_features, ] + t(shift_X) %*% beta; - if (intercept_status == 2) { - beta_out = cbind (ssX_beta, beta); - } else { - beta_out = ssX_beta; - } - - if (intercept_status == 0 & num_features == 1) { - p = sum (X == 1); - if (p == num_records) { - beta_out = beta_out[1,]; - } - } - - - if (intercept_status == 1 | intercept_status == 2) { - intercept_value = as.scalar (beta_out [num_features, 1]); - beta_noicept = beta_out [1 : (num_features - 1), 1]; - } else { - beta_noicept = beta_out [1 : num_features, 1]; - } - min_beta = min (beta_noicept); - max_beta = max (beta_noicept); - tmp_i_min_beta = rowIndexMin (t(beta_noicept)) - i_min_beta = as.scalar (tmp_i_min_beta [1, 1]); - tmp_i_max_beta = rowIndexMax (t(beta_noicept)) - i_max_beta = as.scalar (tmp_i_max_beta [1, 1]); - - ##### OVER-DISPERSION PART ##### - - all_linear_terms = X %*% ssX_beta; - [g_Y, w] = glm_dist (all_linear_terms, Y, distribution_type, variance_as_power_of_the_mean, link_type, link_as_power_of_the_mean); - - pearson_residual_sq = g_Y ^ 2 / w; - pearson_residual_sq = replace (target = pearson_residual_sq, pattern = NaN, replacement = 0); - # pearson_residual_sq = (y_residual ^ 2) / y_var; - - if (num_records > num_features) { - estimated_dispersion = sum (pearson_residual_sq) / (num_records - num_features); - } - if (dispersion <= 0) { - dispersion = estimated_dispersion; - } - deviance = deviance_nodisp / dispersion; - - ##### END OF THE MAIN PART ##### - - str = "BETA_MIN," + min_beta; - str = append (str, "BETA_MIN_INDEX," + i_min_beta); - str = append (str, "BETA_MAX," + max_beta); - str = append (str, "BETA_MAX_INDEX," + i_max_beta); - str = append (str, "INTERCEPT," + intercept_value); - str = append (str, "DISPERSION," + dispersion); - str = append (str, "DISPERSION_EST," + estimated_dispersion); - str = append (str, "DEVIANCE_UNSCALED," + deviance_nodisp); - str = append (str, "DEVIANCE_SCALED," + deviance); - - if (fileO != " ") { - write (str, fileO); - } - else { - print (str); - } - - # Prepare the output matrix - print ("Writing the output matrix..."); - if (intercept_status == 0 & num_features == 1) { - if (p == num_records) { - beta_out_tmp = matrix (0, rows = num_features_orig + 1, cols = 1); - beta_out_tmp[num_features_orig + 1,] = beta_out; - beta_out = beta_out_tmp; - write (beta_out, fileB, format=fmt); - stop (""); - } else if (sum (X) == 0){ - beta_out = matrix (0, rows = num_features_orig, cols = 1); - write (beta_out, fileB, format=fmt); - stop (""); - } - } - - no_selected = ncol (Selected); - max_selected = max (Selected); - last = max_selected + 1; - - if (intercept_status != 0) { - - Selected_ext = cbind (Selected, as.matrix (last)); - P1 = table (seq (1, ncol (Selected_ext)), t(Selected_ext)); - - if (intercept_status == 2) { - - P1_ssX_beta = P1 * ssX_beta; - P2_ssX_beta = colSums (P1_ssX_beta); - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - - P2_ssX_beta = cbind (P2_ssX_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - - P2_ssX_beta[1, num_features_orig+1] = P2_ssX_beta[1, max_selected + 1]; - P2_ssX_beta[1, max_selected + 1] = 0; - - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1]; - P2_beta[1, max_selected + 1] = 0; - - } - beta_out = cbind (t(P2_ssX_beta), t(P2_beta)); - - } else { - - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - P2_beta[1, num_features_orig+1] = P2_beta[1, max_selected + 1] ; - P2_beta[1, max_selected + 1] = 0; - } - beta_out = t(P2_beta); - - } - } else { - - P1 = table (seq (1, no_selected), t(Selected)); - P1_beta = P1 * beta; - P2_beta = colSums (P1_beta); - - if (max_selected < num_features_orig) { - P2_beta = cbind (P2_beta, matrix (0, rows=1, cols=(num_features_orig - max_selected))); - } - - beta_out = t(P2_beta); - } - - write ( beta_out, fileB, format=fmt ); - - } - - } else { - stop ("Input matrices X and/or Y are out of range!"); - } - } else { - stop ("Response matrix with " + num_response_columns + " columns, distribution family (" + distribution_type + ", " + variance_as_power_of_the_mean - + ") and link family (" + link_type + ", " + link_as_power_of_the_mean + ") are NOT supported together."); - } -} - -glm_initialize = function (Matrix[double] X, Matrix[double] Y, int dist_type, double var_power, int link_type, double link_power, int icept_status, int max_iter_CG) - return (Matrix[double] beta, double saturated_log_l, int isNaN) -{ - saturated_log_l = 0.0; - isNaN = 0; - y_corr = Y [, 1]; - if (dist_type == 2) { - n_corr = rowSums (Y); - is_n_zero = (n_corr == 0); - y_corr = Y [, 1] / (n_corr + is_n_zero) + (0.5 - Y [, 1]) * is_n_zero; - } - linear_terms = y_corr; - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - if (link_power == 0) { - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { isNaN = 1; } - } else { if (link_power == 1.0) { - linear_terms = y_corr; - } else { if (link_power == -1.0) { - linear_terms = 1.0 / y_corr; - } else { if (link_power == 0.5) { - if (sum (y_corr < 0) == 0) { - linear_terms = sqrt (y_corr); - } else { isNaN = 1; } - } else { if (link_power > 0) { - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - }}}}} - } - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1 & link_power == 0) { # Binomial.log - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = log (y_corr + is_zero_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { isNaN = 1; } - } else { if (link_type == 1 & link_power > 0) { # Binomial.power_nonlog pos - if (sum (y_corr < 0) == 0) { - is_zero_y_corr = (y_corr == 0); - linear_terms = (y_corr + is_zero_y_corr) ^ link_power - is_zero_y_corr; - } else { isNaN = 1; } - } else { if (link_type == 1) { # Binomial.power_nonlog neg - if (sum (y_corr <= 0) == 0) { - linear_terms = y_corr ^ link_power; - } else { isNaN = 1; } - } else { - is_zero_y_corr = (y_corr <= 0); - is_one_y_corr = (y_corr >= 1.0); - y_corr = y_corr * (1.0 - is_zero_y_corr) * (1.0 - is_one_y_corr) + 0.5 * (is_zero_y_corr + is_one_y_corr); - if (link_type == 2) { # Binomial.logit - linear_terms = log (y_corr / (1.0 - y_corr)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 3) { # Binomial.probit - y_below_half = y_corr + (1.0 - 2.0 * y_corr) * (y_corr > 0.5); - t = sqrt (- 2.0 * log (y_below_half)); - approx_inv_Gauss_CDF = - t + (2.515517 + t * (0.802853 + t * 0.010328)) / (1.0 + t * (1.432788 + t * (0.189269 + t * 0.001308))); - linear_terms = approx_inv_Gauss_CDF * (1.0 - 2.0 * (y_corr > 0.5)) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 4) { # Binomial.cloglog - linear_terms = log (- log (1.0 - y_corr)) - - log (- log (0.5)) * (is_zero_y_corr + is_one_y_corr) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - } else { if (link_type == 5) { # Binomial.cauchit - linear_terms = tan ((y_corr - 0.5) * pi) - + is_one_y_corr / (1.0 - is_one_y_corr) - is_zero_y_corr / (1.0 - is_zero_y_corr); - }} }}}}} - } - - if (isNaN == 0) { - [saturated_log_l, isNaN] = - glm_log_likelihood_part (linear_terms, Y, dist_type, var_power, link_type, link_power); - } - - if ((dist_type == 1 & link_type == 1 & link_power == 0) | - (dist_type == 2 & link_type >= 2)) - { - desired_eta = 0.0; - } else { if (link_type == 1 & link_power == 0) { - desired_eta = log (0.5); - } else { if (link_type == 1) { - desired_eta = 0.5 ^ link_power; - } else { - desired_eta = 0.5; - }}} - - beta = matrix (0.0, rows = ncol(X), cols = 1); - - if (desired_eta != 0) { - if (icept_status == 1 | icept_status == 2) { - beta [nrow(beta), 1] = desired_eta; - } else { - # We want: avg (X %*% ssX_transform %*% beta) = desired_eta - # Note that "ssX_transform" is trivial here, hence ignored - - beta = straightenX (X, 0.000001, max_iter_CG); - beta = beta * desired_eta; - } } } - - -glm_dist = function (Matrix[double] linear_terms, Matrix[double] Y, - int dist_type, double var_power, int link_type, double link_power) - return (Matrix[double] g_Y, Matrix[double] w) -# ORIGINALLY we returned more meaningful vectors, namely: -# Matrix[double] y_residual : y - y_mean, i.e. y observed - y predicted -# Matrix[double] link_gradient : derivative of the link function -# Matrix[double] var_function : variance without dispersion, i.e. the V(mu) function -# BUT, this caused roundoff errors, so we had to compute "directly useful" vectors -# and skip over the "meaningful intermediaries". Now we output these two variables: -# g_Y = y_residual / (var_function * link_gradient); -# w = 1.0 / (var_function * link_gradient ^ 2); -{ - num_records = nrow (linear_terms); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1 + zeros_r; - g_Y = zeros_r; - w = zeros_r; - - # Some constants - - one_over_sqrt_two_pi = 0.39894228040143267793994605993438; - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - flip_pos = matrix (0, rows = 2, cols = 2); - flip_neg = flip_pos; - flip_pos [1, 2] = 1; - flip_pos [2, 1] = 1; - flip_neg [1, 2] = -1; - flip_neg [2, 1] = 1; - - if (dist_type == 1 & link_type == 1) { # POWER DISTRIBUTION - y_mean = zeros_r; - if (link_power == 0) { - y_mean = exp (linear_terms); - y_mean_pow = y_mean ^ (1 - var_power); - w = y_mean_pow * y_mean; - g_Y = y_mean_pow * (Y - y_mean); - } else { if (link_power == 1.0) { - y_mean = linear_terms; - w = y_mean ^ (- var_power); - g_Y = w * (Y - y_mean); - } else { - y_mean = linear_terms ^ (1.0 / link_power); - c1 = (1 - var_power) / link_power - 1; - c2 = (2 - var_power) / link_power - 2; - g_Y = (linear_terms ^ c1) * (Y - y_mean) / link_power; - w = (linear_terms ^ c2) / (link_power ^ 2); - } }} - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - if (link_type == 1) { # BINOMIAL.POWER LINKS - if (link_power == 0) { # Binomial.log - vec1 = 1 / (exp (- linear_terms) - 1); - g_Y = Y [, 1] - Y [, 2] * vec1; - w = rowSums (Y) * vec1; - } else { # Binomial.nonlog - vec1 = zeros_r; - if (link_power == 0.5) { - vec1 = 1 / (1 - linear_terms ^ 2); - } else { if (sum (linear_terms < 0) == 0) { - vec1 = linear_terms ^ (- 2 + 1 / link_power) / (1 - linear_terms ^ (1 / link_power)); - } else {isNaN = 1;}} - # We want a "zero-protected" version of - # vec2 = Y [, 1] / linear_terms; - is_y_0 = (Y [, 1] == 0); - vec2 = (Y [, 1] + is_y_0) / (linear_terms * (1 - is_y_0) + is_y_0) - is_y_0; - g_Y = (vec2 - Y [, 2] * vec1 * linear_terms) / link_power; - w = rowSums (Y) * vec1 / link_power ^ 2; - } - } else { - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - g_Y = rowSums (Y * (Y_prob %*% flip_neg)); ### = y_residual; - w = rowSums (Y * (Y_prob %*% flip_pos) * Y_prob); ### = y_variance; - } else { if (link_type == 3) { # Binomial.probit - is_lt_pos = (linear_terms >= 0); - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (linear_terms ^ 2) / 2.0); - vec1 = 0.25 * pt_gp * (2 - the_gauss_exp * pt_gp); - vec2 = Y [, 1] - rowSums (Y) * is_lt_pos + the_gauss_exp * pt_gp * rowSums (Y) * (is_lt_pos - 0.5); - w = the_gauss_exp * (one_over_sqrt_two_pi ^ 2) * rowSums (Y) / vec1; - g_Y = one_over_sqrt_two_pi * vec2 / vec1; - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (linear_terms) - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - the_exp_ratio = (1 - is_too_small) * (1 - the_exp_exp) / (the_exp + is_too_small) + is_too_small * (1 - the_exp / 2); - g_Y = (rowSums (Y) * the_exp_exp - Y [, 2]) / the_exp_ratio; - w = the_exp_exp * the_exp * rowSums (Y) / the_exp_ratio; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - y_residual = Y [, 1] * Y_prob [, 2] - Y [, 2] * Y_prob [, 1]; - var_function = rowSums (Y) * Y_prob [, 1] * Y_prob [, 2]; - link_gradient_normalized = (1 + linear_terms ^ 2) * pi; - g_Y = rowSums (Y) * y_residual / (var_function * link_gradient_normalized); - w = (rowSums (Y) ^ 2) / (var_function * link_gradient_normalized ^ 2); - }}}} - } - } - } - - -glm_log_likelihood_part = function (Matrix[double] linear_terms, Matrix[double] Y, - int dist_type, double var_power, int link_type, double link_power) - return (double log_l, int isNaN) -{ - isNaN = 0; - log_l = 0.0; - num_records = nrow (Y); - zeros_r = matrix (0.0, rows = num_records, cols = 1); - - if (dist_type == 1 & link_type == 1) - { # POWER DISTRIBUTION - b_cumulant = zeros_r; - natural_parameters = zeros_r; - is_natural_parameter_log_zero = zeros_r; - if (var_power == 1.0 & link_power == 0) { # Poisson.log - b_cumulant = exp (linear_terms); - is_natural_parameter_log_zero = (linear_terms == -Inf); - natural_parameters = replace (target = linear_terms, pattern = -Inf, replacement = 0); - } else { if (var_power == 1.0 & link_power == 1.0) { # Poisson.id - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = log (linear_terms + is_natural_parameter_log_zero); - } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power == 0.5) { # Poisson.sqrt - if (sum (linear_terms < 0) == 0) { - b_cumulant = linear_terms ^ 2; - is_natural_parameter_log_zero = (linear_terms == 0); - natural_parameters = 2.0 * log (linear_terms + is_natural_parameter_log_zero); - } else {isNaN = 1;} - } else { if (var_power == 1.0 & link_power > 0) { # Poisson.power_nonlog, pos - if (sum (linear_terms < 0) == 0) { - is_natural_parameter_log_zero = (linear_terms == 0); - b_cumulant = (linear_terms + is_natural_parameter_log_zero) ^ (1.0 / link_power) - is_natural_parameter_log_zero; - natural_parameters = log (linear_terms + is_natural_parameter_log_zero) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 1.0) { # Poisson.power_nonlog, neg - if (sum (linear_terms <= 0) == 0) { - b_cumulant = linear_terms ^ (1.0 / link_power); - natural_parameters = log (linear_terms) / link_power; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == -1.0) { # Gamma.inverse - if (sum (linear_terms <= 0) == 0) { - b_cumulant = - log (linear_terms); - natural_parameters = - linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 1.0) { # Gamma.id - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms); - natural_parameters = - 1.0 / linear_terms; - } else {isNaN = 1;} - } else { if (var_power == 2.0 & link_power == 0) { # Gamma.log - b_cumulant = linear_terms; - natural_parameters = - exp (- linear_terms); - } else { if (var_power == 2.0) { # Gamma.power_nonlog - if (sum (linear_terms <= 0) == 0) { - b_cumulant = log (linear_terms) / link_power; - natural_parameters = - linear_terms ^ (- 1.0 / link_power); - } else {isNaN = 1;} - } else { if (link_power == 0) { # PowerDist.log - natural_parameters = exp (linear_terms * (1.0 - var_power)) / (1.0 - var_power); - b_cumulant = exp (linear_terms * (2.0 - var_power)) / (2.0 - var_power); - } else { # PowerDist.power_nonlog - if (-2 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / (linear_terms ^ 2) / (1.0 - var_power); - } else { if (-1 * link_power == 1.0 - var_power) { - natural_parameters = 1.0 / linear_terms / (1.0 - var_power); - } else { if ( link_power == 1.0 - var_power) { - natural_parameters = linear_terms / (1.0 - var_power); - } else { if ( 2 * link_power == 1.0 - var_power) { - natural_parameters = linear_terms ^ 2 / (1.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (1.0 - var_power) / link_power; - natural_parameters = (linear_terms ^ power) / (1.0 - var_power); - } else {isNaN = 1;} - }}}} - if (-2 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / (linear_terms ^ 2) / (2.0 - var_power); - } else { if (-1 * link_power == 2.0 - var_power) { - b_cumulant = 1.0 / linear_terms / (2.0 - var_power); - } else { if ( link_power == 2.0 - var_power) { - b_cumulant = linear_terms / (2.0 - var_power); - } else { if ( 2 * link_power == 2.0 - var_power) { - b_cumulant = linear_terms ^ 2 / (2.0 - var_power); - } else { - if (sum (linear_terms <= 0) == 0) { - power = (2.0 - var_power) / link_power; - b_cumulant = (linear_terms ^ power) / (2.0 - var_power); - } else {isNaN = 1;} - }}}} - }}}}} }}}}} - if (sum (is_natural_parameter_log_zero * abs (Y)) > 0) { - log_l = -Inf; - isNaN = 1; - } - if (isNaN == 0) - { - log_l = sum (Y * natural_parameters - b_cumulant); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - log_l = -Inf; - isNaN = 1; - } } } - - if (dist_type == 2 & link_type >= 1 & link_type <= 5) - { # BINOMIAL/BERNOULLI DISTRIBUTION - - [Y_prob, isNaN] = binomial_probability_two_column (linear_terms, link_type, link_power); - - if (isNaN == 0) { - does_prob_contradict = (Y_prob <= 0); - if (sum (does_prob_contradict * abs (Y)) == 0) { - log_l = sum (Y * log (Y_prob * (1 - does_prob_contradict) + does_prob_contradict)); - if (log_l != log_l | (log_l == log_l + 1.0 & log_l == log_l * 2.0)) { - isNaN = 1; - } - } else { - log_l = -Inf; - isNaN = 1; - } } } - - if (isNaN == 1) { - log_l = - Inf; - } - } - - - -binomial_probability_two_column = - function (Matrix[double] linear_terms, int link_type, double link_power) - return (Matrix[double] Y_prob, int isNaN) -{ - isNaN = 0; - num_records = nrow (linear_terms); - - # Define some auxiliary matrices - - ones_2 = matrix (1.0, rows = 1, cols = 2); - p_one_m_one = ones_2; - p_one_m_one [1, 2] = -1.0; - m_one_p_one = ones_2; - m_one_p_one [1, 1] = -1.0; - zero_one = ones_2; - zero_one [1, 1] = 0.0; - one_zero = ones_2; - one_zero [1, 2] = 0.0; - - zeros_r = matrix (0.0, rows = num_records, cols = 1); - ones_r = 1.0 + zeros_r; - - # Begin the function body - - Y_prob = zeros_r %*% ones_2; - if (link_type == 1) { # Binomial.power - if (link_power == 0) { # Binomial.log - Y_prob = exp (linear_terms) %*% p_one_m_one + ones_r %*% zero_one; - } else { if (link_power == 0.5) { # Binomial.sqrt - Y_prob = (linear_terms ^ 2) %*% p_one_m_one + ones_r %*% zero_one; - } else { # Binomial.power_nonlog - if (sum (linear_terms < 0) == 0) { - Y_prob = (linear_terms ^ (1.0 / link_power)) %*% p_one_m_one + ones_r %*% zero_one; - } else {isNaN = 1;} - }} - } else { # Binomial.non_power - is_LT_pos_infinite = (linear_terms == Inf); - is_LT_neg_infinite = (linear_terms == -Inf); - is_LT_infinite = is_LT_pos_infinite %*% one_zero + is_LT_neg_infinite %*% zero_one; - finite_linear_terms = replace (target = linear_terms, pattern = Inf, replacement = 0); - finite_linear_terms = replace (target = finite_linear_terms, pattern = -Inf, replacement = 0); - if (link_type == 2) { # Binomial.logit - Y_prob = exp (finite_linear_terms) %*% one_zero + ones_r %*% zero_one; - Y_prob = Y_prob / (rowSums (Y_prob) %*% ones_2); - } else { if (link_type == 3) { # Binomial.probit - lt_pos_neg = (finite_linear_terms >= 0) %*% p_one_m_one + ones_r %*% zero_one; - t_gp = 1.0 / (1.0 + abs (finite_linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0) - pt_gp = t_gp * ( 0.254829592 - + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun, - + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299 - + t_gp * (-1.453152027 - + t_gp * 1.061405429)))); - the_gauss_exp = exp (- (finite_linear_terms ^ 2) / 2.0); - Y_prob = lt_pos_neg + ((the_gauss_exp * pt_gp) %*% ones_2) * (0.5 - lt_pos_neg); - } else { if (link_type == 4) { # Binomial.cloglog - the_exp = exp (finite_linear_terms); - the_exp_exp = exp (- the_exp); - is_too_small = ((10000000 + the_exp) == 10000000); - Y_prob [, 1] = (1 - is_too_small) * (1 - the_exp_exp) + is_too_small * the_exp * (1 - the_exp / 2); - Y_prob [, 2] = the_exp_exp; - } else { if (link_type == 5) { # Binomial.cauchit - Y_prob = 0.5 + (atan (finite_linear_terms) %*% p_one_m_one) / pi; - } else { - isNaN = 1; - }}}} - Y_prob = Y_prob * ((1.0 - rowSums (is_LT_infinite)) %*% ones_2) + is_LT_infinite; - } } - - -# THE CG-STEIHAUG PROCEDURE SCRIPT - -# Apply Conjugate Gradient - Steihaug algorithm in order to approximately minimize -# 0.5 z^T (X^T diag(w) X + diag (lambda)) z + (g + lambda * beta)^T z -# under constraint: ||z|| <= trust_delta. -# See Alg. 7.2 on p. 171 of "Numerical Optimization" 2nd ed. by Nocedal and Wright -# IN THE ABOVE, "X" IS UNDERSTOOD TO BE "X %*% (SHIFT/SCALE TRANSFORM)"; this transform -# is given separately because sparse "X" may become dense after applying the transform. -# -get_CG_Steihaug_point = - function (Matrix[double] X, Matrix[double] scale_X, Matrix[double] shift_X, Matrix[double] w, - Matrix[double] g, Matrix[double] beta, Matrix[double] lambda, double trust_delta, int max_iter_CG) - return (Matrix[double] z, double neg_log_l_change, int i_CG, int reached_trust_boundary) -{ - trust_delta_sq = trust_delta ^ 2; - size_CG = nrow (g); - z = matrix (0.0, rows = size_CG, cols = 1); - neg_log_l_change = 0.0; - reached_trust_boundary = 0; - g_reg = g + lambda * beta; - r_CG = g_reg; - p_CG = -r_CG; - rr_CG = sum(r_CG * r_CG); - eps_CG = rr_CG * min (0.25, sqrt (rr_CG)); - converged_CG = 0; - if (rr_CG < eps_CG) { - converged_CG = 1; - } - - max_iteration_CG = max_iter_CG; - if (max_iteration_CG <= 0) { - max_iteration_CG = size_CG; - } - i_CG = 0; - while (converged_CG == 0) - { - i_CG = i_CG + 1; - ssX_p_CG = diag (scale_X) %*% p_CG; - ssX_p_CG [size_CG, ] = ssX_p_CG [size_CG, ] + t(shift_X) %*% p_CG; - temp_CG = t(X) %*% (w * (X %*% ssX_p_CG)); - q_CG = (lambda * p_CG) + diag (scale_X) %*% temp_CG + shift_X %*% temp_CG [size_CG, ]; - pq_CG = sum (p_CG * q_CG); - if (pq_CG <= 0) { - pp_CG = sum (p_CG * p_CG); - if (pp_CG > 0) { - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - } else { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - } - converged_CG = 1; - } - if (converged_CG == 0) { - alpha_CG = rr_CG / pq_CG; - new_z = z + alpha_CG * p_CG; - if (sum(new_z * new_z) >= trust_delta_sq) { - pp_CG = sum (p_CG * p_CG); - [z, neg_log_l_change] = - get_trust_boundary_point (g_reg, z, p_CG, q_CG, r_CG, pp_CG, pq_CG, trust_delta_sq); - reached_trust_boundary = 1; - converged_CG = 1; - } - if (converged_CG == 0) { - z = new_z; - old_rr_CG = rr_CG; - r_CG = r_CG + alpha_CG * q_CG; - rr_CG = sum(r_CG * r_CG); - if (i_CG == max_iteration_CG | rr_CG < eps_CG) { - neg_log_l_change = 0.5 * sum (z * (r_CG + g_reg)); - reached_trust_boundary = 0; - converged_CG = 1; - } - if (converged_CG == 0) { - p_CG = -r_CG + (rr_CG / old_rr_CG) * p_CG; - } } } } } - - -# An auxiliary function used twice inside the CG-STEIHAUG loop: -get_trust_boundary_point = - function (Matrix[double] g, Matrix[double] z, Matrix[double] p, - Matrix[double] q, Matrix[double] r, double pp, double pq, - double trust_delta_sq) - return (Matrix[double] new_z, double f_change) -{ - zz = sum (z * z); pz = sum (p * z); - sq_root_d = sqrt (pz * pz - pp * (zz - trust_delta_sq)); - tau_1 = (- pz + sq_root_d) / pp; - tau_2 = (- pz - sq_root_d) / pp; - zq = sum (z * q); gp = sum (g * p); - f_extra = 0.5 * sum (z * (r + g)); - f_change_1 = f_extra + (0.5 * tau_1 * pq + zq + gp) * tau_1; - f_change_2 = f_extra + (0.5 * tau_2 * pq + zq + gp) * tau_2; - ind1 = as.integer(f_change_1 < f_change_2); - ind2 = as.integer(f_change_1 >= f_change_2); - new_z = z + ((ind1 * tau_1 + ind2 * tau_2) * p); - f_change = ind1 * f_change_1 + ind2 * f_change_2; -} - - -# Computes vector w such that ||X %*% w - 1|| -> MIN given avg(X %*% w) = 1 -# We find z_LS such that ||X %*% z_LS - 1|| -> MIN unconditionally, then scale -# it to compute w = c * z_LS such that sum(X %*% w) = nrow(X). -straightenX = - function (Matrix[double] X, double eps, int max_iter_CG) - return (Matrix[double] w) -{ - w_X = t(colSums(X)); - lambda_LS = 0.000001 * sum(X ^ 2) / ncol(X); - eps_LS = eps * nrow(X); - - # BEGIN LEAST SQUARES - - r_LS = - w_X; - z_LS = matrix (0.0, rows = ncol(X), cols = 1); - p_LS = - r_LS; - norm_r2_LS = sum (r_LS ^ 2); - i_LS = 0; - while (i_LS < max_iter_CG & i_LS < ncol(X) & norm_r2_LS >= eps_LS) - { - q_LS = t(X) %*% X %*% p_LS; - q_LS = q_LS + lambda_LS * p_LS; - alpha_LS = norm_r2_LS / sum (p_LS * q_LS); - z_LS = z_LS + alpha_LS * p_LS; - old_norm_r2_LS = norm_r2_LS; - r_LS = r_LS + alpha_LS * q_LS; - norm_r2_LS = sum (r_LS ^ 2); - p_LS = -r_LS + (norm_r2_LS / old_norm_r2_LS) * p_LS; - i_LS = i_LS + 1; - } - - # END LEAST SQUARES - - w = (nrow(X) / sum (w_X * z_LS)) * z_LS; - } - - \ No newline at end of file diff --git a/scripts/builtin/stepGLM.dml b/scripts/builtin/stepGLM.dml new file mode 100644 index 00000000000..42492c9b276 --- /dev/null +++ b/scripts/builtin/stepGLM.dml @@ -0,0 +1,266 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# +# THIS SCRIPT CHOOSES A GLM REGRESSION MODEL IN A STEPWISE ALGIRITHM USING AIC +# EACH GLM REGRESSION IS SOLVED USING NEWTON/FISHER SCORING WITH TRUST REGIONS +# +# INPUT PARAMETERS: +# --------------------------------------------------------------------------------------------- +# NAME TYPE DEFAULT MEANING +# --------------------------------------------------------------------------------------------- +# X Matrix --- Matrix X of feature vectors +# Y Matrix --- Response Matrix Y with 1 column +# link Int 2 Link function code: 1 = log, 2 = Logit, 3 = Probit, 4 = Cloglog +# yneg Double 0.0 Response value for Bernoulli "No" label, usually 0.0 or -1.0 +# icpt Int 0 Intercept presence, X columns shifting and rescaling: +# 0 = no intercept, no shifting, no rescaling; +# 1 = add intercept, but neither shift nor rescale X; +# 2 = add intercept, shift & rescale X columns to mean = 0, variance = 1 +# tol Double 0.000001 Tolerance (epsilon) +# disp Double 0.0 (Over-)dispersion value, or 0.0 to estimate it from data +# moi Int 200 Maximum number of outer (Newton / Fisher Scoring) iterations +# mii Int 0 Maximum number of inner (Conjugate Gradient) iterations, 0 = no maximum +# thr Double 0.01 Threshold to stop the algorithm: if the decrease in the value of AIC falls below thr +# no further features are being checked and the algorithm stops +# --------------------------------------------------------------------------------------------- +# OUTPUT: Matrix beta, whose size depends on icpt: +# icpt=0: ncol(X) x 1; icpt=1: (ncol(X) + 1) x 1; icpt=2: (ncol(X) + 1) x 2 +# +# AIC Double --- AIC value +# B Matrix --- Estimated regression parameters (betas) +# S Matrix --- The selected features ordered as computed by the algorithm +# --------------------------------------------------------------------------------------------- + +# THE StepGLM SCRIPT CURRENTLY SUPPORTS BERNOULLI DISTRIBUTION FAMILY AND THE FOLLOWING LINK FUNCTIONS ONLY! +# - LOG +# - LOGIT +# - PROBIT +# - CLOGLOG + +source("./scripts/builtin/glm.dml") as glm; + +m_stepGLM = function ( + Matrix[Double] X, + Matrix[Double] Y, + Int link = 2, + Double yneg = 0.0, + Int icpt = 0, + Double tol = 0.000001, + Double disp = 0.0, + Int moi = 200, + Int mii = 0, + Double thr = 0.01 +) return ( + Double AIC, + Matrix[Double] B, + Matrix[Double] S + ) + { + intercept_status = icpt; + bernoulli_No_label = yneg; + distribution_type = 2; + + + if (distribution_type == 2 & ncol(Y) == 1) { + is_Y_negative = (Y == bernoulli_No_label); + Y = cbind (1 - is_Y_negative, is_Y_negative); + count_Y_negative = sum (is_Y_negative); + if (count_Y_negative == 0) { + stop ("StepGLM Input Error: all Y-values encode Bernoulli YES-label, none encode NO-label"); + } + if (count_Y_negative == nrow(Y)) { + stop ("StepGLM Input Error: all Y-values encode Bernoulli NO-label, none encode YES-label"); + } + } + + X_orig = X; + num_records = nrow (X_orig); + num_features = ncol (X_orig); + + # BEGIN STEPWISE GENERALIZED LINEAR MODELS + + continue = TRUE; + columns_fixed = matrix (0, rows = 1, cols = num_features); + columns_fixed_ordered = matrix (0, rows = 1, cols = 1); + + # X_global stores the best model found at each step + X_global = matrix (0, rows = num_records, cols = 1); + + if (intercept_status == 0) { + # Compute AIC of an empty model with no features and no intercept (all Ys are zero) + [AIC_best, ignore_B1, ignore_S1] = internal_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + } else { + # compute AIC of an empty model with only intercept (all Ys are constant) + all_ones = matrix (1, rows = num_records, cols = 1); + [AIC_best, ignore_beta2, ignore_S2] = internal_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + } + + # First pass to examine single features + AICs = matrix (AIC_best, rows = 1, cols = num_features); + parfor (i in 1:num_features) { + [AIC_1, ignore_beta3, ignore_S3] = internal_glm(X=X_orig[,i], Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + AICs[1,i] = AIC_1; + } + + # Determine the best AIC + column_best = 0; + for (k in 1:num_features) { + AIC_cur = as.scalar (AICs[1,k]); + if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) ) { + column_best = k; + AIC_best = as.scalar(AICs[1,k]); + } + } + + if (column_best == 0) { + if (intercept_status == 0) { + # Compute AIC of an empty model with no features and no intercept (all Ys are zero) + [AIC_best, ignore_beta4, ignore_S4] = internal_glm(X=X_global, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + } else { + # compute AIC of an empty model with only intercept (all Ys are constant) + [AIC_best, ignore_beta5, ignore_S5] = internal_glm(X=all_ones, Y=Y, intercept_status=0, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + } + }; + + columns_fixed[1,column_best] = 1; + columns_fixed_ordered[1,1] = column_best; + X_global = X_orig[,column_best]; + + while (continue) { + # Subsequent passes over the features + parfor (i in 1:num_features) { + if (as.scalar(columns_fixed[1,i]) == 0) { + + # Construct the feature matrix + X_loop = cbind (X_global, X_orig[,i]); + + [AIC_2, ignore_beta6, ignore_S6] = internal_glm(X=X_loop, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + AICs[1,i] = AIC_2; + } + } + + # Determine the best AIC + for (k in 1:num_features) { + AIC_cur = as.scalar (AICs[1,k]); + if ( (AIC_cur < AIC_best) & ((AIC_best - AIC_cur) > abs (thr * AIC_best)) & (as.scalar(columns_fixed[1,k]) == 0) ) { + column_best = k; + AIC_best = as.scalar(AICs[1,k]); + } + } + + # cbind best found features (i.e., columns) to X_global + if (as.scalar(columns_fixed[1,column_best]) == 0) { # new best feature found + columns_fixed[1,column_best] = 1; + columns_fixed_ordered = cbind (columns_fixed_ordered, as.matrix(column_best)); + if (ncol(columns_fixed_ordered) == num_features) { # all features examined + X_global = cbind (X_global, X_orig[,column_best]); + continue = FALSE; + } else { + X_global = cbind (X_global, X_orig[,column_best]); + } + } else { + continue = FALSE; + } + } + + # run GLM with selected set of features + [AIC, B, S] = internal_glm(X=X_global, Y=Y, intercept_status=intercept_status, num_features_orig=num_features, Selected=columns_fixed_ordered, link=link, disp=disp, tol=tol, moi=moi, mii=mii); + } + +compute_AIC = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] B, Int link, Int icpt) + return (Double aic) +{ + # Isolate unscaled parameters; accounts for m_glm returning 2 columns when icpt=2 + beta = B[, 1]; + + if (icpt > 0) { + ones = matrix(1, rows=nrow(X), cols=1); + X_design = cbind(X, ones); + } else { + X_design = X; + } + + eta = X_design %*% beta; + + # Map inverse link functions + if (link == 1) { + # log, see https://en.wikipedia.org/wiki/Generalized_linear_model#Link_function + mu = exp(eta); + } else if (link == 2) { + # logit, see https://en.wikipedia.org/wiki/Generalized_linear_model#Link_function + mu = 1 / (1 + exp(-eta)); + } else if (link == 3) { + # probit approximation, page 1487 in "Qualitative Response Models: A Survey (1981)" by Takeshi Amemiya + mu = 1 / (1 + exp(-1.6 * eta)); + } else if (link == 4) { + # cloglog, see https://search.r-project.org/CRAN/refmans/VGAM/html/clogloglink.html + mu = 1 - exp(-exp(eta)); + } else { + stop("Unsupported link function code: " + link); + mu = eta; + } + + # Constrain to prevent numerical discontinuity in log(0) + mu = max(mu, 1e-15); + mu = min(mu, 1 - 1e-15); + + Y_yes = Y[, 1]; + Y_neg = Y[, 2]; + + LL = sum(Y_yes * log(mu) + Y_neg * log(1 - mu)); + k = nrow(beta); + + aic = 2 * k - 2 * LL; +} + +internal_glm = function ( + Matrix[Double] X, + Matrix[Double] Y, + Int intercept_status, + Double num_features_orig, + Matrix[Double] Selected, + Int link, + Double disp, + Double tol, + Int moi, + Int mii +) return ( + Double AIC, + Matrix[Double] B, + Matrix[Double] S +) { + # bernoulli family parameters, see table in ./scripts/builtin/m_glm.dml for details. + new_link = link; + new_lpow = 1.0; + + if (link == 1) { + new_link = 1; + new_lpow = 0.0; + } + B = glm::m_glm(X=X, Y=Y, dfam=2, vpow=0.0, link=new_link, lpow=new_lpow, yneg=0.0, + icpt=intercept_status, disp=disp, reg=0.0, tol=tol, + moi=moi, mii=mii, verbose=FALSE); + + AIC = compute_AIC(X, Y, B, link, intercept_status); + + S = Selected; +} diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index a7a175bb7b6..0bb1e9b462d 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -508,9 +508,9 @@ private static void execute(String dmlScriptStr, String fnameOptConfig, Map inHops1 = new ArrayList<>(); - inHops1.add(expr); - inHops1.add(expr2); - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), inHops1); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL: - case MAX_POOL: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForPoolingForwardIM2COL(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case AVG_POOL_BACKWARD: - case MAX_POOL_BACKWARD: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOpPoolingCOL2IM(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - case CONV2D: - case CONV2D_BACKWARD_FILTER: - case CONV2D_BACKWARD_DATA: { - currBuiltinOp = new DnnOp(target.getName(), DataType.MATRIX, target.getValueType(), - OpOpDnn.valueOf(source.getOpCode().name()), getALHopsForConvOp(expr, source, 1, hops)); - setBlockSizeAndRefreshSizeInfo(expr, currBuiltinOp); - break; - } - - case ROW_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Row, expr); - break; - - case COL_COUNT_DISTINCT: - currBuiltinOp = new AggUnaryOp(target.getName(), - DataType.MATRIX, target.getValueType(), AggOp.COUNT_DISTINCT, Direction.Col, expr); - break; - - case GET_CATEGORICAL_MASK: - currBuiltinOp = new BinaryOp(target.getName(), DataType.MATRIX, ValueType.FP64, OpOp2.GET_CATEGORICAL_MASK, expr, expr2); - break; - default: - throw new ParseException("Unsupported builtin function type: "+source.getOpCode()); - } - - boolean isConvolution = source.getOpCode() == Builtins.CONV2D || source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || - source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || - source.getOpCode() == Builtins.MAX_POOL || source.getOpCode() == Builtins.MAX_POOL_BACKWARD || - source.getOpCode() == Builtins.AVG_POOL || source.getOpCode() == Builtins.AVG_POOL_BACKWARD; - if( !isConvolution) { + boolean isConvolution = source.getOpCode() == Builtins.CONV2D || + source.getOpCode() == Builtins.CONV2D_BACKWARD_DATA || + source.getOpCode() == Builtins.CONV2D_BACKWARD_FILTER || source.getOpCode() == Builtins.MAX_POOL || + source.getOpCode() == Builtins.MAX_POOL_BACKWARD || source.getOpCode() == Builtins.AVG_POOL || + source.getOpCode() == Builtins.AVG_POOL_BACKWARD; + if(!isConvolution) { // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 68a3d1b7ffe..3d3a90b4f6f 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -1176,52 +1176,72 @@ else if( getVarParam(READNNZPARAM) != null ) { boolean isHDF5 = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); - boolean isCOG = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + // handle all csv default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(DELIM_FILL_VALUE, ValueType.FP64, conditional); + handleCSVDefaultParam(DELIM_HAS_HEADER_ROW, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_FILL, ValueType.BOOLEAN, conditional); + handleCSVDefaultParam(DELIM_NA_STRINGS, ValueType.STRING, conditional); + } - // Delta tables are self-describing (schema + dimensions discovered from the - // transaction log at read time), so dimensions are optional like CSV. - boolean isDelta = (formatTypeString != null && formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + boolean isLIBSVM = false; + isLIBSVM = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.LIBSVM.toString())); + if(isLIBSVM) { + // Handle libsvm file format + shouldReadMTD = true; + + // only allow IO_FILENAME, READROWPARAM, READCOLPARAM + // as valid parameters + if(!inferredFormatType) { + for(String key : _varParams.keySet()) { + if(!(key.equals(IO_FILENAME) || key.equals(FORMAT_TYPE) || key.equals(READROWPARAM) || + key.equals(READCOLPARAM) || key.equals(READNNZPARAM) || key.equals(DATATYPEPARAM) || + key.equals(VALUETYPEPARAM) || key.equals(DELIM_DELIMITER) || + key.equals(LIBSVM_INDEX_DELIM))) { + String msg = "Only parameters allowed are: " + IO_FILENAME + "," + READROWPARAM + "," + + READCOLPARAM + DELIM_DELIMITER + "," + LIBSVM_INDEX_DELIM; + + raiseValidateError( + "Invalid parameter " + key + " in read statement: " + toString() + ". " + msg, + conditional, LanguageErrorCodes.INVALID_PARAMETERS); + } + } + } + // handle all default parameters + handleCSVDefaultParam(DELIM_DELIMITER, ValueType.STRING, conditional); + handleCSVDefaultParam(LIBSVM_INDEX_DELIM, ValueType.STRING, conditional); + } - dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); - - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) - || dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { - - boolean isMatrix = false; - if ( dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) + boolean isHDF5 = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.HDF5.toString())); + + boolean isCOG = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.COG.toString())); + + // Delta tables are self-describing (schema + dimensions discovered from the + // transaction log at read time), so dimensions are optional like CSV. + boolean isDelta = (formatTypeString != null && + formatTypeString.equalsIgnoreCase(FileFormat.DELTA.toString())); + + dataTypeString = (getVarParam(DATATYPEPARAM) == null) ? null : getVarParam(DATATYPEPARAM).toString(); + + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE) || + dataTypeString.equalsIgnoreCase(Statement.FRAME_DATA_TYPE)) { + + boolean isMatrix = false; + if(dataTypeString == null || dataTypeString.equalsIgnoreCase(Statement.MATRIX_DATA_TYPE)) isMatrix = true; - - // set data type - getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); - - // set number non-zeros - Expression ennz = getVarParam("nnz"); - long nnz = -1; - if( ennz != null ) { - nnz = Long.valueOf(ennz.toString()); - getOutput().setNnz(nnz); - } - // Following dimension checks must be done when data type = MATRIX_DATA_TYPE - // initialize size of target data identifier to UNKNOWN - getOutput().setDimensions(-1, -1); - - if (!isCSV && !isLIBSVM && !isHDF5 && !isCOG && !isDelta && ConfigurationManager.getCompilerConfig() - .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv/libsvm/delta format / jmlc api - && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) { - raiseValidateError("Missing or incomplete dimension information in read statement: " - + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - if (getVarParam(READROWPARAM) instanceof ConstIdentifier - && getVarParam(READCOLPARAM) instanceof ConstIdentifier) - { - // these are strings that are long values - Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString()); - Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString()); - if ( !isCSV && !isDelta && (dim1 < 0 || dim2 < 0) && ConfigurationManager - .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) { - raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS); + // set data type + getOutput().setDataType(isMatrix ? DataType.MATRIX : DataType.FRAME); + + // set number non-zeros + Expression ennz = getVarParam("nnz"); + long nnz = -1; + if(ennz != null) { + nnz = Long.valueOf(ennz.toString()); + getOutput().setNnz(nnz); } // set dim1 and dim2 values @@ -1252,104 +1272,10 @@ && getVarParam(READCOLPARAM) instanceof ConstIdentifier) catch(Exception ex) { raiseValidateError("Invalid format '" + fmt+ "' in statement: " + toString(), conditional); } - - if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) { - Integer rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? - null : Integer.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); - getOutput().setBlocksize(rowBlockCount != null ? rowBlockCount : -1); - } - - // block dimensions must be -1x-1 when format="text" - // NOTE MB: disabled validate of default blocksize for inputs w/ format="binary" - // because we automatically introduce reblocks if blocksizes don't match - if ( (getOutput().getFileFormat().isTextFormat() || !isMatrix) && getOutput().getBlocksize() != -1 ){ - raiseValidateError("Invalid block dimensions (" + getOutput().getBlocksize() + ") when format=" + getVarParam(FORMAT_TYPE) + " in \"" + this.toString() + "\".", conditional); - } - - } - else if ( dataTypeString.equalsIgnoreCase(Statement.SCALAR_DATA_TYPE)) { - getOutput().setDataType(DataType.SCALAR); - getOutput().setNnz(-1L); - } - else if ( dataTypeString.equalsIgnoreCase(DataType.LIST.name())) { - getOutput().setDataType(DataType.LIST); - } - else{ - raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " - + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE+", " + Statement.FRAME_DATA_TYPE - +", " + DataType.LIST.name().toLowerCase(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - } - - // handle value type parameter - if (getVarParam(VALUETYPEPARAM) != null && !(getVarParam(VALUETYPEPARAM) instanceof StringIdentifier)){ - raiseValidateError("for read method, parameter " + VALUETYPEPARAM + " can only be a string. " + - "Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - // Identify the value type (used only for read method) - String valueTypeString = getVarParam(VALUETYPEPARAM) == null ? null : getVarParam(VALUETYPEPARAM).toString(); - if (valueTypeString != null) { - if (valueTypeString.equalsIgnoreCase(Statement.DOUBLE_VALUE_TYPE)) - getOutput().setValueType(ValueType.FP64); - else if (valueTypeString.equalsIgnoreCase(Statement.STRING_VALUE_TYPE)) - getOutput().setValueType(ValueType.STRING); - else if (valueTypeString.equalsIgnoreCase(Statement.INT_VALUE_TYPE)) - getOutput().setValueType(ValueType.INT64); - else if (valueTypeString.equalsIgnoreCase(Statement.BOOLEAN_VALUE_TYPE)) - getOutput().setValueType(ValueType.BOOLEAN); - else if (valueTypeString.equalsIgnoreCase(ValueType.UNKNOWN.name())) - getOutput().setValueType(ValueType.UNKNOWN); - else { - raiseValidateError("Unknown Value Type " + valueTypeString - + ". Valid values are: " + Statement.DOUBLE_VALUE_TYPE +", " + Statement.INT_VALUE_TYPE + ", " + Statement.BOOLEAN_VALUE_TYPE + ", " + Statement.STRING_VALUE_TYPE, conditional); - } - } else { - getOutput().setValueType(ValueType.FP64); - } - - break; - - case WRITE: - - // for CSV format, if no delimiter specified THEN set default "," - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.CSV) ){ - if (getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if (getVarParam(DELIM_HAS_HEADER_ROW) == null) { - addVarParam(DELIM_HAS_HEADER_ROW, new BooleanIdentifier(DEFAULT_DELIM_HAS_HEADER_ROW, this)); - } - if (getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - // for LIBSVM format, add the default separators if not specified - if (getVarParam(FORMAT_TYPE) == null || checkFormatType(FileFormat.LIBSVM)) { - if(getVarParam(DELIM_DELIMITER) == null) { - addVarParam(DELIM_DELIMITER, new StringIdentifier(DEFAULT_DELIM_DELIMITER, this)); - } - if(getVarParam(LIBSVM_INDEX_DELIM) == null) { - addVarParam(LIBSVM_INDEX_DELIM, new StringIdentifier(DEFAULT_LIBSVM_INDEX_DELIM, this)); - } - if(getVarParam(DELIM_SPARSE) == null) { - addVarParam(DELIM_SPARSE, new BooleanIdentifier(DEFAULT_DELIM_SPARSE, this)); - } - } - - //validate read filename - if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()) - || checkFormatType(FileFormat.DELTA)) //delta: columnar, no block layout - getOutput().setBlocksize(-1); - else if (checkFormatType(FileFormat.BINARY, FileFormat.COMPRESSED, FileFormat.UNKNOWN)) { - if( getVarParam(ROWBLOCKCOUNTPARAM)!=null ) - getOutput().setBlocksize(Integer.parseInt(getVarParam(ROWBLOCKCOUNTPARAM).toString())); - else - getOutput().setBlocksize(ConfigurationManager.getBlocksize()); - } - else if( getVarParam(FORMAT_TYPE) instanceof StringIdentifier ) //literal format - raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) - + " in statement: " + toString(), conditional); - break; + else if(getVarParam(FORMAT_TYPE) instanceof StringIdentifier) // literal format + raiseValidateError("Invalid format " + getVarParam(FORMAT_TYPE) + " in statement: " + toString(), + conditional); + break; case RAND: diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index d0ba5363939..042e0dc0328 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -484,7 +484,7 @@ public static CompressedMatrixBlock read(DataInput in) throws IOException { long nonZeros = in.readLong(); boolean overlappingColGroups = in.readBoolean(); List groups = ColGroupIO.readGroups(in, rlen); - CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); + CompressedMatrixBlock ret = new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups); LOG.debug("Compressed read serialization time: " + t.stop()); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 354325e293b..66d4e78cb0f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -402,7 +402,8 @@ public final AColGroup rightMultByMatrix(MatrixBlock right) { * @param cru The right hand side column upper * @param nRows The number of rows in this column group */ - public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, int cru) { + public void rightDecompressingMult(MatrixBlock right, MatrixBlock ret, int rl, int ru, int nRows, int crl, + int cru) { throw new NotImplementedException( "not supporting right Decompressing Multiply on class: " + this.getClass().getSimpleName()); } @@ -977,9 +978,9 @@ public AColGroup[] splitReshapePushDown(final int multiplier, final int nRow, fi /** * Sort the values of the column group according to double comparison operations and return as another compressed * group. - * + * * This sorting assumes that the column group is sorted independently of everything else. - * + * * @return The sorted group */ public abstract AColGroup sort(); @@ -996,9 +997,9 @@ public String toString() { /** * Return a new column group containing only the selected rows in the given boolean vector. - * + * * Whenever possible only modify the index structure, not the dictionary of the column groups. - * + * * @param selectV The selection vector * @param rOut The number of rows in the output * @return The new column group @@ -1007,9 +1008,9 @@ public String toString() { /** * Return a new column group containing only the selected columns in the given boolean vector. - * + * * Whenever possible only modify the column index, and reduce the dictionaries of the column groups. - * + * * @param selectV The selection vector * @return The new column group, or {@code null} if no column of this group is selected */ @@ -1045,7 +1046,7 @@ public AColGroup removeEmptyCols(boolean[] selectV) { /** * Using the selection of columns, slice out those and return in a new column group with the given column indexes. * Ideally this method should only modify the dictionaries. - * + * * @param newColumnIDs the new column indexes * @param selectedColumns The selected columns of this column group (guaranteed < current number of columns) * @return A new Column group diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java index d825b91f089..d610c1b586c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupValue.java @@ -210,7 +210,6 @@ public void clear() { counts = null; } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java index 30de5e120c5..794d90c0d11 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java @@ -212,8 +212,8 @@ public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, in // TODO make sparse decompression where the iterator is known in argument decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock()); else - decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(), - it); + decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, + mb.getDenseBlockValues(), it); } else decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it); @@ -240,7 +240,7 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, } public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru, - int offR, int offC, double[] values, AIterator it); + int offR, int offC, double[] values, AIterator it); public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru, int offR, int offC, double[] values, AIterator it); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java index b316e48474a..d643cae440c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java @@ -674,8 +674,8 @@ private void defaultRightDecompressingMult(MatrixBlock right, MatrixBlock ret, i } } - final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, int vLen, - DoubleVector vVec) { + final void vectMM(double aa, double[] b, double[] c, int endT, int jd, int crl, int cru, int offOut, int k, + int vLen, DoubleVector vVec) { vVec = vVec.broadcast(aa); final int offj = k * jd; final int end = endT + offj; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index 64114a054ab..d5ad55772c7 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -478,14 +478,13 @@ public AColGroup combineWithSameIndex(int nRow, int nCol, List right) return new ColGroupEmpty(combinedIndex); } - @Override - public AColGroup removeEmptyRows(boolean[] selectV, int rOut){ + @Override + public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { return this; } - @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { return new ColGroupEmpty(newColumnIDs); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java index fa8aa104ffb..e0bea3c3696 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java @@ -747,7 +747,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java index a251d828b5f..b4f0c144a73 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java @@ -738,7 +738,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java index 347cea9c0da..43df7fa3b94 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java @@ -1195,9 +1195,9 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { throw new NotImplementedException("Unimplemented method 'removeEmptyRows'"); } - + @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java index 815ecacf378..4566106a3e2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java @@ -634,8 +634,8 @@ protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList s for(int i = 0; i < selectedColumns.size(); i++) { ref[i] = _reference[selectedColumns.get(i)]; } - return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), _indexes, _data, null, - ref); + return ColGroupSDCFOR.create(newColumnIDs, _numRows, _dict.sliceColumns(selectedColumns, getNumCols()), + _indexes, _data, null, ref); } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 611add6480f..9797087f8c3 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -85,7 +85,7 @@ public class ColGroupUncompressed extends AColGroup { /** * Do not use this constructor of column group uncompressed, instead use the create constructor. - * + * * @param mb The contained data. * @param colIndexes Column indexes for this Columngroup */ @@ -96,9 +96,10 @@ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes) { /** * Do not use this constructor of column group quantization-fused uncompressed, instead use the create constructor. - * + * * @param mb The contained data. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @param colIndexes Column indexes for this Columngroup */ protected ColGroupUncompressed(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -138,7 +139,8 @@ public static AColGroup create(MatrixBlock mb, IColIndex colIndexes) { * * @param mb The MB / data to contain in the uncompressed column * @param colIndexes The column indexes for the group - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return An Uncompressed Column group */ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, double[] scaleFactors) { @@ -157,7 +159,8 @@ public static AColGroup createQuantized(MatrixBlock mb, IColIndex colIndexes, do * @param rawBlock The uncompressed block; uncompressed data must be present at the time that the constructor is * called * @param transposed Says if the input matrix raw block have been transposed. - * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire matrix + * @param scaleFactors For quantization-fused compression, scale factors per row, or a single value for entire + * matrix * @return AColGroup. */ public static AColGroup createQuantized(IColIndex colIndexes, MatrixBlock rawBlock, boolean transposed, diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java index 51e26a3f9d2..de8a740ceb2 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressedArray.java @@ -290,7 +290,7 @@ public AColGroup removeEmptyRows(boolean[] selectV, int rOut) { } @Override - protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns){ + protected AColGroup removeEmptyColsSubset(IColIndex newColumnIDs, IntArrayList selectedColumns) { throw new NotImplementedException("Unimplemented method 'removeEmptyColumns'"); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java index a7e715b59b8..6e66ef6ef9b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/AIdentityDictionary.java @@ -76,8 +76,8 @@ public double[] productAllRowsToDoubleWithDefault(double[] defaultTuple) { return ret; } - @Override - public int[] sort(){ + @Override + public int[] sort() { throw new NotImplementedException(); } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java index 9a0412145f0..7ebba2f1a76 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DeltaDictionary.java @@ -138,8 +138,8 @@ public IDictionary clone() { throw new NotImplementedException(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { throw new NotImplementedException(); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java index c8ddfc4883a..b5e1a99355b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IDictionary.java @@ -1055,7 +1055,7 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Slice out the selected columns given of this encoded group. - * + * * @param selectedColumns The columns to slice out and return as a new matrix. * @param nCol The number of columns in this dictionary. * @return The returned matrix @@ -1064,9 +1064,9 @@ public IDictionary rightMMPreAggSparse(int numVals, SparseBlock b, IColIndex thi /** * Sort the values of this dictionary via an index of how the values mapped previously. - * + * * In practice this design means we can reuse the previous dictionary for the resulting column group - * + * * @return The sorted index. */ public int[] sort(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java index c2540de959a..4337da7307f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java @@ -541,8 +541,8 @@ public String getString(int colIndexes) { return "IdentityMatrix of size: " + nRowCol + " with empty: " + withEmpty; } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java index c7f642edfd0..47628b43d2a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java @@ -311,8 +311,8 @@ public String getString(int colIndexes) { return toString(); } - @Override - public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol){ + @Override + public IDictionary sliceColumns(IntArrayList selectedColumns, int nCol) { return getMBDict().sliceColumns(selectedColumns, nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java index 83a74972db7..6d516713689 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java @@ -1064,7 +1064,7 @@ public AMapToData removeEmpty(final boolean[] selectV, final int rOut) { /** * Use the offsets of the select vector to choose which values to keep. - * + * * @param select The row indexes to keep * @return A New MapToData */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java index f65876b7f37..bf8ee7f9ee1 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java @@ -56,11 +56,11 @@ public abstract class AOffset implements Serializable { protected static final Log LOG = LogFactory.getLog(AOffset.class.getName()); /** - * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's - * static initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a - * superclass/subclass class-initialization cycle that deadlocks when several threads first touch the offset - * classes concurrently (e.g. parallel tests). Deferring it to first use guarantees AOffset is already - * initialized by the time OffsetEmpty is loaded, so no cycle exists. + * Lazy holder for the cached empty slice. The empty slice is built on first use rather than in AOffset's static + * initializer: instantiating the OffsetEmpty subclass from AOffset's {@code } forms a superclass/subclass + * class-initialization cycle that deadlocks when several threads first touch the offset classes concurrently (e.g. + * parallel tests). Deferring it to first use guarantees AOffset is already initialized by the time OffsetEmpty is + * loaded, so no cycle exists. */ private static final class EmptySliceHolder { static final OffsetSliceInfo EMPTY_SLICE = new OffsetSliceInfo(-1, -1, new OffsetEmpty()); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java index 866168ded2f..37ff41cf817 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/OffsetEmpty.java @@ -76,6 +76,7 @@ public int getOffsetToLast() { public long getInMemorySize() { return estimateInMemorySize(); } + @Override public boolean equals(AOffset b) { return b instanceof OffsetEmpty; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java index d981ab87838..7953322350e 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java @@ -139,7 +139,8 @@ private static boolean isDoubleCompressedOpApplicable(CompressedMatrixBlock m1, m1.getColGroups().get(0) instanceof ColGroupDDC && !((CompressedMatrixBlock) that).isOverlapping() && ((CompressedMatrixBlock) that).getColGroups().get(0) instanceof ColGroupDDC && ((IMapToDataGroup) m1.getColGroups().get(0)) - .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)).getMapToData(); + .getMapToData() == ((IMapToDataGroup) ((CompressedMatrixBlock) that).getColGroups().get(0)) + .getMapToData(); } private static CompressedMatrixBlock doubleCompressedBinaryOp(BinaryOperator op, CompressedMatrixBlock m1, @@ -1062,7 +1063,8 @@ public Long call() { return _ret.recomputeNonZeros(_rl, _ru - 1); } - private final void processBlock(final int rl, final int ru, final List groups, final AIterator[] its) { + private final void processBlock(final int rl, final int ru, final List groups, + final AIterator[] its) { decompressToTmpBlock(rl, ru, tmp.getSparseBlock(), groups, its); // decompressing multiple column groups can leave the temp rows with unsorted column indices, so sort // before reading them in stored order into the (column-sorted) output sparse block. diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java index cc7953f8c5d..a91b75ae73c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java @@ -96,7 +96,7 @@ public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, Matrix if(x.isEmpty()) return returnEmpty(x, out); - if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns()> 30){ + if(ctype == ChainType.XtXv && x.getColGroups().size() < 5 && x.getNumColumns() > 30) { MatrixBlock tmp = CLALibTSMM.leftMultByTransposeSelf(x, k); return tmp.aggregateBinaryOperations(tmp, v, out, InstructionUtils.getMatMultOperator(k)); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java index 3755e4040e7..802eddffcb8 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRemoveEmpty.java @@ -36,7 +36,7 @@ public class CLALibRemoveEmpty { /** * CP rmempty operation (single input, single output matrix) - * + * * @param in The input matrix * @param ret The output matrix * @param rows If we are removing based on rows, or columns. @@ -66,13 +66,13 @@ private static MatrixBlock rmEmptyCols(CompressedMatrixBlock in, MatrixBlock ret int cOut = (int) select.getNonZeros(); if(cOut == -1) cOut = (int) select.recomputeNonZeros(); - if(cOut == 0){ + if(cOut == 0) { ret.reset(in.getNumRows(), !emptyReturn ? 0 : 1); return ret; } - final boolean[] selectV = DataConverter - .convertToBooleanVector(CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); + final boolean[] selectV = DataConverter.convertToBooleanVector( + CompressedMatrixBlock.getUncompressed(select, "decompressing selection in rmempty")); final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); @@ -102,18 +102,17 @@ private static MatrixBlock rmEmptyRows(CompressedMatrixBlock in, MatrixBlock ret int rOut = (int) select.getNonZeros(); if(rOut == -1) rOut = (int) select.recomputeNonZeros(); - if(rOut == 0){ + if(rOut == 0) { ret.reset(!emptyReturn ? 0 : 1, in.getNumColumns()); return ret; } - // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to number + // TODO: add optimization to avoid linear scan and make selectV indexes, if selection is small relative to + // number // of rows // TODO: add decompress to boolean vector. final boolean[] selectV = DataConverter.convertToBooleanVector(select); - - final List inG = in.getColGroups(); final List retG = new ArrayList<>(inG.size()); try { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java index b94f11ae723..5ae7bd5103b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSort.java @@ -40,10 +40,10 @@ private CLALibSort() { /** * Sort (order) a compressed matrix in place of the {@code order} built-in, while keeping the result compressed. * - * The compressed fast-path only supports the case the user can benefit from: a single column held in a single column - * group, sorted ascending and returning the sorted values (not the index permutation). For everything else (multiple - * columns, multiple column groups, descending order, index return, or a column-group encoding without a sort - * implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. + * The compressed fast-path only supports the case the user can benefit from: a single column held in a single + * column group, sorted ascending and returning the sorted values (not the index permutation). For everything else + * (multiple columns, multiple column groups, descending order, index return, or a column-group encoding without a + * sort implementation) this returns {@code null} so the caller can fall back to a decompressed reorg. * * @param mb the compressed matrix to sort * @param fn the sort specification carried by the reorg operator diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java index 87d14dbf87e..9ccaa474f39 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/FrameObject.java @@ -208,8 +208,8 @@ protected FrameBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcept if(data == null) throw new IOException("Unable to load frame from file: " + fname); - //Delta and CSV discover dimensions (and Delta also schema) at read time, so - //refresh the cached metadata to reflect the materialized frame block. + // Delta and CSV discover dimensions (and Delta also schema) at read time, so + // refresh the cached metadata to reflect the materialized frame block. if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(data.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(data.getDataCharacteristics()); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index 28fa70f7741..4331da2b426 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -454,7 +454,7 @@ protected MatrixBlock readBlobFromHDFS(String fname, long[] dims) throws IOExcep rlen, clen, blen, mc.getNonZeros(), getFileFormatProperties()); if(iimd.getFileFormat() == FileFormat.CSV || iimd.getFileFormat() == FileFormat.DELTA) { - //dimensions/nnz are discovered at read time for these self-describing formats + // dimensions/nnz are discovered at read time for these self-describing formats _metaData = _metaData instanceof MetaDataFormat ? new MetaDataFormat(newData.getDataCharacteristics(), iimd.getFileFormat()) : new MetaData(newData.getDataCharacteristics()); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java index b52f3777e1f..fbae4925c66 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java @@ -122,9 +122,9 @@ public class SparkExecutionContext extends ExecutionContext //singleton spark context (as there can be only one spark context per JVM) private static JavaSparkContext _spctx = null; - //registered users of the singleton context (guarded by the - //SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ - //exitSparkExecution(), and close() only stops the context once it hits zero + // registered users of the singleton context (guarded by the + // SparkExecutionContext.class monitor); maintained by enterSparkExecution()/ + // exitSparkExecution(), and close() only stops the context once it hits zero private static int _activeExecutions = 0; //registry of parallelized RDDs to enforce that at any time, we spent at most @@ -175,8 +175,8 @@ public synchronized static JavaSparkContext getSparkContextStatic() { initSparkContext(); if(_spctx.sc().isStopped()){ _spctx = null; - //the previous context was stopped; reset the active-execution count so a - //stale registration cannot skip a future legitimate stop of the new one + // the previous context was stopped; reset the active-execution count so a + // stale registration cannot skip a future legitimate stop of the new one _activeExecutions = 0; initSparkContext(); } @@ -196,16 +196,15 @@ public synchronized static boolean isSparkContextCreated() { public static void resetSparkContextStatic() { synchronized(SparkExecutionContext.class) { _spctx = null; - //force-discarding the shared context: drop the active-execution count so - //a stale registration cannot skip a future legitimate stop + // force-discarding the shared context: drop the active-execution count so + // a stale registration cannot skip a future legitimate stop _activeExecutions = 0; } } /** - * Registers an active user of the shared spark context. Must be balanced by a - * later {@link #exitSparkExecution()} so a concurrent execution cannot stop the - * context while this one still has in-flight jobs. + * Registers an active user of the shared spark context. Must be balanced by a later {@link #exitSparkExecution()} + * so a concurrent execution cannot stop the context while this one still has in-flight jobs. */ public static void enterSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -214,9 +213,8 @@ public static void enterSparkExecution() { } /** - * Releases an active user previously registered via {@link #enterSparkExecution()}. - * Only adjusts the count; the actual teardown is left to {@link #close()}, which - * stops the context once no registered execution remains. + * Releases an active user previously registered via {@link #enterSparkExecution()}. Only adjusts the count; the + * actual teardown is left to {@link #close()}, which stops the context once no registered execution remains. */ public static void exitSparkExecution() { synchronized(SparkExecutionContext.class) { @@ -227,13 +225,13 @@ public static void exitSparkExecution() { public void close() { synchronized(SparkExecutionContext.class) { - //keep the shared context alive while a registered execution still uses - //it; close() never changes the count, so an unpaired close() (a caller - //that never entered) cannot stop a context another execution is using + // keep the shared context alive while a registered execution still uses + // it; close() never changes the count, so an unpaired close() (a caller + // that never entered) cannot stop a context another execution is using if(_activeExecutions > 0) { if(LOG.isDebugEnabled()) - LOG.debug("Keeping shared spark context alive; " + _activeExecutions - + " execution(s) still active"); + LOG.debug( + "Keeping shared spark context alive; " + _activeExecutions + " execution(s) still active"); return; } if(_spctx != null) { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index 682cc8e3fff..c502817e026 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -95,8 +95,7 @@ private void run() { int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); // Daemon event loops so a leaked in-JVM (test) worker cannot block JVM exit. - NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, - new DefaultThreadFactory("fed-worker-boss", true)); + NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("fed-worker-boss", true)); ThreadPoolExecutor workerTPE = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, new SynchronousQueue(true), new DefaultThreadFactory("fed-worker-pool", true)); NioEventLoopGroup workerGroup = new NioEventLoopGroup(EVENT_LOOP_THREADS, workerTPE); diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index 80a5d699dfa..ebf05972b87 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -125,13 +125,15 @@ public static RaggedArray create(T[] col, int m) { /** * Wrap a fully populated raw typed column array into an {@link Array} of the given value type. The runtime type of - * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for {@link ValueType#FP64}, - * {@code String[]} for {@link ValueType#STRING}). + * {@code col} must match the primitive backing type of {@code vt} (e.g. {@code double[]} for + * {@link ValueType#FP64}, {@code String[]} for {@link ValueType#STRING}). * - *

For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than - * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a - * plain {@code boolean[]} still ends up with the same representation as every other frame allocation path), - * while shorter columns stay a plain {@link BooleanArray}.

+ *

+ * For {@link ValueType#BOOLEAN} this mirrors {@link #allocateBoolean(int)}: a {@code boolean[]} longer than + * {@link #bitSetSwitchPoint} is bit-packed into a compact {@link BitSetArray} (so a bulk decoder that fills a plain + * {@code boolean[]} still ends up with the same representation as every other frame allocation path), while shorter + * columns stay a plain {@link BooleanArray}. + *

* * @param vt the value type of the column * @param col the backing array to wrap @@ -168,10 +170,10 @@ public static Array create(ValueType vt, Object col) { /** * Allocate the raw backing array for a column of the given value type: the inverse of - * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, - * {@code int[]} for INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The - * runtime array type matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this - * primitive array directly and then wrap it via {@code create(vt, backing)}. + * {@link #create(ValueType, Object)}. Returns {@code double[]} for {@link ValueType#FP64}, {@code int[]} for + * INT32/UINT/HASH32, {@code long[]} for INT64/HASH64, {@code String[]} for STRING, etc. The runtime array type + * matches what {@link #create(ValueType, Object)} expects, so a bulk decoder can fill this primitive array directly + * and then wrap it via {@code create(vt, backing)}. * * @param vt the value type of the column * @param nRow the number of rows to allocate diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java index 9ff58065d97..95be95117e2 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/MatrixBlockFromFrame.java @@ -41,7 +41,7 @@ public class MatrixBlockFromFrame { public static Boolean WARNED_FOR_FAILED_CAST = false; - private MatrixBlockFromFrame(){ + private MatrixBlockFromFrame() { // private constructor for code coverage. } @@ -115,7 +115,7 @@ private static long convert(FrameBlock frame, MatrixBlock mb, int n, int rl, int return convertStrict(frame, mb, n, rl, ru); } catch(NumberFormatException | DMLRuntimeException e) { - synchronized(WARNED_FOR_FAILED_CAST){ + synchronized(WARNED_FOR_FAILED_CAST) { if(!WARNED_FOR_FAILED_CAST) { LOG.error( "Failed to convert to Matrix because of number format errors, falling back to NaN on incompatible cells", diff --git a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java index eed2c58f78c..c12a187cf17 100644 --- a/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java +++ b/src/main/java/org/apache/sysds/runtime/functionobjects/Builtin.java @@ -30,32 +30,13 @@ import jdk.incubator.vector.VectorSpecies; -/** - * Class with pre-defined set of objects. This class can not be instantiated elsewhere. - * - * Notes on commons.math FastMath: - * * FastMath uses lookup tables and interpolation instead of native calls. - * * The memory overhead for those tables is roughly 48KB in total (acceptable) - * * Micro and application benchmarks showed significantly (30%-3x) performance improvements - * for most operations; without loss of accuracy. - * * atan / sqrt were 20% slower in FastMath and hence, we use Math there - * * round / abs were equivalent in FastMath and hence, we use Math there - * * Finally, there is just one argument against FastMath - The comparison heavily depends - * on the JVM. For example, currently the IBM JDK JIT compiles to HW instructions for sqrt - * which makes this operation very efficient; as soon as other operations like log/exp are - * similarly compiled, we should rerun the micro benchmarks, and switch back if necessary. - * - */ -public class Builtin extends ValueFunction -{ - private static final long serialVersionUID = 3836744687789840574L; - - public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, - MAX, ABS, SIGN, SQRT, EXP, PLOGP, PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, - STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, - TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, - DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, GET_CATEGORICAL_MASK, - MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE} + public enum BuiltinCode { + AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, ATAN, LOG, LOG_NZ, MIN, MAX, ABS, SIGN, SQRT, EXP, PLOGP, + PRINT, PRINTF, NROW, NCOL, LENGTH, LINEAGE, ROUND, MAXINDEX, MININDEX, STOP, CEIL, FLOOR, CUMSUM, ROWCUMSUM, + CUMPROD, CUMMIN, CUMMAX, CUMSUMPROD, INVERSE, SPROP, SIGMOID, EVAL, LIST, TYPEOF, APPLY_SCHEMA, DETECTSCHEMA, + ISNA, ISNAN, ISINF, DROP_INVALID_TYPE, DROP_INVALID_LENGTH, VALUE_SWAP, FRAME_ROW_REPLICATE, + GET_CATEGORICAL_MASK, MAP, COUNT_DISTINCT, COUNT_DISTINCT_APPROX, UNIQUE + } private static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; private static final int vLen = SPECIES.length(); @@ -68,59 +49,59 @@ public enum BuiltinCode { AUTODIFF, SIN, COS, TAN, SINH, COSH, TANH, ASIN, ACOS, static public HashMap String2BuiltinCode; static { String2BuiltinCode = new HashMap<>(); - String2BuiltinCode.put( "autoDiff" , BuiltinCode.AUTODIFF); - String2BuiltinCode.put( "sin" , BuiltinCode.SIN); - String2BuiltinCode.put( "cos" , BuiltinCode.COS); - String2BuiltinCode.put( "tan" , BuiltinCode.TAN); - String2BuiltinCode.put( "sinh" , BuiltinCode.SINH); - String2BuiltinCode.put( "cosh" , BuiltinCode.COSH); - String2BuiltinCode.put( "tanh" , BuiltinCode.TANH); - String2BuiltinCode.put( "asin" , BuiltinCode.ASIN); - String2BuiltinCode.put( "acos" , BuiltinCode.ACOS); - String2BuiltinCode.put( "atan" , BuiltinCode.ATAN); - String2BuiltinCode.put( "log" , BuiltinCode.LOG); - String2BuiltinCode.put( "log_nz" , BuiltinCode.LOG_NZ); - String2BuiltinCode.put( "min" , BuiltinCode.MIN); - String2BuiltinCode.put( "max" , BuiltinCode.MAX); - String2BuiltinCode.put( "maxindex", BuiltinCode.MAXINDEX); - String2BuiltinCode.put( "minindex", BuiltinCode.MININDEX); - String2BuiltinCode.put( "abs" , BuiltinCode.ABS); - String2BuiltinCode.put( "sign" , BuiltinCode.SIGN); - String2BuiltinCode.put( "sqrt" , BuiltinCode.SQRT); - String2BuiltinCode.put( "exp" , BuiltinCode.EXP); - String2BuiltinCode.put( "plogp" , BuiltinCode.PLOGP); - String2BuiltinCode.put( "print" , BuiltinCode.PRINT); - String2BuiltinCode.put( "printf" , BuiltinCode.PRINTF); - String2BuiltinCode.put( "eval" , BuiltinCode.EVAL); - String2BuiltinCode.put( "list" , BuiltinCode.LIST); - String2BuiltinCode.put( "nrow" , BuiltinCode.NROW); - String2BuiltinCode.put( "ncol" , BuiltinCode.NCOL); - String2BuiltinCode.put( "length" , BuiltinCode.LENGTH); - String2BuiltinCode.put( "round" , BuiltinCode.ROUND); - String2BuiltinCode.put( "stop" , BuiltinCode.STOP); - String2BuiltinCode.put( "ceil" , BuiltinCode.CEIL); - String2BuiltinCode.put( "floor" , BuiltinCode.FLOOR); - String2BuiltinCode.put( "ucumk+" , BuiltinCode.CUMSUM); - String2BuiltinCode.put( "urowcumk+" , BuiltinCode.ROWCUMSUM); - String2BuiltinCode.put( "ucum*" , BuiltinCode.CUMPROD); - String2BuiltinCode.put( "ucumk+*", BuiltinCode.CUMSUMPROD); - String2BuiltinCode.put( "ucummin", BuiltinCode.CUMMIN); - String2BuiltinCode.put( "ucummax", BuiltinCode.CUMMAX); - String2BuiltinCode.put( "inverse", BuiltinCode.INVERSE); - String2BuiltinCode.put( "sprop", BuiltinCode.SPROP); - String2BuiltinCode.put( "sigmoid", BuiltinCode.SIGMOID); - String2BuiltinCode.put( "typeOf", BuiltinCode.TYPEOF); - String2BuiltinCode.put( "detectSchema", BuiltinCode.DETECTSCHEMA); - String2BuiltinCode.put( "isna", BuiltinCode.ISNA); - String2BuiltinCode.put( "isnan", BuiltinCode.ISNAN); - String2BuiltinCode.put( "isinf", BuiltinCode.ISINF); - String2BuiltinCode.put( "dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); - String2BuiltinCode.put( "freplicate", BuiltinCode.FRAME_ROW_REPLICATE); - String2BuiltinCode.put( "dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); - String2BuiltinCode.put( "_map", BuiltinCode.MAP); - String2BuiltinCode.put( "valueSwap", BuiltinCode.VALUE_SWAP); - String2BuiltinCode.put( "applySchema", BuiltinCode.APPLY_SCHEMA); - String2BuiltinCode.put( "get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); + String2BuiltinCode.put("autoDiff", BuiltinCode.AUTODIFF); + String2BuiltinCode.put("sin", BuiltinCode.SIN); + String2BuiltinCode.put("cos", BuiltinCode.COS); + String2BuiltinCode.put("tan", BuiltinCode.TAN); + String2BuiltinCode.put("sinh", BuiltinCode.SINH); + String2BuiltinCode.put("cosh", BuiltinCode.COSH); + String2BuiltinCode.put("tanh", BuiltinCode.TANH); + String2BuiltinCode.put("asin", BuiltinCode.ASIN); + String2BuiltinCode.put("acos", BuiltinCode.ACOS); + String2BuiltinCode.put("atan", BuiltinCode.ATAN); + String2BuiltinCode.put("log", BuiltinCode.LOG); + String2BuiltinCode.put("log_nz", BuiltinCode.LOG_NZ); + String2BuiltinCode.put("min", BuiltinCode.MIN); + String2BuiltinCode.put("max", BuiltinCode.MAX); + String2BuiltinCode.put("maxindex", BuiltinCode.MAXINDEX); + String2BuiltinCode.put("minindex", BuiltinCode.MININDEX); + String2BuiltinCode.put("abs", BuiltinCode.ABS); + String2BuiltinCode.put("sign", BuiltinCode.SIGN); + String2BuiltinCode.put("sqrt", BuiltinCode.SQRT); + String2BuiltinCode.put("exp", BuiltinCode.EXP); + String2BuiltinCode.put("plogp", BuiltinCode.PLOGP); + String2BuiltinCode.put("print", BuiltinCode.PRINT); + String2BuiltinCode.put("printf", BuiltinCode.PRINTF); + String2BuiltinCode.put("eval", BuiltinCode.EVAL); + String2BuiltinCode.put("list", BuiltinCode.LIST); + String2BuiltinCode.put("nrow", BuiltinCode.NROW); + String2BuiltinCode.put("ncol", BuiltinCode.NCOL); + String2BuiltinCode.put("length", BuiltinCode.LENGTH); + String2BuiltinCode.put("round", BuiltinCode.ROUND); + String2BuiltinCode.put("stop", BuiltinCode.STOP); + String2BuiltinCode.put("ceil", BuiltinCode.CEIL); + String2BuiltinCode.put("floor", BuiltinCode.FLOOR); + String2BuiltinCode.put("ucumk+", BuiltinCode.CUMSUM); + String2BuiltinCode.put("urowcumk+", BuiltinCode.ROWCUMSUM); + String2BuiltinCode.put("ucum*", BuiltinCode.CUMPROD); + String2BuiltinCode.put("ucumk+*", BuiltinCode.CUMSUMPROD); + String2BuiltinCode.put("ucummin", BuiltinCode.CUMMIN); + String2BuiltinCode.put("ucummax", BuiltinCode.CUMMAX); + String2BuiltinCode.put("inverse", BuiltinCode.INVERSE); + String2BuiltinCode.put("sprop", BuiltinCode.SPROP); + String2BuiltinCode.put("sigmoid", BuiltinCode.SIGMOID); + String2BuiltinCode.put("typeOf", BuiltinCode.TYPEOF); + String2BuiltinCode.put("detectSchema", BuiltinCode.DETECTSCHEMA); + String2BuiltinCode.put("isna", BuiltinCode.ISNA); + String2BuiltinCode.put("isnan", BuiltinCode.ISNAN); + String2BuiltinCode.put("isinf", BuiltinCode.ISINF); + String2BuiltinCode.put("dropInvalidType", BuiltinCode.DROP_INVALID_TYPE); + String2BuiltinCode.put("freplicate", BuiltinCode.FRAME_ROW_REPLICATE); + String2BuiltinCode.put("dropInvalidLength", BuiltinCode.DROP_INVALID_LENGTH); + String2BuiltinCode.put("_map", BuiltinCode.MAP); + String2BuiltinCode.put("valueSwap", BuiltinCode.VALUE_SWAP); + String2BuiltinCode.put("applySchema", BuiltinCode.APPLY_SCHEMA); + String2BuiltinCode.put("get_categorical_mask", BuiltinCode.GET_CATEGORICAL_MASK); } protected Builtin(BuiltinCode bf) { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java index 86184f47be6..08d28512d5c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryCPInstruction.java @@ -59,7 +59,7 @@ else if (in1.getDataType() == DataType.TENSOR && in2.getDataType() == DataType.T return new BinaryTensorTensorCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.FRAME) return new BinaryFrameFrameCPInstruction(operator, in1, in2, out, opcode, str); - else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) + else if(in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.SCALAR) return new BinaryFrameScalarCPInstruction(operator, in1, in2, out, opcode, str); else if (in1.getDataType() == DataType.FRAME && in2.getDataType() == DataType.MATRIX) return new BinaryFrameMatrixCPInstruction(operator, in1, in2, out, opcode, str); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java index 193894fd9bc..de76fca18b8 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryFrameScalarCPInstruction.java @@ -37,8 +37,8 @@ public class BinaryFrameScalarCPInstruction extends BinaryCPInstruction { // private static final Log LOG = LogFactory.getLog(BinaryFrameFrameCPInstruction.class.getName()); - private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, - TfMethod.WORD_EMBEDDING, TfMethod.BAG_OF_WORDS, TfMethod.UDF}; + private static final TfMethod[] UNSUPPORTED_MASK_METHODS = new TfMethod[] {TfMethod.BIN, TfMethod.WORD_EMBEDDING, + TfMethod.BAG_OF_WORDS, TfMethod.UDF}; protected BinaryFrameScalarCPInstruction(MultiThreadedOperator op, CPOperand in1, CPOperand in2, CPOperand out, String opcode, String istr) { @@ -96,9 +96,9 @@ public void processGetCategorical(ExecutionContext ec, FrameBlock f, ScalarObjec } /** - * Accumulates, per input column, how many output columns it expands to (lengths) and whether those - * output columns are categorical (categorical). The arrays are allocated lazily: a column that no - * method touches keeps the implicit default of a single, non-categorical output column. + * Accumulates, per input column, how many output columns it expands to (lengths) and whether those output columns + * are categorical (categorical). The arrays are allocated lazily: a column that no method touches keeps the + * implicit default of a single, non-categorical output column. */ private static final class CategoricalMask { private final FrameBlock f; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java index d76dbe0d45e..2c8093c3717 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/BinaryMatrixMatrixCPInstruction.java @@ -80,10 +80,10 @@ public void processInstruction(ExecutionContext ec) { retBlock = inBlock1; } else { - if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode()) ){ + if(LibCommonsMath.isSupportedMatrixMatrixOperation(getOpcode())) { if(compressedLeft) inBlock1 = CompressedMatrixBlock.getUncompressed(inBlock1, getOpcode()); - + if(compressedRight) inBlock2 = CompressedMatrixBlock.getUncompressed(inBlock2, getOpcode()); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index e53958ac4b8..97ae151ecc0 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -350,9 +350,10 @@ else if(opcode.equalsIgnoreCase(Opcodes.TRANSFORMDECODE.toString())) { String[] colnames = meta.getColumnNames(); // compute transformdecode - Decoder decoder = DecoderFactory - .createDecoder(getParameterMap().get("spec"), colnames, null, meta, data.getNumColumns()); - FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), InfrastructureAnalyzer.getLocalParallelism()); + Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta, + data.getNumColumns()); + FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()), + InfrastructureAnalyzer.getLocalParallelism()); fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); // release locks diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java index 40a677e5d71..04353806ca3 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReorgOOCInstruction.java @@ -45,8 +45,8 @@ protected ReorgOOCInstruction(ReorgOperator op, CPOperand in1, CPOperand out, St this(op, in1, out, null, null, null, opcode, istr); } - private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, CPOperand ixret, - String opcode, String istr) { + private ReorgOOCInstruction(Operator op, CPOperand in, CPOperand out, CPOperand col, CPOperand desc, + CPOperand ixret, String opcode, String istr) { super(OOCType.Reorg, op, in, out, opcode, istr); _col = col; _desc = desc; @@ -76,8 +76,8 @@ else if(opcode.equalsIgnoreCase(Opcodes.SORT.toString())) { CPOperand desc = new CPOperand(parts[3]); CPOperand ixret = new CPOperand(parts[4]); int k = Integer.parseInt(parts[6]); - return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), - in, out, col, desc, ixret, opcode, str); + return new ReorgOOCInstruction(new ReorgOperator(new SortIndex(1, false, false), k), in, out, col, desc, + ixret, opcode, str); } else throw new NotImplementedException(); diff --git a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java index 7590438b949..091c6785b3c 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/ooc/ReshapeOOCInstruction.java @@ -128,17 +128,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in row - return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes(1, (r - 1) * numBlocksPerRowIn + c), + tmp.getValue()); }); } else { f.join(); - reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapeFullColBlocks(rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, + singleRowBlocks, qOut); } } else { f.join(); - reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, numBlocksPerColOut, singleRowBlocks, qOut); + reshapePartialColBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowIn, numBlocksPerRowOut, + numBlocksPerColOut, singleRowBlocks, qOut); } } else { @@ -166,17 +169,20 @@ public void processInstruction(ExecutionContext ec) { long r = tmp.getIndexes().getRowIndex(); long c = tmp.getIndexes().getColumnIndex(); // adapt index to new position in col - return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), tmp.getValue()); + return new IndexedMatrixValue(new MatrixIndexes((c - 1) * numBlocksPerColIn + r, 1), + tmp.getValue()); }); } else { f.join(); - reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapeFullRowBlocks(rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, + singleColBlocks, qOut); } } else { f.join(); - reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, numBlocksPerColOut, singleColBlocks, qOut); + reshapePartialRowBlocks(rlen, clen, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColIn, + numBlocksPerColOut, singleColBlocks, qOut); } } } @@ -226,7 +232,8 @@ private void reshapeFullColBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowIn, - int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, OOCStream qOut) { + int numBlocksPerRowOut, int numBlocksPerColOut, OOCStream singleRowBlocks, + OOCStream qOut) { // use cache for accessing input rows by index CachingStream singleRowBlockCache = new CachingStream(singleRowBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -237,14 +244,16 @@ private void reshapePartialColBlocks(long rlen, long clen, long rows, long cols, int r = 0; // allocate row of output blocks - MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut,true); + MatrixBlock[] outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + true); int offsetOut = 0; int localColsOut = (cols > blen) ? blen : (int) cols; // iterate through input rows and add to row of output blocks for(int i = 1; i <= rlen; i++) { for(int j = 1; j <= numBlocksPerRowIn; j++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -278,10 +287,12 @@ else if(remIn == remOut) { if(r == outputBlockRow[0].getNumRows()) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockRow.length; b++) - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(br + 1, b + 1), outputBlockRow[b])); br++; // allocate new block row - outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, true); + outputBlockRow = allocateSliceBlocks(br, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, true); r = 0; } bc = 0; @@ -341,7 +352,8 @@ private void reshapeFullRowBlocks(long rows, long cols, int blen, int numBlocksP } private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int blen, int numBlocksPerRowOut, - int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, OOCStream qOut) { + int numBlocksPerColIn, int numBlocksPerColOut, OOCStream singleColBlocks, + OOCStream qOut) { // use cache for accessing input cols by index CachingStream singleRowBlockCache = new CachingStream(singleColBlocks); singleRowBlockCache.incrSubscriberCount(1); @@ -352,14 +364,16 @@ private void reshapePartialRowBlocks(long rlen, long clen, long rows, long cols, int c = 0; // allocate col of output blocks - MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + MatrixBlock[] outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, + false); int offsetOut = 0; int localRowsOut = (rows > blen) ? blen : (int) rows; // iterate through input cols and add to col of output blocks for(int j = 1; j <= clen; j++) { for(int i = 1; i <= numBlocksPerColIn; i++) { - try(OOCStream.QueueCallback qcb = singleRowBlockCache.findCached(new MatrixIndexes(i, j))) { + try(OOCStream.QueueCallback qcb = singleRowBlockCache + .findCached(new MatrixIndexes(i, j))) { MatrixBlock blk = (MatrixBlock) qcb.get().getValue(); int offsetIn = 0; @@ -394,11 +408,13 @@ else if(remIn == remOut) { // enqueue filled output blocks and allocate new ones for(int b = 0; b < outputBlockCol.length; b++) { outputBlockCol[b].recomputeNonZeros(); - qOut.enqueue(new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); + qOut.enqueue( + new IndexedMatrixValue(new MatrixIndexes(b + 1, bc + 1), outputBlockCol[b])); } bc++; // allocate new block col - outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, numBlocksPerColOut, false); + outputBlockCol = allocateSliceBlocks(bc, rows, cols, blen, numBlocksPerRowOut, + numBlocksPerColOut, false); c = 0; } br = 0; @@ -411,16 +427,20 @@ else if(remIn == remOut) { qOut.closeInput(); } - private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, int numBlocksPerCol, boolean isBlockRowSlice) { + private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int blen, int numBlocksPerRow, + int numBlocksPerCol, boolean isBlockRowSlice) { int num = isBlockRowSlice ? numBlocksPerRow : numBlocksPerCol; MatrixBlock[] res = new MatrixBlock[num]; // full inner blocks, adjust for outer blocks - int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % blen : blen; - int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % blen : blen; + int localRows = ((!isBlockRowSlice || idx == numBlocksPerCol - 1) && rows % blen != 0) ? (int) rows % + blen : blen; + int localCols = ((isBlockRowSlice || idx == numBlocksPerRow - 1) && cols % blen != 0) ? (int) cols % + blen : blen; for(int k = 0; k < num - 1; k++) { - res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, false); + res[k] = isBlockRowSlice ? new MatrixBlock(localRows, blen, false) : new MatrixBlock(blen, localCols, + false); res[k].allocateDenseBlock(); } res[num - 1] = new MatrixBlock(localRows, localCols, false); @@ -428,10 +448,13 @@ private MatrixBlock[] allocateSliceBlocks(int idx, long rows, long cols, int ble return res; } - private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, boolean rowWise) { + private void setOutputEntries(MatrixBlock src, MatrixBlock dest, int idx, int srcOffset, int destOffset, int length, + boolean rowWise) { if(rowWise) - ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialRow(src.getDenseBlock(), idx, srcOffset, destOffset, + length); else - ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, length); + ((DenseBlockFP64) dest.getDenseBlock()).setPartialCol(src.getDenseBlock(), idx, srcOffset, destOffset, + length); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java index 75f84882478..e25219b80ba 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -114,8 +114,7 @@ public void processInstruction(ExecutionContext ec) { case VALUEPICK: { if( input2.isScalar() ) { ScalarObject quantile = ec.getScalarInput(input2); - double[] wt = getWeightedQuantileSummary(in, mc, - new double[] {quantile.getDoubleValue()}, true); + double[] wt = getWeightedQuantileSummary(in, mc, new double[] {quantile.getDoubleValue()}, true); ec.setScalarOutput(output.getName(), new DoubleObject(wt[3])); } else { diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java index 2f83caa5526..f007558ebdc 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/data/IndexedMatrixValue.java @@ -30,8 +30,7 @@ import org.apache.sysds.runtime.matrix.data.MatrixValue; import org.apache.sysds.runtime.ooc.cache.io.SpillableObject; -public class IndexedMatrixValue implements SpillableObject, Serializable -{ +public class IndexedMatrixValue implements SpillableObject, Serializable { private static final long serialVersionUID = 6723389820806752110L; private MatrixIndexes _indexes = null; @@ -110,8 +109,8 @@ public void discard() { @Override public void read(DataInput dataInput) throws IOException { - _indexes = new MatrixIndexes(); - _value = new MatrixBlock(); + _indexes = new MatrixIndexes(); + _value = new MatrixBlock(); _indexes.readFields(dataInput); _value.readFields(dataInput); } diff --git a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java index c3b9351d3d3..cc8491d4515 100644 --- a/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java +++ b/src/main/java/org/apache/sysds/runtime/io/DeltaKernelUtils.java @@ -87,11 +87,10 @@ import io.delta.kernel.utils.FileStatus; /** - * Shared helpers for the native (Spark-free) Delta Lake read/write paths used - * by both the matrix and frame readers/writers. Centralizes engine creation, - * path qualification, the scan loop (snapshot -> data files -> logical - * columnar batches, honoring deletion vectors), and the write transaction - * (logical data -> parquet -> commit). + * Shared helpers for the native (Spark-free) Delta Lake read/write paths used by both the matrix and frame + * readers/writers. Centralizes engine creation, path qualification, the scan loop (snapshot -> data files -> + * logical columnar batches, honoring deletion vectors), and the write transaction (logical data -> parquet -> + * commit). */ public class DeltaKernelUtils { @@ -102,23 +101,29 @@ public class DeltaKernelUtils { /** Reused thread-safe JSON reader for the per-file Delta stats (numRecords). */ private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - /** Delta Kernel config key: number of rows per parquet read batch, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. */ + /** + * Delta Kernel config key: number of rows per parquet read batch, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_READER_BATCH_SIZE}. + */ private static final String CONF_READER_BATCH_SIZE = "delta.kernel.default.parquet.reader.batch-size"; - /** Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via - * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. */ + /** + * Delta Kernel config key: target size (bytes) at which the writer rolls a new data file, overridable via + * {@link org.apache.sysds.conf.DMLConfig#DELTA_WRITER_TARGET_FILE_SIZE}. + */ private static final String CONF_WRITER_TARGET_FILE_SIZE = "delta.kernel.default.parquet.writer.targetMaxFileSize"; - /** Internal Delta column type codes shared by the matrix and frame readers to - * dispatch boxing-free primitive column access. */ - public static final int T_DOUBLE = 0; - public static final int T_FLOAT = 1; - public static final int T_LONG = 2; - public static final int T_INT = 3; - public static final int T_SHORT = 4; - public static final int T_BYTE = 5; + /** + * Internal Delta column type codes shared by the matrix and frame readers to dispatch boxing-free primitive column + * access. + */ + public static final int T_DOUBLE = 0; + public static final int T_FLOAT = 1; + public static final int T_LONG = 2; + public static final int T_INT = 3; + public static final int T_SHORT = 4; + public static final int T_BYTE = 5; public static final int T_BOOLEAN = 6; - public static final int T_STRING = 7; + public static final int T_STRING = 7; /** * Parquet physical type each {@code T_*} column is stored as, indexed by type code (delta int/short/byte columns @@ -128,22 +133,22 @@ public class DeltaKernelUtils { PrimitiveTypeName.INT64, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.INT32, PrimitiveTypeName.BOOLEAN, PrimitiveTypeName.BINARY}; - //derived configuration cached to avoid copying the (large) base conf on every - //engine creation (createEngine is called once per data file in parallel reads); - //rebuilt whenever the base conf or the relevant SystemDS settings change. + // derived configuration cached to avoid copying the (large) base conf on every + // engine creation (createEngine is called once per data file in parallel reads); + // rebuilt whenever the base conf or the relevant SystemDS settings change. private static Configuration cachedConf; private static Configuration cachedConfBase; private static int cachedBatchSize; private static long cachedTargetFileSize; - private DeltaKernelUtils() {} + private DeltaKernelUtils() { + } /** - * Consumes a whole columnar batch. {@code selected} is {@code null} when all - * {@code size} rows are live; otherwise {@code selected[r]} indicates whether - * row {@code r} survived the deletion/selection vector. Batch-level consumption - * lets callers extract data column-at-a-time (cache friendly, boxing free) - * instead of paying a per-row callback. + * Consumes a whole columnar batch. {@code selected} is {@code null} when all {@code size} rows are live; otherwise + * {@code selected[r]} indicates whether row {@code r} survived the deletion/selection vector. Batch-level + * consumption lets callers extract data column-at-a-time (cache friendly, boxing free) instead of paying a per-row + * callback. */ @FunctionalInterface public interface BatchConsumer { @@ -151,22 +156,29 @@ public interface BatchConsumer { } /** - * Map a Delta Kernel {@link DataType} to an internal type code (see the - * {@code T_*} constants). Returned once per column so the per-cell read loop - * can switch on a primitive int instead of repeating {@code instanceof} checks. + * Map a Delta Kernel {@link DataType} to an internal type code (see the {@code T_*} constants). Returned once per + * column so the per-cell read loop can switch on a primitive int instead of repeating {@code instanceof} checks. * * @param dt the Delta column data type * @return the matching {@code T_*} code, or {@code -1} if the type is not supported */ public static int typeCode(DataType dt) { - if( dt instanceof DoubleType ) return T_DOUBLE; - if( dt instanceof FloatType ) return T_FLOAT; - if( dt instanceof LongType ) return T_LONG; - if( dt instanceof IntegerType ) return T_INT; - if( dt instanceof ShortType ) return T_SHORT; - if( dt instanceof ByteType ) return T_BYTE; - if( dt instanceof BooleanType ) return T_BOOLEAN; - if( dt instanceof StringType ) return T_STRING; + if(dt instanceof DoubleType) + return T_DOUBLE; + if(dt instanceof FloatType) + return T_FLOAT; + if(dt instanceof LongType) + return T_LONG; + if(dt instanceof IntegerType) + return T_INT; + if(dt instanceof ShortType) + return T_SHORT; + if(dt instanceof ByteType) + return T_BYTE; + if(dt instanceof BooleanType) + return T_BOOLEAN; + if(dt instanceof StringType) + return T_STRING; return -1; } @@ -429,8 +441,10 @@ private static void decodeColumnInto(ColumnReader creader, int maxDef, int nrow, } } - /** Floor on the adaptive writer target file size. Below this the per-file metadata/open - * overhead (and tiny-file proliferation) outweighs the extra read parallelism. */ + /** + * Floor on the adaptive writer target file size. Below this the per-file metadata/open overhead (and tiny-file + * proliferation) outweighs the extra read parallelism. + */ public static final long ADAPTIVE_WRITER_MIN_FILE_SIZE = 4L * 1024 * 1024; private static Configuration buildConf(Configuration base, int batchSize, long targetFileSize) { @@ -444,9 +458,8 @@ private static synchronized Configuration deltaConf() { Configuration base = ConfigurationManager.getCachedJobConf(); int batchSize = ConfigurationManager.getDeltaReaderBatchSize(); long targetFileSize = ConfigurationManager.getDeltaWriterTargetFileSize(); - if(cachedConf == null || cachedConfBase != base - || cachedBatchSize != batchSize || cachedTargetFileSize != targetFileSize) - { + if(cachedConf == null || cachedConfBase != base || cachedBatchSize != batchSize || + cachedTargetFileSize != targetFileSize) { cachedConf = buildConf(base, batchSize, targetFileSize); cachedConfBase = base; cachedBatchSize = batchSize; @@ -460,12 +473,11 @@ public static Engine createEngine() { } /** - * Compute the parquet target data-file size (bytes) for writing a table of the given - * estimated size. With adaptive sizing enabled the writer aims for roughly one data - * file per expected parallel reader (so the native per-file parallel read can use all - * threads): never above the configured target, and never below - * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller - * than that floor (in which case the configured target wins). + * Compute the parquet target data-file size (bytes) for writing a table of the given estimated size. With adaptive + * sizing enabled the writer aims for roughly one data file per expected parallel reader (so the native per-file + * parallel read can use all threads): never above the configured target, and never below + * {@code ADAPTIVE_WRITER_MIN_FILE_SIZE} unless the configured target is itself smaller than that floor (in which + * case the configured target wins). * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return the target max parquet data-file size in bytes @@ -476,7 +488,7 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { return configured; int par = Math.max(1, OptimizerUtils.getParallelBinaryReadParallelism()); long perReader = Math.max(1, estimatedBytes / par); - //never above the configured cap, never below the floor (unless the cap itself is lower) + // never above the configured cap, never below the floor (unless the cap itself is lower) long target = Math.min(configured, Math.max(ADAPTIVE_WRITER_MIN_FILE_SIZE, perReader)); if(LOG.isDebugEnabled()) LOG.debug("Delta adaptive file size: est=" + estimatedBytes + "B par=" + par + " -> target=" + target @@ -485,24 +497,24 @@ public static long adaptiveWriterTargetFileSize(long estimatedBytes) { } /** - * Create an engine for writing a table of the given estimated size, configured with an - * adaptive target data-file size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh - * (uncached) configuration is built since writes happen once per table, not per data file. + * Create an engine for writing a table of the given estimated size, configured with an adaptive target data-file + * size (see {@link #adaptiveWriterTargetFileSize(long)}). A fresh (uncached) configuration is built since writes + * happen once per table, not per data file. * * @param estimatedBytes estimate of the table's size (the block in-memory size is a fine proxy) * @return a Delta Kernel engine for the write */ public static Engine createWriteEngine(long estimatedBytes) { - //the reader batch size is irrelevant on the write path but is set to keep the - //conf shape identical to deltaConf(); only the target file size matters here. + // the reader batch size is irrelevant on the write path but is set to keep the + // conf shape identical to deltaConf(); only the target file size matters here. Configuration c = buildConf(ConfigurationManager.getCachedJobConf(), ConfigurationManager.getDeltaReaderBatchSize(), adaptiveWriterTargetFileSize(estimatedBytes)); return DefaultEngine.create(c); } /** - * Resolve a (possibly relative) path to a fully-qualified URI so the - * kernel's default engine can locate the table on the right filesystem. + * Resolve a (possibly relative) path to a fully-qualified URI so the kernel's default engine can locate the table + * on the right filesystem. * * @param fname input path * @return fully-qualified table path @@ -519,11 +531,9 @@ public static String qualify(String fname) { } /** - * Opened latest snapshot of a Delta table: the logical schema plus everything - * needed to (re)read its data files, including the list of per-data-file scan - * rows. Delta Kernel scan-file rows are self-contained (the kernel's - * distributed design serializes them to workers), so they can be retained and - * read independently / in parallel. + * Opened latest snapshot of a Delta table: the logical schema plus everything needed to (re)read its data files, + * including the list of per-data-file scan rows. Delta Kernel scan-file rows are self-contained (the kernel's + * distributed design serializes them to workers), so they can be retained and read independently / in parallel. */ public static final class ScanHandle { public final StructType schema; @@ -531,19 +541,18 @@ public static final class ScanHandle { public final StructType physicalReadSchema; public final List scanFiles; /** - * Per-file record counts taken from the Delta {@code numRecords} statistic, - * aligned with {@link #scanFiles}; {@code -1} where the statistic is absent. + * Per-file record counts taken from the Delta {@code numRecords} statistic, aligned with {@link #scanFiles}; + * {@code -1} where the statistic is absent. */ public final long[] numRecords; /** - * Per-file flag indicating a deletion vector is present (so the live row - * count differs from {@link #numRecords}), aligned with {@link #scanFiles}. + * Per-file flag indicating a deletion vector is present (so the live row count differs from + * {@link #numRecords}), aligned with {@link #scanFiles}. */ public final boolean[] hasDeletionVector; - private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, - List scanFiles, long[] numRecords, boolean[] hasDeletionVector) - { + private ScanHandle(StructType schema, Row scanState, StructType physicalReadSchema, List scanFiles, + long[] numRecords, boolean[] hasDeletionVector) { this.schema = schema; this.scanState = scanState; this.physicalReadSchema = physicalReadSchema; @@ -553,13 +562,12 @@ private ScanHandle(StructType schema, Row scanState, StructType physicalReadSche } /** - * @return true iff every data file carries a {@code numRecords} statistic - * and none has a deletion vector, i.e. exact per-file row offsets - * can be derived from metadata without reading the data. + * @return true iff every data file carries a {@code numRecords} statistic and none has a deletion vector, i.e. + * exact per-file row offsets can be derived from metadata without reading the data. */ public boolean hasExactRowCounts() { - for( int i=0; i scanFileIter = (scan instanceof ScanImpl) - ? ((ScanImpl) scan).getScanFiles(engine, true) - : scan.getScanFiles(engine); + // request the scan files WITH per-file statistics (numRecords) so callers can + // pre-size output and place rows without reading the data; harmless extra + // column for the data-read path. Fall back to the stats-less iterator if the + // concrete scan does not support it. + CloseableIterator scanFileIter = (scan instanceof ScanImpl) ? ((ScanImpl) scan) + .getScanFiles(engine, true) : scan.getScanFiles(engine); List files = new ArrayList<>(); List recs = new ArrayList<>(); List dvs = new ArrayList<>(); - try( CloseableIterator scanFiles = scanFileIter ) { - while( scanFiles.hasNext() ) { + try(CloseableIterator scanFiles = scanFileIter) { + while(scanFiles.hasNext()) { FilteredColumnarBatch scanFileBatch = scanFiles.next(); - try( CloseableIterator scanFileRows = scanFileBatch.getRows() ) { - while( scanFileRows.hasNext() ) { + try(CloseableIterator scanFileRows = scanFileBatch.getRows()) { + while(scanFileRows.hasNext()) { Row scanFileRow = scanFileRows.next(); files.add(scanFileRow); recs.add(numRecords(scanFileRow)); @@ -609,7 +615,7 @@ public static ScanHandle openScan(Engine engine, String tablePath) throws IOExce } long[] numRecords = new long[recs.size()]; boolean[] hasDv = new boolean[dvs.size()]; - for( int i=0; i physicalData = engine.getParquetHandler() .readParquetFiles(Utils.singletonCloseableIterator(dataFile), physicalReadSchema, Optional.empty()); - try( CloseableIterator logicalData = - Scan.transformPhysicalData(engine, scanState, scanFileRow, physicalData) ) - { - while( logicalData.hasNext() ) + try(CloseableIterator logicalData = Scan.transformPhysicalData(engine, scanState, + scanFileRow, physicalData)) { + while(logicalData.hasNext()) consumeBatch(logicalData.next(), consumer); } } /** - * Scan the latest snapshot of a Delta table sequentially, invoking the batch - * consumer for every data batch. The consumer is created lazily from the table - * schema (so callers can size buffers / derive per-column types up front). + * Scan the latest snapshot of a Delta table sequentially, invoking the batch consumer for every data batch. The + * consumer is created lazily from the table schema (so callers can size buffers / derive per-column types up + * front). * * @param engine delta kernel engine * @param tablePath fully-qualified table path @@ -677,11 +679,10 @@ public static void readScanFile(Engine engine, Row scanState, StructType physica * @throws IOException on read failure */ public static StructType scan(Engine engine, String tablePath, Function consumerFactory) - throws IOException - { + throws IOException { ScanHandle h = openScan(engine, tablePath); BatchConsumer consumer = consumerFactory.apply(h.schema); - for( Row scanFileRow : h.scanFiles ) + for(Row scanFileRow : h.scanFiles) readScanFile(engine, h.scanState, h.physicalReadSchema, scanFileRow, consumer); return h.schema; } @@ -690,28 +691,27 @@ private static void consumeBatch(FilteredColumnarBatch fcb, BatchConsumer consum ColumnarBatch batch = fcb.getData(); int ncol = batch.getSchema().length(); ColumnVector[] cols = new ColumnVector[ncol]; - for( int c=0; c all rows live) + // materialize the deletion/selection mask once (null => all rows live) Optional selVector = fcb.getSelectionVector(); boolean[] selected = null; - if( selVector.isPresent() ) { + if(selVector.isPresent()) { ColumnVector sv = selVector.get(); selected = new boolean[size]; - for( int r=0; r logicalData) throws IOException - { - //replace any existing table at the path (the other SystemDS writers delete - //the output first; the caching layer does not do it on our behalf) + CloseableIterator logicalData) throws IOException { + // replace any existing table at the path (the other SystemDS writers delete + // the output first; the caching layer does not do it on our behalf) HDFSTool.deleteFileIfExistOnHDFS(tablePath); Table table = Table.forPath(engine, tablePath); - TransactionBuilder txnBuilder = table - .createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) + TransactionBuilder txnBuilder = table.createTransactionBuilder(engine, ENGINE_INFO, Operation.CREATE_TABLE) .withSchema(engine, schema); Transaction txn = txnBuilder.build(engine); Row txnState = txn.getTransactionState(engine); - CloseableIterator physicalData = - Transaction.transformLogicalData(engine, txnState, logicalData, Collections.emptyMap()); - DataWriteContext writeContext = - Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); + CloseableIterator physicalData = Transaction.transformLogicalData(engine, txnState, + logicalData, Collections.emptyMap()); + DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, Collections.emptyMap()); CloseableIterator dataFiles = engine.getParquetHandler() .writeParquetFiles(writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns()); - CloseableIterator appendActions = - Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext); + CloseableIterator appendActions = Transaction.generateAppendActions(engine, txnState, dataFiles, + writeContext); txn.commit(engine, CloseableIterable.inMemoryIterable(appendActions)); } } diff --git a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java index 58a98741975..55d8f8f7c2d 100644 --- a/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/ReaderDelta.java @@ -33,29 +33,27 @@ import io.delta.kernel.types.StructType; /** - * Single-threaded native Delta Lake reader for matrices, built on the - * Spark-free Delta Kernel library. It opens the latest snapshot of a Delta - * table directory, reads its parquet data files through the kernel's default - * engine (honoring deletion vectors), and materializes the numeric columns - * into a dense {@link MatrixBlock}. + * Single-threaded native Delta Lake reader for matrices, built on the Spark-free Delta Kernel library. It opens the + * latest snapshot of a Delta table directory, reads its parquet data files through the kernel's default engine + * (honoring deletion vectors), and materializes the numeric columns into a dense {@link MatrixBlock}. * - *

Only numeric columns (double/float/long/int/short/byte/boolean) are - * supported, matching the all-double nature of a SystemDS matrix. Dimensions - * do not need to be known up front: the row count is discovered while scanning - * and the column count is taken from the table schema.

+ *

+ * Only numeric columns (double/float/long/int/short/byte/boolean) are supported, matching the all-double nature of a + * SystemDS matrix. Dimensions do not need to be known up front: the row count is discovered while scanning and the + * column count is taken from the table schema. + *

*/ public class ReaderDelta extends MatrixReader { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); - //Scan column-at-a-time into one row-major buffer per batch (no per-row - //allocation, no boxing, no per-cell set()). Buffers are concatenated into - //the dense output via bulk array copies below. + // Scan column-at-a-time into one row-major buffer per batch (no per-row + // allocation, no boxing, no per-cell set()). Buffers are concatenated into + // the dense output via bulk array copies below. ArrayList batches = new ArrayList<>(); int[] nrowH = new int[1]; StructType schema = DeltaKernelUtils.scan(engine, tablePath, sch -> { @@ -72,7 +70,7 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl long lestnnz = (estnnz >= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, batches); ret.recomputeNonZeros(); ret.examSparsity(); @@ -83,14 +81,14 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl static int[] columnTypes(StructType schema) { int ncol = schema.length(); int[] types = new int[ncol]; - for( int c=0; c batches) { DenseBlock db = ret.getDenseBlock(); - if( db.isContiguous() ) { + if(db.isContiguous()) { double[] dv = db.valuesAt(0); int off = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { System.arraycopy(buf, 0, dv, off, buf.length); off += buf.length; } } else { - //rare large multi-block fallback: route each row through the block API + // rare large multi-block fallback: route each row through the block API int ncol = ret.getNumColumns(); int r = 0; - for( double[] buf : batches ) { + for(double[] buf : batches) { int rowsInBuf = buf.length / ncol; - for( int i=0; iThe expensive part of a Delta read is the parquet decode, which the kernel - * performs per data file; parallelizing across files is therefore the natural - * way to bridge the gap to the (near-raw) binary reader. A table backed by a - * single data file (the default for tables <= the parquet target file size) - * cannot be split this way, so the reader transparently falls back to the - * sequential {@link ReaderDelta} path in that case.

+ *

+ * The expensive part of a Delta read is the parquet decode, which the kernel performs per data file; parallelizing + * across files is therefore the natural way to bridge the gap to the (near-raw) binary reader. A table backed by a + * single data file (the default for tables <= the parquet target file size) cannot be split this way, so the reader + * transparently falls back to the sequential {@link ReaderDelta} path in that case. + *

*/ public class ReaderDeltaParallel extends ReaderDelta { @@ -57,28 +56,27 @@ public ReaderDeltaParallel() { @Override public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int blen, long estnnz) - throws IOException, DMLRuntimeException - { + throws IOException, DMLRuntimeException { Engine engine = DeltaKernelUtils.createEngine(); String tablePath = DeltaKernelUtils.qualify(fname); DeltaKernelUtils.ScanHandle handle = DeltaKernelUtils.openScan(engine, tablePath); final int nfiles = handle.scanFiles.size(); - //nothing to gain from parallelism for single-file (or empty) tables - if( _numThreads <= 1 || nfiles <= 1 ) + // nothing to gain from parallelism for single-file (or empty) tables + if(_numThreads <= 1 || nfiles <= 1) return super.readMatrixFromHDFS(fname, rlen, clen, blen, estnnz); final int ncol = handle.schema.length(); final int[] types = columnTypes(handle.schema); - //fast path: exact per-file row counts are known from metadata and the dense - //output fits a single contiguous array -> pre-size once and let each thread - //decode directly into its slice (no intermediate buffers, no serial copy). - if( useDirectPath(handle) ) { + // fast path: exact per-file row counts are known from metadata and the dense + // output fits a single contiguous array -> pre-size once and let each thread + // decode directly into its slice (no intermediate buffers, no serial copy). + if(useDirectPath(handle)) { long total = 0; - for( long r : handle.numRecords ) + for(long r : handle.numRecords) total += r; - if( total > 0 && (long) total * ncol <= Integer.MAX_VALUE ) + if(total > 0 && (long) total * ncol <= Integer.MAX_VALUE) return readDirect(fname, handle, ncol, types, (int) total, estnnz); } @@ -86,11 +84,9 @@ public MatrixBlock readMatrixFromHDFS(String fname, long rlen, long clen, int bl } /** - * Whether the metadata-driven direct-write fast path can be used for this - * table (exact per-file row counts and no deletion vectors). Visible for - * testing: the buffered fallback is otherwise only reachable for tables - * lacking row statistics or carrying deletion vectors, which the SystemDS - * Delta writer never produces. + * Whether the metadata-driven direct-write fast path can be used for this table (exact per-file row counts and no + * deletion vectors). Visible for testing: the buffered fallback is otherwise only reachable for tables lacking row + * statistics or carrying deletion vectors, which the SystemDS Delta writer never produces. * * @param handle the opened scan handle * @return true if the direct path is applicable @@ -100,38 +96,37 @@ protected boolean useDirectPath(DeltaKernelUtils.ScanHandle handle) { } /** - * Fast path: each thread decodes one data file straight into the final dense - * array at a metadata-derived row offset. Single allocation, fully parallel. + * Fast path: each thread decodes one data file straight into the final dense array at a metadata-derived row + * offset. Single allocation, fully parallel. */ - private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, int nrow, long estnnz) throws IOException - { + private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, int nrow, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); final int[] rowOffset = new int[nfiles]; int acc = 0; - for( int i=0; i> tasks = new ArrayList<>(nfiles); - for( int i=0; i { int[] cur = new int[] {base}; Engine eng = DeltaKernelUtils.createEngine(); DeltaKernelUtils.readScanFile(eng, handle.scanState, handle.physicalReadSchema, scanFileRow, (cols, size, selected) -> { - if( cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit ) + if(cur[0] + DeltaKernelUtils.countSelected(size, selected) > limit) throw new DMLRuntimeException("Delta file produced more rows than its " + "numRecords statistic; refusing parallel direct read of " + fname); cur[0] += extractBatchInto(cols, size, selected, types, ncol, dv, cur[0]); @@ -147,19 +142,17 @@ private MatrixBlock readDirect(String fname, DeltaKernelUtils.ScanHandle handle, } /** - * Fallback path: decode each file in parallel into per-file buffers (used when - * row counts are unknown, deletion vectors are present, or the matrix exceeds a - * single contiguous array), then concatenate in file order. + * Fallback path: decode each file in parallel into per-file buffers (used when row counts are unknown, deletion + * vectors are present, or the matrix exceeds a single contiguous array), then concatenate in file order. */ - private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, - int ncol, int[] types, long estnnz) throws IOException - { + private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handle, int ncol, int[] types, + long estnnz) throws IOException { final int nfiles = handle.scanFiles.size(); @SuppressWarnings("unchecked") final ArrayList[] fileBufs = new ArrayList[nfiles]; final int[] fileRows = new int[nfiles]; ArrayList> tasks = new ArrayList<>(nfiles); - for( int i=0; i { @@ -179,15 +172,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl awaitFileTasks(tasks, fname); int nrow = 0; - for( int i=0; i ordered = new ArrayList<>(); - for( int i=0; i= 0) ? estnnz : (long) nrow * ncol; MatrixBlock ret = createOutputMatrixBlock(nrow, ncol, Math.max(nrow, 1), lestnnz, true, false); - if( nrow > 0 && ncol > 0 ) + if(nrow > 0 && ncol > 0) fillDense(ret, ordered); ret.recomputeNonZeros(_numThreads); ret.examSparsity(); @@ -195,16 +188,15 @@ private MatrixBlock readBuffered(String fname, DeltaKernelUtils.ScanHandle handl } /** - * Run one decode task per data file on the shared common thread pool and await - * completion. Full parallelism is requested (the task count, one per data file, - * naturally caps concurrency); this avoids the per-thread pool-size caching in - * {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a - * smaller pool created earlier on the same thread. + * Run one decode task per data file on the shared common thread pool and await completion. Full parallelism is + * requested (the task count, one per data file, naturally caps concurrency); this avoids the per-thread pool-size + * caching in {@code CommonThreadPool.get(k)} that could otherwise throttle this reader to a smaller pool created + * earlier on the same thread. */ private void awaitFileTasks(List> tasks, String fname) throws IOException { ExecutorService pool = CommonThreadPool.get(_numThreads); try { - for( Future f : pool.invokeAll(tasks) ) + for(Future f : pool.invokeAll(tasks)) f.get(); } catch(Exception ex) { diff --git a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java index 55ea8a54297..602b540c407 100644 --- a/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java +++ b/src/main/java/org/apache/sysds/runtime/io/WriterDelta.java @@ -39,60 +39,54 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Single-threaded native Delta Lake writer for matrices, built on the - * Spark-free Delta Kernel library. It creates a Delta table at the target - * directory with an all-double schema {@code c0..c(n-1)} (replacing any existing - * table at that path), streams the {@link MatrixBlock} rows as columnar batches - * into parquet data files via the kernel's default engine, and commits the - * corresponding add-file actions to the transaction log. + * Single-threaded native Delta Lake writer for matrices, built on the Spark-free Delta Kernel library. It creates a + * Delta table at the target directory with an all-double schema {@code c0..c(n-1)} (replacing any existing table at + * that path), streams the {@link MatrixBlock} rows as columnar batches into parquet data files via the kernel's default + * engine, and commits the corresponding add-file actions to the transaction log. */ public class WriterDelta extends MatrixWriter { @Override public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag) - throws IOException - { - if( src.getNumRows() != rlen || src.getNumColumns() != clen ) - throw new IOException("Matrix dimensions mismatch with metadata: (" - + src.getNumRows() + "x" + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); + throws IOException { + if(src.getNumRows() != rlen || src.getNumColumns() != clen) + throw new IOException("Matrix dimensions mismatch with metadata: (" + src.getNumRows() + "x" + + src.getNumColumns() + ") vs (" + rlen + "x" + clen + ")."); int ncol = (int) clen; int nrow = (int) rlen; int batchRows = ConfigurationManager.getDeltaWriterBatchSize(); - //fast path: a contiguous dense block lets the column views read straight - //from the backing double[] (avoids per-cell MatrixBlock.get dispatch). - double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null - && src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; - //size data files adaptively (toward one file per parallel reader) for faster parallel reads. - //Delta writes every cell as a double, so size by the dense footprint rather than the (possibly - //sparse) in-memory size, which would understate the on-disk table for sparse inputs. + // fast path: a contiguous dense block lets the column views read straight + // from the backing double[] (avoids per-cell MatrixBlock.get dispatch). + double[] dense = (!src.isInSparseFormat() && src.getDenseBlock() != null && + src.getDenseBlock().isContiguous()) ? src.getDenseBlockValues() : null; + // size data files adaptively (toward one file per parallel reader) for faster parallel reads. + // Delta writes every cell as a double, so size by the dense footprint rather than the (possibly + // sparse) in-memory size, which would understate the on-disk table for sparse inputs. long estimatedBytes = (long) nrow * ncol * 8L; Engine engine = DeltaKernelUtils.createWriteEngine(estimatedBytes); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema(ncol), new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema(ncol), + new MatrixBatchIterator(src, dense, nrow, ncol, batchRows)); } @Override - public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) - throws IOException - { - //empty table: create with schema but no data files + public void writeEmptyMatrixToHDFS(String fname, long rlen, long clen, int blen) throws IOException { + // empty table: create with schema but no data files Engine engine = DeltaKernelUtils.createEngine(); - DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), - buildSchema((int) clen), CloseableIterable.emptyIterable().iterator()); + DeltaKernelUtils.commit(engine, DeltaKernelUtils.qualify(fname), buildSchema((int) clen), + CloseableIterable.emptyIterable().iterator()); } private static StructType buildSchema(int ncol) { StructType schema = new StructType(); - for( int c=0; c stream, long rlen, long clen, int blen) - throws IOException - { + public long writeMatrixFromStream(String fname, OOCStream stream, long rlen, long clen, + int blen) throws IOException { throw new UnsupportedOperationException("Out-of-core stream write is not supported for the Delta format."); } @@ -122,18 +116,18 @@ public boolean hasNext() { @Override public FilteredColumnarBatch next() { - if( !hasNext() ) + if(!hasNext()) throw new NoSuchElementException(); int size = Math.min(_batchRows, _nrow - _pos); ColumnarBatch batch = new MatrixColumnarBatch(_mb, _dense, _schema, _pos, size, _ncol); _pos += size; - //no selection vector: all rows in the batch are written + // no selection vector: all rows in the batch are written return new FilteredColumnarBatch(batch, Optional.empty()); } @Override public void close() { - //nothing to release + // nothing to release } } @@ -162,7 +156,7 @@ public StructType getSchema() { @Override public ColumnVector getColumnVector(int ordinal) { - if( ordinal < 0 || ordinal >= _ncol ) + if(ordinal < 0 || ordinal >= _ncol) throw new IndexOutOfBoundsException("column ordinal " + ordinal); return new MatrixColumnVector(_mb, _dense, _rowStart, _size, _ncol, ordinal); } @@ -208,16 +202,14 @@ public boolean isNullAt(int rowId) { @Override public double getDouble(int rowId) { - //dense contiguous single block => index fits in int (getDenseBlockValues - //is only handed over for single-block dense matrices) - return (_dense != null) - ? _dense[(_rowStart + rowId) * _ncol + _col] - : _mb.get(_rowStart + rowId, _col); + // dense contiguous single block => index fits in int (getDenseBlockValues + // is only handed over for single-block dense matrices) + return (_dense != null) ? _dense[(_rowStart + rowId) * _ncol + _col] : _mb.get(_rowStart + rowId, _col); } @Override public void close() { - //nothing to release + // nothing to release } } } diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java index 5f478979104..0bf9f7d87ba 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixReorg.java @@ -977,7 +977,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, if(ret == null) ret = new MatrixBlock(); MatrixBlock ret2 = rmemptyEarlyAbort(in, ret, rows, emptyReturn, select); - if(ret2 != null ) + if(ret2 != null) return ret2; // core removeEmpty return rmemptyUnsafe(in, ret, rows, emptyReturn, select); @@ -985,7 +985,7 @@ public static MatrixBlock rmempty(MatrixBlock in, MatrixBlock ret, boolean rows, public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select) { - if( rows ) + if(rows) return removeEmptyRows(in, ret, select, emptyReturn); else // cols return removeEmptyColumns(in, ret, select, emptyReturn); @@ -999,14 +999,15 @@ public static MatrixBlock rmemptyUnsafe(MatrixBlock in, MatrixBlock ret, boolean * @param rows If removing based on rows, or columns * @param emptyReturn Return a row/column of zeros for empty input * @param select An optional selection vector - * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. - * For the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). + * @return The early-abort result, or {@code null} if no early termination applies and the caller must continue. For + * the select-all case the returned block is the input {@code in} itself (a shallow alias, not a copy). */ - public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, MatrixBlock select){ - //check for empty inputs - //(the semantics of removeEmpty are that for an empty m-by-n matrix, the output - //is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) - if( in.isEmptyBlock(false) && select == null ) { + public static MatrixBlock rmemptyEarlyAbort(MatrixBlock in, MatrixBlock ret, boolean rows, boolean emptyReturn, + MatrixBlock select) { + // check for empty inputs + // (the semantics of removeEmpty are that for an empty m-by-n matrix, the output + // is an empty 1-by-n or m-by-1 matrix because we don't allow matrices with dims 0) + if(in.isEmptyBlock(false) && select == null) { int n = emptyReturn ? 1 : 0; if( rows ) ret.reset(n, in.clen, in.sparse); @@ -3651,7 +3652,7 @@ private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, Matr /** * Remove selected rows, based on the boolean array given. Note this function is internal use only, and require a * boolean vector to be constructed first. - * + * * @param in Input to remove rows from * @param ret Output to assign the result into * @param emptyReturn If the output is allowed to be empty. @@ -3672,9 +3673,9 @@ public static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, boole ret.reset(rlen2, n, sp); if( in.isEmptyBlock(false) ) return ret; - - if( SHALLOW_COPY_REORG && m == rlen2 && selectNull ) { - // the condition m==rlen2 is not enough with non-empty 1-row input but empty + + if(SHALLOW_COPY_REORG && m == rlen2 && selectNull) { + // the condition m==rlen2 is not enough with non-empty 1-row input but empty // 1-row select vector because if emptyReturn should output a single empty row ret.sparse = in.sparse; if( ret.sparse ) @@ -3714,10 +3715,9 @@ else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE ci++; } } - - //check sparsity - ret.nonZeros = (selectNull) ? - in.nonZeros : ret.recomputeNonZeros(); + + // check sparsity + ret.nonZeros = (selectNull) ? in.nonZeros : ret.recomputeNonZeros(); ret.examSparsity(); return ret; diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 7525dab2f7f..c450ecbe1f5 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4756,13 +4756,13 @@ public static double computeIQMCorrection(double sum, double sum_wt, } /** - * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. - * If a single column it is unweighted. - * + * Pick the quantiles out of this matrix. If this matrix contains two columns it is weighted quantile picking. If a + * single column it is unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantiles The quantiles to pick - * @param ret The result matrix + * @param ret The result matrix * @return The result matrix */ public final MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret) { @@ -4792,11 +4792,11 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av } /** - * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the - * weight column, otherwise it is unweighted over the single column. - * + * Pick the median value from this matrix. If this matrix has two columns it is weighted picking using the weight + * column, otherwise it is unweighted over the single column. + * * Note the values are assumed to be sorted. - * + * * @return The median value */ public double median() { @@ -4807,10 +4807,11 @@ public double median() { } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick * @return The quantile */ @@ -4819,12 +4820,13 @@ public final double pickValue(double quantile){ } /** - * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is unweighted. - * + * Pick a specific quantile from this matrix. If this matrix has two columns it is weighted picking, otherwise it is + * unweighted. + * * Note the values are assumed to be sorted. - * + * * @param quantile The quantile to pick - * @param average If the quantile is averaged. + * @param average If the quantile is averaged. * @return The quantile */ public final double pickValue(double quantile, boolean average) { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java index 9e3494a7d48..9ea39bd1e2f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/OOCFuture.java @@ -34,8 +34,8 @@ import java.util.function.Function; /** - * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without - * the completion-stage support of {@link java.util.concurrent.CompletableFuture}. + * Small future implementation for OOC hot paths. It supports multiple synchronous subscribers without the + * completion-stage support of {@link java.util.concurrent.CompletableFuture}. */ public class OOCFuture { private Subscriber _subscribers; @@ -240,7 +240,7 @@ private static void accept(Function mapper, Consu if(resultError == null) { try { @SuppressWarnings("unchecked") - R mapped = mapper == null ? (R)value : mapper.apply(value); + R mapped = mapper == null ? (R) value : mapper.apply(value); result = mapped; } catch(Throwable t) { @@ -345,8 +345,7 @@ public T get() throws InterruptedException, ExecutionException { } @Override - public T get(long timeout, TimeUnit unit) - throws InterruptedException, ExecutionException, TimeoutException { + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { try { return mapper.apply(source.get(timeout, unit)); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java index 94411242df1..df981f40223 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/CloseableQueue.java @@ -25,20 +25,22 @@ public class CloseableQueue { private final BlockingQueue queue = new LinkedBlockingQueue<>(); - private final Object POISON = new Object(); // sentinel + private final Object POISON = new Object(); // sentinel private volatile boolean closed = false; - public CloseableQueue() { } + public CloseableQueue() { + } /** * Enqueue if the queue is not closed. + * * @return false if already closed */ public boolean enqueueIfOpen(T task) throws InterruptedException { - if (task == null) + if(task == null) throw new IllegalArgumentException("null tasks not allowed"); - synchronized (this) { - if (closed) + synchronized(this) { + if(closed) return false; queue.put(task); } @@ -47,12 +49,12 @@ public boolean enqueueIfOpen(T task) throws InterruptedException { @SuppressWarnings("unchecked") public T take() throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.take(); - if (x == POISON) + if(x == POISON) return null; return (T) x; @@ -60,33 +62,31 @@ public T take() throws InterruptedException { /** * Poll with max timeout. - * @return item, or null if: - * - timeout, or - * - queue has been closed and this consumer reached its poison pill + * + * @return item, or null if: - timeout, or - queue has been closed and this consumer reached its poison pill */ @SuppressWarnings("unchecked") public T poll(long timeout, TimeUnit unit) throws InterruptedException { - if (closed && queue.isEmpty()) + if(closed && queue.isEmpty()) return null; Object x = queue.poll(timeout, unit); - if (x == null) - return null; // timeout + if(x == null) + return null; // timeout - if (x == POISON) + if(x == POISON) return null; return (T) x; } /** - * Close queue for N consumers. - * Each consumer will receive exactly one poison pill and then should stop. + * Close queue for N consumers. Each consumer will receive exactly one poison pill and then should stop. */ public boolean close() throws InterruptedException { - synchronized (this) { - if (closed) - return false; // idempotent + synchronized(this) { + if(closed) + return false; // idempotent closed = true; } queue.put(POISON); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java index ee02b28404c..ccc73ff6160 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataInputStream.java @@ -78,7 +78,7 @@ public void readFully(byte[] b) throws IOException { } @Override - public void readFully(byte [] b, int off, int len) throws IOException { + public void readFully(byte[] b, int off, int len) throws IOException { if(len < 0) throw new IndexOutOfBoundsException(); @@ -127,12 +127,12 @@ public int readUnsignedByte() throws IOException { @Override public short readShort() throws IOException { if(_count - _pos >= 2) { - short ret = (short)baToShort(_buff, _pos); + short ret = (short) baToShort(_buff, _pos); _pos += 2; return ret; } readFully(_tmp, 0, 2); - return (short)baToShort(_tmp, 0); + return (short) baToShort(_tmp, 0); } @Override @@ -142,7 +142,7 @@ public int readUnsignedShort() throws IOException { @Override public char readChar() throws IOException { - return (char)readUnsignedShort(); + return (char) readUnsignedShort(); } @Override @@ -222,7 +222,7 @@ public long readDoubleArray(int len, double[] varr) throws IOException { @Override public long readSparseRows(int rlen, long nnz, SparseBlock rows) throws IOException { if(rows instanceof SparseBlockCSR) { - ((SparseBlockCSR)rows).initSparse(rlen, (int)nnz, this); + ((SparseBlockCSR) rows).initSparse(rlen, (int) nnz, this); return nnz; } @@ -256,7 +256,7 @@ private void refill() throws IOException { } private int getRefillLength() { - int pageOffset = (int)(_filePos & PAGE_MASK); + int pageOffset = (int) (_filePos & PAGE_MASK); if(pageOffset == 0) return _bufflen; return Math.min(_bufflen, PAGE_SIZE - pageOffset); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java index 9854a94586b..22b7b23706c 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCBufferedDataOutputStream.java @@ -65,7 +65,7 @@ long getFlushedPosition() { public void write(int b) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)b; + _buff[_count++] = (byte) b; _position++; } @@ -109,7 +109,7 @@ public void close() throws IOException { public void writeBoolean(boolean v) throws IOException { if(_count >= _bufflen) flushBuffer(); - _buff[_count++] = (byte)(v ? 1 : 0); + _buff[_count++] = (byte) (v ? 1 : 0); _position++; } @@ -153,7 +153,7 @@ public void writeFloat(float v) throws IOException { public void writeByte(int v) throws IOException { if(_count + 1 > _bufflen) flushBuffer(); - _buff[_count++] = (byte)v; + _buff[_count++] = (byte) v; _position++; } @@ -194,18 +194,18 @@ public void writeUTF(String s) throws IOException { flushBuffer(); final char c = s.charAt(i); if(c >= 0x0001 && c <= 0x007F) { - _buff[_count++] = (byte)c; + _buff[_count++] = (byte) c; _position++; } else if(c >= 0x0800) { - _buff[_count++] = (byte)(0xE0 | ((c >> 12) & 0x0F)); - _buff[_count++] = (byte)(0x80 | ((c >> 6) & 0x3F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); + _buff[_count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 3; } else { - _buff[_count++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); - _buff[_count++] = (byte)(0x80 | (c & 0x3F)); + _buff[_count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); + _buff[_count++] = (byte) (0x80 | (c & 0x3F)); _position += 2; } } @@ -213,7 +213,7 @@ else if(c >= 0x0800) { @Override public void writeDoubleArray(int len, double[] varr) throws IOException { - for(int i = 0; i < len; ) { + for(int i = 0; i < len;) { if(_count >= _bufflen) flushBuffer(); int lblen = Math.min(len - i, (_bufflen - _count) / 8); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java index ab28df0ef0f..6c13a062561 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCIOHandler.java @@ -50,10 +50,10 @@ public interface OOCIOHandler { void registerSourceLocation(BlockKey key, SourceBlockDescriptor descriptor); /** - * Schedule an asynchronous read from an external source into the provided target stream. - * The returned future completes when either EOF is reached or the requested byte budget - * is exhausted. When the budget is reached and keepOpenOnLimit is true, the target stream - * is kept open and a continuation token is provided so the caller can resume. + * Schedule an asynchronous read from an external source into the provided target stream. The returned future + * completes when either EOF is reached or the requested byte budget is exhausted. When the budget is reached and + * keepOpenOnLimit is true, the target stream is kept open and a continuation token is provided so the caller can + * resume. */ CompletableFuture scheduleSourceRead(SourceReadRequest request); @@ -62,7 +62,8 @@ public interface OOCIOHandler { */ CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight); - interface SourceReadContinuation {} + interface SourceReadContinuation { + } class SourceReadRequest { public final String path; @@ -75,9 +76,8 @@ class SourceReadRequest { public final boolean keepOpenOnLimit; public final OOCStream target; - public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, - int blen, long estNnz, long maxBytesInFlight, boolean keepOpenOnLimit, - OOCStream target) { + public SourceReadRequest(String path, Types.FileFormat format, long rows, long cols, int blen, long estNnz, + long maxBytesInFlight, boolean keepOpenOnLimit, OOCStream target) { this.path = path; this.format = format; this.rows = rows; @@ -113,9 +113,8 @@ class SourceBlockDescriptor { public final int recordLength; public final long serializedSize; - public SourceBlockDescriptor(String path, Types.FileFormat format, - MatrixIndexes indexes, long offset, int recordLength, - long serializedSize) { + public SourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize) { this.path = path; this.format = format; this.indexes = indexes; @@ -129,8 +128,8 @@ class GroupSourceBlockDescriptor extends SourceBlockDescriptor { public final List blocks; public final int count; - public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, int recordLength, - long serializedSize, List blocks) { + public GroupSourceBlockDescriptor(String path, Types.FileFormat format, MatrixIndexes indexes, long offset, + int recordLength, long serializedSize, List blocks) { super(path, format, indexes, offset, recordLength, serializedSize); this.blocks = blocks; this.count = blocks.size(); diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java index e9487f7bd45..55ac038205f 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/OOCMatrixIOHandler.java @@ -81,7 +81,7 @@ public class OOCMatrixIOHandler implements OOCIOHandler { private final AtomicLong _readSeq = new AtomicLong(0); // Spill related structures - private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); + private final ConcurrentHashMap _spillLocations = new ConcurrentHashMap<>(); private final ConcurrentHashMap _partitions = new ConcurrentHashMap<>(); private final ConcurrentHashMap _sourceLocations = new ConcurrentHashMap<>(); private final AtomicInteger _partitionCounter = new AtomicInteger(0); @@ -97,38 +97,21 @@ public class OOCMatrixIOHandler implements OOCIOHandler { @SuppressWarnings("unchecked") public OOCMatrixIOHandler() { this._spillDir = LocalFileUtils.getUniqueWorkingDir("ooc_stream"); - _writeExec = new ThreadPoolExecutor( - WRITER_SIZE, - WRITER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _writeExec = new ThreadPoolExecutor(WRITER_SIZE, WRITER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); - _readExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, + _readExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new PriorityBlockingQueue<>()); - _srcReadExec = new ThreadPoolExecutor( - READER_SIZE, - READER_SIZE, - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(100000)); - _deleteExec = new ThreadPoolExecutor( - 1, - 1, - 0L, - TimeUnit.MILLISECONDS, + _srcReadExec = new ThreadPoolExecutor(READER_SIZE, READER_SIZE, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); + _deleteExec = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(100000)); _q = new CloseableQueue[WRITER_SIZE]; _wCtr = new AtomicLong(0); - _started = new AtomicBoolean(false); + _started = new AtomicBoolean(false); } private synchronized void start() { - if (_started.compareAndSet(false, true)) { - for (int i = 0; i < WRITER_SIZE; i++) { + if(_started.compareAndSet(false, true)) { + for(int i = 0; i < WRITER_SIZE; i++) { final int finalIdx = i; _q[i] = new CloseableQueue<>(); _writeExec.submit(() -> evictTask(_q[finalIdx])); @@ -139,7 +122,7 @@ private synchronized void start() { @Override public void shutdown() { boolean started = _started.get(); - if (started) { + if(started) { try { for(int i = 0; i < WRITER_SIZE; i++) { if(_q[i] != null) @@ -159,7 +142,7 @@ public void shutdown() { _deleteExec.shutdownNow(); _spillLocations.clear(); _partitions.clear(); - if (started) + if(started) LocalFileUtils.deleteFileIfExists(_spillDir); } @@ -169,7 +152,7 @@ public CompletableFuture scheduleEviction(BlockEntry block) { CompletableFuture future = new CompletableFuture<>(); try { long q = _wCtr.getAndAdd(block.getSize()) / OVERFLOW; - int i = (int)(q % WRITER_SIZE); + int i = (int) (q % WRITER_SIZE); if(!_q[i].enqueueIfOpen(new Tuple2<>(block, future))) future.completeExceptionally(new DMLRuntimeException("OOC writer queue is closed")); } @@ -189,7 +172,8 @@ public OOCFuture scheduleRead(final BlockEntry block) { ReadTask task = new ReadTask(block, future, _readSeq.getAndIncrement(), pinnedPartitionId); _pendingReads.put(block.getKey(), task); _readExec.execute(task); - } catch (RejectedExecutionException e) { + } + catch(RejectedExecutionException e) { unpinPartitionForRead(pinnedPartitionId); _pendingReads.remove(block.getKey()); future.completeExceptionally(e); @@ -199,12 +183,12 @@ public OOCFuture scheduleRead(final BlockEntry block) { @Override public void prioritizeRead(BlockKey key, double priority) { - if (priority == 0) + if(priority == 0) return; ReadTask task = _pendingReads.get(key); - if (task == null) + if(task == null) return; - if (_readExec.getQueue().remove(task)) { + if(_readExec.getQueue().remove(task)) { task.addPriority(priority); _readExec.getQueue().offer(task); } @@ -228,8 +212,9 @@ public CompletableFuture scheduleSourceRead(SourceReadRequest } @Override - public CompletableFuture continueSourceRead(SourceReadContinuation continuation, long maxBytesInFlight) { - if (!(continuation instanceof SourceReadState state)) { + public CompletableFuture continueSourceRead(SourceReadContinuation continuation, + long maxBytesInFlight) { + if(!(continuation instanceof SourceReadState state)) { CompletableFuture failed = new CompletableFuture<>(); failed.completeExceptionally(new DMLRuntimeException("Unsupported continuation type: " + continuation)); return failed; @@ -240,8 +225,8 @@ public CompletableFuture continueSourceRead(SourceReadContinua private CompletableFuture submitSourceRead(SourceReadRequest request, SourceReadState state, long maxBytesInFlight) { if(request.format != Types.FileFormat.BINARY) - return CompletableFuture.failedFuture( - new DMLRuntimeException("Unsupported format for source read: " + request.format)); + return CompletableFuture + .failedFuture(new DMLRuntimeException("Unsupported format for source read: " + request.format)); return readBinarySourceParallel(request, state, maxBytesInFlight); } @@ -335,17 +320,18 @@ private CompletableFuture readBinarySourceParallel(SourceReadR return result; } - private void completeResult(CompletableFuture future, AtomicLong bytesRead, AtomicBoolean budgetHit, - AtomicReference error, SourceReadRequest request, Path[] files, AtomicLongArray filePositions, - AtomicIntegerArray completed, ConcurrentLinkedDeque descriptors) { + private void completeResult(CompletableFuture future, AtomicLong bytesRead, + AtomicBoolean budgetHit, AtomicReference error, SourceReadRequest request, Path[] files, + AtomicLongArray filePositions, AtomicIntegerArray completed, + ConcurrentLinkedDeque descriptors) { Throwable err = error.get(); - if (err != null) { + if(err != null) { future.completeExceptionally(err instanceof Exception ? err : new Exception(err)); return; } try { - if (budgetHit.get()) { + if(budgetHit.get()) { if(!request.keepOpenOnLimit) { closeTarget(request.target, false); } @@ -365,29 +351,29 @@ private void completeResult(CompletableFuture future, AtomicLo private void readSequenceFile(JobConf job, Path path, SourceReadRequest request, int fileIdx, AtomicLongArray filePositions, AtomicIntegerArray completed, AtomicBoolean stop, AtomicBoolean budgetHit, - AtomicLong bytesRead, long byteLimit, Object budgetLock, ConcurrentLinkedDeque descriptors) - throws IOException { + AtomicLong bytesRead, long byteLimit, Object budgetLock, + ConcurrentLinkedDeque descriptors) throws IOException { MatrixIndexes key = new MatrixIndexes(); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { long pos = filePositions.get(fileIdx); - if (pos > 0) + if(pos > 0) reader.seek(pos); long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; while(!stop.get()) { long recordStart = reader.getPosition(); MatrixBlock value = new MatrixBlock(); - if (!reader.next(key, value)) + if(!reader.next(key, value)) break; long recordEnd = reader.getPosition(); long blockSize = value.getExactSerializedSize(); boolean shouldBreak = false; synchronized(budgetLock) { - if (stop.get()) + if(stop.get()) shouldBreak = true; - else if (bytesRead.get() + blockSize > byteLimit) { + else if(bytesRead.get() + blockSize > byteLimit) { stop.set(true); budgetHit.set(true); shouldBreak = true; @@ -398,7 +384,7 @@ else if (bytesRead.get() + blockSize > byteLimit) { MatrixIndexes outIdx = new MatrixIndexes(key); IndexedMatrixValue imv = new IndexedMatrixValue(outIdx, value); SourceBlockDescriptor descriptor = new SourceBlockDescriptor(path.toString(), request.format, outIdx, - recordStart, (int)(recordEnd - recordStart), blockSize); + recordStart, (int) (recordEnd - recordStart), blockSize); if(request.target instanceof SourceOOCStream src) src.enqueue(imv, descriptor); @@ -407,22 +393,23 @@ else if (bytesRead.get() + blockSize > byteLimit) { descriptors.add(descriptor); filePositions.set(fileIdx, reader.getPosition()); - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { long currTime = System.nanoTime(); OOCEventLog.onDiskReadEvent(_srcReadCallerId, ioStart, currTime, blockSize); ioStart = currTime; } - if (shouldBreak) + if(shouldBreak) break; // Note that we knowingly go over limit, which could result in READER_SIZE*8MB overshoot } - if (!stop.get()) + if(!stop.get()) completed.set(fileIdx, 1); } } - private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, boolean close) { + private void closeTarget(org.apache.sysds.runtime.instructions.ooc.OOCStream target, + boolean close) { if(close) { try { target.closeInput(); @@ -437,10 +424,10 @@ private void loadFromDisk(BlockEntry block) { String key = block.getKey().toFileKey(); SourceBlockDescriptor src = _sourceLocations.get(block.getKey()); - if (src != null) { + if(src != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; loadFromSource(block, src); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(System.nanoTime() - ioStart); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -451,34 +438,36 @@ private void loadFromDisk(BlockEntry block) { long ioDuration = 0; // 1. find the blocks address (spill location) SpillLocation sloc = _spillLocations.get(key); - if (sloc == null) + if(sloc == null) throw new DMLRuntimeException("Failed to load spill location for: " + key); PartitionFile partFile = _partitions.get(sloc.partitionId); - if (partFile == null) + if(partFile == null) throw new DMLRuntimeException("Failed to load partition for: " + sloc.partitionId); String filename = partFile.filePath; SpillableObject obj; - try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { + try(RandomAccessFile raf = new RandomAccessFile(filename, "r")) { raf.seek(sloc.offset); DataInput dis = new OOCBufferedDataInputStream(raf); long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; obj = SpillableObjectRegistry.read(dis); - if (DMLScript.OOC_STATISTICS) + if(DMLScript.OOC_STATISTICS) ioDuration = System.nanoTime() - ioStart; - } catch (ClosedByInterruptException ignored) { + } + catch(ClosedByInterruptException ignored) { return; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } block.setDataUnsafe(obj); - if (DMLScript.OOC_STATISTICS) { + if(DMLScript.OOC_STATISTICS) { Statistics.incrementOOCLoadFromDisk(); Statistics.accumulateOOCLoadFromDiskTime(ioDuration); Statistics.accumulateOOCLoadFromDiskBytes(block.getSize()); @@ -486,22 +475,23 @@ private void loadFromDisk(BlockEntry block) { } private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { - if (src.format != Types.FileFormat.BINARY) + if(src.format != Types.FileFormat.BINARY) throw new DMLRuntimeException("Unsupported format for source read: " + src.format); JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); Path path = new Path(src.path); - if (src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { + if(src instanceof OOCIOHandler.GroupSourceBlockDescriptor gsrc) { List values = new ArrayList<>(gsrc.count); try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(gsrc.offset); - for (int i = 0; i < gsrc.blocks.size(); i++) { + for(int i = 0; i < gsrc.blocks.size(); i++) { SourceBlockDescriptor d = gsrc.blocks.get(i); MatrixIndexes ix = new MatrixIndexes(); MatrixBlock mb = new MatrixBlock(); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + d.offset + " in " + d.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + d.offset + " in " + d.path); values.add(new IndexedMatrixValue(ix, mb)); } } @@ -516,8 +506,9 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { try(SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path))) { reader.seek(src.offset); - if (!reader.next(ix, mb)) - throw new DMLRuntimeException("Failed to read source block at offset " + src.offset + " in " + src.path); + if(!reader.next(ix, mb)) + throw new DMLRuntimeException( + "Failed to read source block at offset " + src.offset + " in " + src.path); } catch(IOException e) { throw new DMLRuntimeException(e); @@ -530,7 +521,7 @@ private void loadFromSource(BlockEntry block, SourceBlockDescriptor src) { private void evictTask(CloseableQueue>> q) { long byteCtr = 0; - while (!q.isFinished()) { + while(!q.isFinished()) { // --- 1. WRITE PHASE --- int partitionId = _partitionCounter.getAndIncrement(); @@ -572,17 +563,17 @@ private void evictTask(CloseableQueue } byteCtr += wrote; - if (byteCtr >= MAX_PARTITION_SIZE) { + if(byteCtr >= MAX_PARTITION_SIZE) { closePartition = true; byteCtr = 0; break; } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } - if (!closePartition && q.close()) { + if(!closePartition && q.close()) { while((tpl = q.take()) != null) { long ioStart = DMLScript.OOC_STATISTICS ? System.nanoTime() : 0; BlockEntry entry = tpl._1(); @@ -595,7 +586,7 @@ private void evictTask(CloseableQueue Statistics.accumulateOOCEvictionWriteTime(System.nanoTime() - ioStart); } - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskWriteEvent(_evictCallerId, ioStart, System.nanoTime(), wrote); } } @@ -617,13 +608,13 @@ private void evictTask(CloseableQueue } private long writeOut(int partitionId, BlockEntry entry, CompletableFuture future, - OOCBufferedDataOutputStream dos, - ConcurrentLinkedDeque>> flushQueue) throws IOException { + OOCBufferedDataOutputStream dos, ConcurrentLinkedDeque>> flushQueue) + throws IOException { String key = entry.getKey().toFileKey(); boolean alreadySpilled = _spillLocations.containsKey(key); - if (!alreadySpilled) { + if(!alreadySpilled) { long offsetBefore = dos.getPosition(); if(future.isCancelled()) @@ -656,9 +647,10 @@ private long writeOut(int partitionId, BlockEntry entry, CompletableFuture return 0; } - private void flushQueue(long offset, ConcurrentLinkedDeque>> flushQueue) { + private void flushQueue(long offset, + ConcurrentLinkedDeque>> flushQueue) { Tuple3> tmp; - while ((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { + while((tmp = flushQueue.peek()) != null && tmp._2() <= offset) { flushQueue.poll(); tmp._3().complete(null); } @@ -777,12 +769,14 @@ public void run() { try { long ioStart = DMLScript.OOC_LOG_EVENTS ? System.nanoTime() : 0; loadFromDisk(_block); - if (DMLScript.OOC_LOG_EVENTS) + if(DMLScript.OOC_LOG_EVENTS) OOCEventLog.onDiskReadEvent(_readCallerId, ioStart, System.nanoTime(), _block.getSize()); _future.complete(_block); - } catch (Throwable e) { + } + catch(Throwable e) { _future.completeExceptionally(e); - } finally { + } + finally { unpinPartitionForRead(_pinnedPartitionId); } } @@ -790,15 +784,12 @@ public void run() { @Override public int compareTo(ReadTask other) { int byPriority = Double.compare(other._priority, _priority); - if (byPriority != 0) + if(byPriority != 0) return byPriority; return Long.compare(_sequence, other._sequence); } } - - - private static class SpillLocation { // structure of spillLocation: file, offset final int partitionId; diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java index a93b1d18f2a..3458838c071 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/io/SpillableObject.java @@ -24,8 +24,10 @@ import java.io.IOException; public interface SpillableObject { - boolean tryWrite(DataOutput out) throws IOException; - void read(DataInput in) throws IOException; + boolean tryWrite(DataOutput out) throws IOException; + + void read(DataInput in) throws IOException; + long size(); default void discard() { diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java index d8ff79dd797..0d318d83d94 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCCacheScheduler.java @@ -34,14 +34,16 @@ public interface OOCCacheScheduler { /** * Requests a single block from the cache. + * * @param key the requested key associated to the block * @return the available BlockEntry */ CompletableFuture request(BlockKey key); /** - * Tries to request a single block from the cache. - * Immediately returns the entry if present, otherwise null without scheduling reads. + * Tries to request a single block from the cache. Immediately returns the entry if present, otherwise null without + * scheduling reads. + * * @param key the requested key associated to the block * @return the available BlockEntry or null */ @@ -52,14 +54,16 @@ default BlockEntry tryRequest(BlockKey key) { /** * Requests a list of blocks from the cache that must be available at the same time. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ CompletableFuture> request(List keys); /** - * Tries to request a list of blocks from the cache that must be available at the same time. - * Immediately returns the list of entries if present, otherwise null without scheduling reads. + * Tries to request a list of blocks from the cache that must be available at the same time. Immediately returns the + * list of entries if present, otherwise null without scheduling reads. + * * @param keys the requested keys associated to the block * @return the list of available BlockEntries */ @@ -76,24 +80,25 @@ default BlockEntry tryRequest(BlockKey key) { List tryRequestAnyOf(List keys, int n, List selectionOut); /** - * Adds the given priority to any pending request accessing the key. - * Multi-requests are prioritized partially. + * Adds the given priority to any pending request accessing the key. Multi-requests are prioritized partially. */ void prioritize(BlockKey key, double priority); /** - * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. - * The object data should now only be accessed via cache, as ownership has been transferred. - * @param key the associated key of the block + * Places a new block in the cache. Note that objects are immutable and cannot be overwritten. The object data + * should now only be accessed via cache, as ownership has been transferred. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ BlockKey put(BlockKey key, Object data, long size); /** - * Places a new block in the cache and returns a pinned handle. - * Note that objects are immutable and cannot be overwritten. - * @param key the associated key of the block + * Places a new block in the cache and returns a pinned handle. Note that objects are immutable and cannot be + * overwritten. + * + * @param key the associated key of the block * @param data the block data * @param size the size of the data */ @@ -101,8 +106,11 @@ default BlockEntry tryRequest(BlockKey key) { interface HandoverHandle { BlockKey getKey(); + boolean isCommitted(); + CompletableFuture getCompletionFuture(); + OOCStream.QueueCallback reclaim(); } @@ -131,27 +139,30 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor); /** - * Notifies the cache that there is another reference to the same block key. - * This will prevent forget(key) from removing the block from cache. - * A block will only be forgotten after all referencing instances called forget(key). + * Notifies the cache that there is another reference to the same block key. This will prevent forget(key) from + * removing the block from cache. A block will only be forgotten after all referencing instances called forget(key). + * * @param key */ void addReference(BlockKey key); /** * Forgets a block from the cache. + * * @param key the associated key of the block */ void forget(BlockKey key); /** * Pins a BlockEntry in cache to prevent eviction. + * * @param entry the entry to be pinned */ void pin(BlockEntry entry); /** * Unpins a pinned block. + * * @param entry the entry to be unpinned */ void unpin(BlockEntry entry); @@ -192,8 +203,7 @@ BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, void updateLimits(long evictionLimit, long hardLimit); /** - * Creates a snapshot of the cache. - * Should only be used for debugging or diagnoses. + * Creates a snapshot of the cache. Should only be used for debugging or diagnoses. */ Collection snapshot(); } diff --git a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java index 96de368ccc9..9b0acc97b35 100644 --- a/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java +++ b/src/main/java/org/apache/sysds/runtime/ooc/cache/legacy/OOCLRUCacheScheduler.java @@ -80,7 +80,7 @@ public class OOCLRUCacheScheduler implements OOCCacheScheduler { public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long hardLimit, long readBuffer) { this._ioHandler = ioHandler; this._cache = new LinkedHashMap<>(1024, 0.75f, true); - this._evictionCache = new HashMap<>(); + this._evictionCache = new HashMap<>(); this._deferredReadRequests = new DeferredReadQueue(); this._processingReadRequests = new ArrayDeque<>(); this._pendingHandovers = new ArrayDeque<>(); @@ -103,7 +103,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har this._maintenanceNeedsIncr = new AtomicBoolean(false); this._callerId = DMLScript.OOC_LOG_EVENTS ? OOCEventLog.registerCaller("LRUCacheScheduler") : 0; - if (DMLScript.OOC_LOG_EVENTS) { + if(DMLScript.OOC_LOG_EVENTS) { OOCEventLog.putRunSetting("CacheEvictionLimit", _evictionLimit); OOCEventLog.putRunSetting("CacheHardLimit", _hardLimit); } @@ -111,7 +111,7 @@ public OOCLRUCacheScheduler(OOCIOHandler ioHandler, long evictionLimit, long har @Override public CompletableFuture request(BlockKey key) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(); @@ -120,21 +120,21 @@ public CompletableFuture request(BlockKey key) { boolean couldPin = false; synchronized(this) { entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { - if (entry.getState().isAvailable()) { - if (pinEntryWithAccounting(entry) == 0) + if(entry.getState().isAvailable()) { + if(pinEntryWithAccounting(entry) == 0) throw new IllegalStateException(); couldPin = true; } } } - if (couldPin) { + if(couldPin) { // Then we could pin the required entry and can terminate return CompletableFuture.completedFuture(entry); } @@ -184,7 +184,7 @@ public CompletableFuture> request(List keys) { } public CompletableFuture> request(List keys, boolean onlyIfAvailable) { - if (!this._running) + if(!this._running) throw new IllegalStateException("Cache scheduler has been shut down."); Statistics.incrementOOCEvictionGet(keys.size()); @@ -193,11 +193,11 @@ public CompletableFuture> request(List keys, boolean boolean allAvailable = true; synchronized(this) { - for (BlockKey key : keys) { + for(BlockKey key : keys) { BlockEntry entry = _cache.get(key); - if (entry == null) + if(entry == null) entry = _evictionCache.get(key); - if (entry == null) + if(entry == null) throw new IllegalArgumentException("Could not find requested block with key " + key); synchronized(entry) { @@ -217,7 +217,7 @@ public CompletableFuture> request(List keys, boolean } } - if (allAvailable) { + if(allAvailable) { // Then we could pin all entries return CompletableFuture.completedFuture(entries); } @@ -226,12 +226,12 @@ public CompletableFuture> request(List keys, boolean return null; // Schedule deferred read otherwise - final CompletableFuture> future = new CompletableFuture<>(); + final CompletableFuture> future = new CompletableFuture<>(); DeferredReadRequest request = new DeferredReadRequest(future, entries); - for (int i = 0; i < entries.size(); i++) { + for(int i = 0; i < entries.size(); i++) { BlockEntry entry = entries.get(i); synchronized(entry) { - if (entry.getState().isAvailable()) { + if(entry.getState().isAvailable()) { entry.addRetainHint(); request.markRetainHinted(i); } @@ -243,9 +243,9 @@ public CompletableFuture> request(List keys, boolean @Override public void prioritize(BlockKey key, double priority) { - if (!this._running) + if(!this._running) return; - if (priority == 0) + if(priority == 0) return; synchronized(this) { @@ -267,18 +267,18 @@ private void scheduleDeferredRead(DeferredReadRequest deferredReadRequest) { if(entry.getState().isAvailable()) readyCount++; BlockReadState state = _blockReads.get(entry.getKey()); - if (state != null) + if(state != null) score += state.priority; } - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score /= deferredReadRequest.getEntries().size(); - if (!deferredReadRequest.getEntries().isEmpty()) + if(!deferredReadRequest.getEntries().isEmpty()) score += ((double) readyCount) / deferredReadRequest.getEntries().size(); deferredReadRequest.setPriorityScore(score); _deferredReadRequests.add(deferredReadRequest); _deferredReadCountHint = _deferredReadRequests.size(); } - onCacheSizeChanged(true); // Apply pressure from deferred read demand. + onCacheSizeChanged(true); // Apply pressure from deferred read demand. onCacheSizeChanged(false); // Attempt to schedule deferred reads. } @@ -317,7 +317,8 @@ public void putSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.S } @Override - public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, OOCIOHandler.SourceBlockDescriptor descriptor) { + public BlockEntry putAndPinSourceBacked(BlockKey key, Object data, long size, + OOCIOHandler.SourceBlockDescriptor descriptor) { return put(key, data, size, true, descriptor); } @@ -333,23 +334,24 @@ public void addReference(BlockKey key) { } } - private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOHandler.SourceBlockDescriptor descriptor) { - if (!this._running) + private BlockEntry put(BlockKey key, Object data, long size, boolean pin, + OOCIOHandler.SourceBlockDescriptor descriptor) { + if(!this._running) throw new IllegalStateException(); - if (data == null) + if(data == null) throw new IllegalArgumentException(); - if (descriptor != null) + if(descriptor != null) _ioHandler.registerSourceLocation(key, descriptor); Statistics.incrementOOCEvictionPut(); BlockEntry entry = new BlockEntry(key, size, data); - if (descriptor != null) + if(descriptor != null) entry.setState(BlockState.WARM); - if (pin) + if(pin) entry.pin(); synchronized(this) { BlockEntry avail = _cache.putIfAbsent(key, entry); - if (avail != null || _evictionCache.containsKey(key)) + if(avail != null || _evictionCache.containsKey(key)) throw new IllegalStateException("Cannot overwrite existing entries: " + key); _cacheSize += size; if(pin) { @@ -364,7 +366,7 @@ private BlockEntry put(BlockKey key, Object data, long size, boolean pin, OOCIOH @Override public void forget(BlockKey key) { - if (!this._running) + if(!this._running) return; final MutableObject mEntry = new MutableObject<>(); BlockEntry entry; @@ -381,7 +383,7 @@ public void forget(BlockKey key) { return e; }); - if (mEntry.getValue() == null) { + if(mEntry.getValue() == null) { _evictionCache.compute(key, (k, e) -> { if(e == null) return null; @@ -395,10 +397,10 @@ public void forget(BlockKey key) { entry = mEntry.getValue(); - if (entry != null) { + if(entry != null) { synchronized(entry) { - shouldScheduleDeletion = entry.getState().isBackedByDisk() - || entry.getState() == BlockState.EVICTING; + shouldScheduleDeletion = entry.getState().isBackedByDisk() || + entry.getState() == BlockState.EVICTING; cacheSizeDelta = transitionMemState(entry, BlockState.REMOVED); if(entry.isPinned() && entry.getDataUnsafe() != null) _pinnedBytes -= entry.getSize(); @@ -408,9 +410,9 @@ public void forget(BlockKey key) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - if (shouldScheduleDeletion) + if(shouldScheduleDeletion) _ioHandler.scheduleDeletion(entry); } @@ -424,11 +426,11 @@ public void pin(BlockEntry entry) { synchronized(this) { synchronized(entry) { int pinCount = pinEntryWithAccounting(entry); - if (pinCount == 0) + if(pinCount == 0) throw new IllegalStateException("Could not pin the requested entry: " + entry.getKey()); } // Access element in cache for Lru - //_cache.get(entry.getKey()); + // _cache.get(entry.getKey()); } } @@ -442,26 +444,26 @@ public void unpin(BlockEntry entry) { synchronized(entry) { if(!unpinEntryWithAccounting(entry)) return; - if (_cacheSize <= _evictionLimit) + if(_cacheSize <= _evictionLimit) return; // Nothing to do - if (entry.isPinned()) + if(entry.isPinned()) return; // Pin state changed so we cannot evict - if (entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { - if (entry.getRetainHintCount() > 0) { + if(entry.getState().isAvailable() && entry.getState().isBackedByDisk()) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { - cacheSizeDelta = transitionMemState(entry, BlockState.COLD); + cacheSizeDelta = transitionMemState(entry, BlockState.COLD); long cleared = entry.clear(); - if (cleared != entry.getSize()) + if(cleared != entry.getSize()) throw new IllegalStateException(); _cache.remove(entry.getKey()); _evictionCache.put(entry.getKey(), entry); } } - else if (entry.getState() == BlockState.HOT) { - if (entry.getRetainHintCount() > 0) { + else if(entry.getState() == BlockState.HOT) { + if(entry.getRetainHintCount() > 0) { shouldCheckEviction = true; } else { @@ -470,9 +472,9 @@ else if (entry.getState() == BlockState.HOT) { } } } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); - else if (shouldCheckEviction) + else if(shouldCheckEviction) onCacheSizeChanged(true); } @@ -508,10 +510,11 @@ public synchronized void shutdown() { System.out.println("[WARN] Cache still holds " + _cache.size() + " / " + _evictionCache.size() + " blocks"); Set cachedStreams = _cache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); - Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId).collect(Collectors.toSet()); + Set evictedStreams = _evictionCache.keySet().stream().map(BlockKey::getStreamId) + .collect(Collectors.toSet()); cachedStreams.addAll(evictedStreams); - System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + _cache.values().stream().mapToInt( - e -> e.isPinned() ? 1 : 0).sum()); + System.out.println("[WARN] Affected stream IDs: " + cachedStreams + ", Pinned: " + + _cache.values().stream().mapToInt(e -> e.isPinned() ? 1 : 0).sum()); } _cache.clear(); _evictionCache.clear(); @@ -575,7 +578,8 @@ private void runMaintenanceLoop() { do { _maintenanceRequested.set(false); onCacheSizeChangedInternal(_maintenanceNeedsIncr.getAndSet(false)); - } while(_maintenanceRequested.get()); + } + while(_maintenanceRequested.get()); } finally { _maintenanceRunning.set(false); @@ -591,7 +595,8 @@ private void onCacheSizeChangedInternal(boolean incr) { if(incr) onCacheSizeIncremented(); else - while(onCacheSizeDecremented()) {} + while(onCacheSizeDecremented()) { + } while(processPendingHandovers()) { onCacheSizeIncremented(); } @@ -601,18 +606,23 @@ private void onCacheSizeChangedInternal(boolean incr) { } private synchronized void sanityCheck() { - if (_cacheSize > _hardLimit * 1.1) { - if (!_warnThrottling) { + if(_cacheSize > _hardLimit * 1.1) { + if(!_warnThrottling) { _warnThrottling = true; - System.out.println("[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) > " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println( + "[WARN] Cache hard limit exceeded by over 10%: " + String.format("%.2f", _cacheSize / 1000000.0) + + "MB (-" + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) > " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } } - else if (_warnThrottling && _cacheSize < _hardLimit) { + else if(_warnThrottling && _cacheSize < _hardLimit) { _warnThrottling = false; - System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize/1000000.0) + "MB (-" + String.format("%.2f", _bytesUpForEviction/1000000.0) + "MB) <= " + String.format("%.2f", _hardLimit/1000000.0) + "MB"); + System.out.println("[INFO] Cache within limit: " + String.format("%.2f", _cacheSize / 1000000.0) + "MB (-" + + String.format("%.2f", _bytesUpForEviction / 1000000.0) + "MB) <= " + + String.format("%.2f", _hardLimit / 1000000.0) + "MB"); } - if (!SANITY_CHECKS) + if(!SANITY_CHECKS) return; int pinned = 0; @@ -625,16 +635,16 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { long actualPinnedEvictingBytes = 0; long actualWarmPinnedBytes = 0; long actualReadingReservedBytes = 0; - for (BlockEntry entry : _cache.values()) { - if (entry.isPinned()) { + for(BlockEntry entry : _cache.values()) { + if(entry.isPinned()) { pinned++; actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } - if (entry.getState().isBackedByDisk()) + if(entry.getState().isBackedByDisk()) backedByDisk++; - if (entry.getState() == BlockState.EVICTING) { + if(entry.getState() == BlockState.EVICTING) { evicting++; upForEviction += entry.getSize(); if(entry.isPinned()) @@ -642,43 +652,44 @@ else if (_warnThrottling && _cacheSize < _hardLimit) { } if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (!entry.getState().isAvailable()) + if(!entry.getState().isAvailable()) throw new IllegalStateException(); total++; actualCacheSize += entry.getSize(); } - for (BlockEntry entry : _evictionCache.values()) { - if (entry.getState().isAvailable()) + for(BlockEntry entry : _evictionCache.values()) { + if(entry.getState().isAvailable()) throw new IllegalStateException("Invalid eviction state: " + entry.getState()); - if (entry.getState() == BlockState.EVICTING && entry.isPinned()) + if(entry.getState() == BlockState.EVICTING && entry.isPinned()) actualPinnedEvictingBytes += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualCacheSize += entry.getSize(); - if (entry.getState() == BlockState.READING) + if(entry.getState() == BlockState.READING) actualReadingReservedBytes += entry.getSize(); - if (entry.isPinned()) { + if(entry.isPinned()) { actualPinnedBytes += entry.getSize(); if(entry.getState() == BlockState.WARM) actualWarmPinnedBytes += entry.getSize(); } } - if (actualCacheSize != _cacheSize) + if(actualCacheSize != _cacheSize) throw new IllegalStateException(actualCacheSize + " != " + _cacheSize); - if (upForEviction != _bytesUpForEviction) + if(upForEviction != _bytesUpForEviction) throw new IllegalStateException(upForEviction + " != " + _bytesUpForEviction); - if (actualPinnedBytes != _pinnedBytes) + if(actualPinnedBytes != _pinnedBytes) throw new IllegalStateException(actualPinnedBytes + " != " + _pinnedBytes); - if (actualPinnedEvictingBytes != _pinnedEvictingBytes) + if(actualPinnedEvictingBytes != _pinnedEvictingBytes) throw new IllegalStateException(actualPinnedEvictingBytes + " != " + _pinnedEvictingBytes); - if (_pinnedEvictingBytes > _bytesUpForEviction) + if(_pinnedEvictingBytes > _bytesUpForEviction) throw new IllegalStateException(_pinnedEvictingBytes + " > " + _bytesUpForEviction); if(actualWarmPinnedBytes != _warmPinnedBytes) throw new IllegalStateException(actualWarmPinnedBytes + " != " + _warmPinnedBytes); - if (actualReadingReservedBytes != _readingReservedBytes) + if(actualReadingReservedBytes != _readingReservedBytes) throw new IllegalStateException(actualReadingReservedBytes + " != " + _readingReservedBytes); System.out.println("=========="); - System.out.println("Limit: " + _evictionLimit/1000 + "KB"); - System.out.println("Memory: (" + _cacheSize/1000 + "KB - " + _bytesUpForEviction/1000 + "KB) / " + _hardLimit/1000 + "KB"); + System.out.println("Limit: " + _evictionLimit / 1000 + "KB"); + System.out.println("Memory: (" + _cacheSize / 1000 + "KB - " + _bytesUpForEviction / 1000 + "KB) / " + + _hardLimit / 1000 + "KB"); System.out.println("Pinned: " + pinned + " / " + total); System.out.println("Disk backed: " + backedByDisk + " / " + total); System.out.println("Evicting: " + evicting + " / " + total); @@ -695,10 +706,11 @@ private void onCacheSizeIncremented() { if(pressure <= _evictionLimit) return; // Nothing to do - long overshoot = Math.max((long)(0.1 * _evictionLimit), 10000000); + long overshoot = Math.max((long) (0.1 * _evictionLimit), 10000000); long lowLimit = _evictionLimit - _readBuffer - overshoot; - //System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); + // System.out.println("[CACHE] Claiming " + (pressure + overshoot - _evictionLimit)/1000 + "kB (last claim + // was " + (System.currentTimeMillis() - _lastEvictRun) + "ms ago)"); // Scan for values that can be evicted Collection entries = _cache.values(); @@ -713,8 +725,8 @@ private void onCacheSizeIncremented() { break; synchronized(entry) { - //if(entry.isPinned()) - // continue; + // if(entry.isPinned()) + // continue; if(!allowRetainHint && entry.getRetainHintCount() > 0) continue; if(entry.getState() == BlockState.COLD || entry.getState() == BlockState.EVICTING) @@ -748,12 +760,12 @@ private void onCacheSizeIncremented() { _lastEvictRun = System.currentTimeMillis(); } - for (BlockEntry entry : upForEvictionNeedsWrite) + for(BlockEntry entry : upForEvictionNeedsWrite) evict(entry, true); - for (BlockEntry entry : upForEvictionNoWrite) + for(BlockEntry entry : upForEvictionNoWrite) evict(entry, false); - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } @@ -813,7 +825,7 @@ private boolean onCacheSizeDecremented() { throw new IllegalStateException(); req.setPinned(idx); } - else if (entry.getState() == BlockState.READING) { + else if(entry.getState() == BlockState.READING) { req.schedule(idx); registerWaiter(entry.getKey(), req, idx); reading = true; @@ -836,7 +848,7 @@ else if (entry.getState() == BlockState.READING) { if(allReserved) { _deferredReadRequests.poll(); _deferredReadCountHint = _deferredReadRequests.size(); - if (!toRead.isEmpty()) + if(!toRead.isEmpty()) _processingReadRequests.add(req); } @@ -943,17 +955,17 @@ private void onEvicted(final BlockEntry entry) { if(tmp != null && tmp != entry) throw new IllegalStateException(); tmp = _evictionCache.put(entry.getKey(), entry); - if (tmp != null) + if(tmp != null) throw new IllegalStateException(); sanityCheck(); } - if (cacheSizeDelta != 0) + if(cacheSizeDelta != 0) onCacheSizeChanged(cacheSizeDelta > 0); } private void clearRetainHints(DeferredReadRequest request) { - for (int i = 0; i < request.getEntries().size(); i++) { - if (!request.isRetainHinted(i)) + for(int i = 0; i < request.getEntries().size(); i++) { + if(!request.isRetainHinted(i)) continue; BlockEntry entry = request.getEntries().get(i); synchronized(entry) { @@ -963,12 +975,12 @@ private void clearRetainHints(DeferredReadRequest request) { } /** - * Cleanly transitions state of a BlockEntry and handles accounting. - * Requires both the scheduler object and the entry to be locked: + * Cleanly transitions state of a BlockEntry and handles accounting. Requires both the scheduler object and the + * entry to be locked: */ private long transitionMemState(BlockEntry entry, BlockState newState) { BlockState oldState = entry.getState(); - if (oldState == newState) + if(oldState == newState) return 0; long sz = entry.getSize(); @@ -976,7 +988,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { boolean pinned = entry.isPinned(); // Remove old contribution - switch (oldState) { + switch(oldState) { case REMOVED: throw new IllegalStateException(); case HOT: @@ -1000,7 +1012,7 @@ private long transitionMemState(BlockEntry entry, BlockState newState) { } // Add new contribution - switch (newState) { + switch(newState) { case REMOVED: case COLD: break; @@ -1056,6 +1068,7 @@ private int pinEntryWithAccounting(BlockEntry entry) { /** * Requires scheduler lock and entry lock. + * * @return true if this call transitioned pin count to zero. */ private boolean unpinEntryWithAccounting(BlockEntry entry) { diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java index 1f731fc3aa5..f04534481c9 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/Decoder.java @@ -66,8 +66,8 @@ protected boolean isHashCol(int colID) { } /** - * Domain size of a dummycoded source column: the hash domain K from the meta cell for - * feature-hashed columns, otherwise the column's {@code numDistinct} (0 when unset). + * Domain size of a dummycoded source column: the hash domain K from the meta cell for feature-hashed columns, + * otherwise the column's {@code numDistinct} (0 when unset). * * @param meta transform meta frame * @param colID 1-based column id of the dummycoded source column @@ -144,13 +144,13 @@ public FrameBlock decode(final MatrixBlock in, final FrameBlock out, final int k try { final List> tasks = new ArrayList<>(); int blz = Math.max((in.getNumRows() + k) / k, 1000); - - for(int i = 0; i < in.getNumRows(); i += blz){ + + for(int i = 0; i < in.getNumRows(); i += blz) { final int start = i; final int end = Math.min(in.getNumRows(), i + blz); tasks.add(pool.submit(() -> decode(in, out, start, end))); } - + for(Future f : tasks) f.get(); return out; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java index a286c03dce8..01a502154cd 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderBin.java @@ -72,10 +72,10 @@ public void decode(MatrixBlock in, FrameBlock out, int rl, int ru) { final double val = in.get(i, _srcCols[j] - 1); if(!Double.isNaN(val)){ final int key = (int) Math.round(val); - if(key == 0){ + if(key == 0) { a.set(i, _binMins[j][key]); } - else{ + else { double bmin = _binMins[j][key - 1]; double bmax = _binMaxs[j][key - 1]; double oval = bmin + (bmax - bmin) / 2 // bin center diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java index ee1a33c49fd..8aa0b60d990 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderDummycode.java @@ -34,7 +34,7 @@ /** * Simple atomic decoder for dummycoded columns. This decoder builds internally inverted column mappings from the given * frame meta data. - * + * */ public class DecoderDummycode extends Decoder { private static final long serialVersionUID = 4758831042891032129L; diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java index 8f6c45d63e8..fc123657032 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderFactory.java @@ -64,15 +64,15 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] try { //parse transform specification JSONObject jSpec = new JSONObject(spec); - - //create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' + + // create decoders 'bin', 'recode', 'hash', 'dummy', and 'pass-through' List binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames, minCol, maxCol); - List rcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); - List hcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); - List dcIDs = Arrays.asList(ArrayUtils.toObject( - TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); + List rcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString(), minCol, maxCol))); + List hcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString(), minCol, maxCol))); + List dcIDs = Arrays.asList(ArrayUtils + .toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString(), minCol, maxCol))); // only specially treat the columns with both recode and dictionary rcIDs = unionDistinct(rcIDs, dcIDs); // hashing is a lossy, one-way transform with no inverse recode map, so hash columns @@ -106,29 +106,18 @@ public static Decoder createDecoder(String spec, String[] colnames, ValueType[] // collect all the decoders in one list. List ldecoders = new ArrayList<>(); - - if( !binIDs.isEmpty() ) { - ldecoders.add(new DecoderBin(schema, - ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), + + if(!binIDs.isEmpty()) { + ldecoders.add(new DecoderBin(schema, ArrayUtils.toPrimitive(binIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } if( !dcIDs.isEmpty() ) { ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); } - if( !rcIDs.isEmpty() ) { - // recode on output (after dummycode rebuilds the categorical columns) when dummycoding is present - ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), - ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])))); - } - if( !ptIDs.isEmpty() ) { - ldecoders.add(new DecoderPassThrough(schema, - ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), - ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0])), hashCols)); - } - - //create composite decoder of all created decoders - //and initialize with given meta data (recode, dummy, bin) + + // create composite decoder of all created decoders + // and initialize with given meta data (recode, dummy, bin) decoder = new DecoderComposite(schema, ldecoders); decoder.setColnames(colnames); decoder.initMetaData(meta); @@ -147,7 +136,7 @@ else if( decoder instanceof DecoderRecode ) return DecoderType.Recode.ordinal(); else if( decoder instanceof DecoderPassThrough ) return DecoderType.PassThrough.ordinal(); - else if( decoder instanceof DecoderBin ) + else if(decoder instanceof DecoderBin) return DecoderType.Bin.ordinal(); throw new DMLRuntimeException("Unsupported decoder type: " + decoder.getClass().getCanonicalName()); @@ -158,10 +147,14 @@ public static Decoder createInstance(int type) { // create instance switch(dtype) { - case Bin: return new DecoderBin(); - case Dummycode: return new DecoderDummycode(null, null); - case PassThrough: return new DecoderPassThrough(null, null, null); - case Recode: return new DecoderRecode(null, false, null); + case Bin: + return new DecoderBin(); + case Dummycode: + return new DecoderDummycode(null, null); + case PassThrough: + return new DecoderPassThrough(null, null, null); + case Recode: + return new DecoderRecode(null, false, null); default: throw new DMLRuntimeException("Unsupported Encoder Type used: " + dtype); } diff --git a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java index 11dd2c7faa5..d8d624122a0 100644 --- a/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java +++ b/src/main/java/org/apache/sysds/runtime/transform/decode/DecoderRecode.java @@ -126,20 +126,20 @@ public void initMetaData(FrameBlock meta) { _rcMaps = new HashMap[_colList.length]; for( int j=0; j<_colList.length; j++ ) { HashMap map = new HashMap<>(); - for( int i=0; i= 0} - * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are - * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply. + * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0} both the minimum and + * maximum fraction digits are pinned to {@code decimal}, so values are printed with exactly that many decimals; + * otherwise the {@link DecimalFormat} defaults apply. + * * @param decimal number of decimal places to print, -1 for default * @return a configured {@link DecimalFormat} */ private static DecimalFormat createDecimalFormat(int decimal) { DecimalFormat df = new DecimalFormat(); df.setGroupingUsed(false); - if (decimal >= 0) { + if(decimal >= 0) { df.setMinimumFractionDigits(decimal); df.setMaximumFractionDigits(decimal); } diff --git a/src/main/java/org/apache/sysds/utils/DoubleParser.java b/src/main/java/org/apache/sysds/utils/DoubleParser.java index c0122f8061f..2252b7270c8 100644 --- a/src/main/java/org/apache/sysds/utils/DoubleParser.java +++ b/src/main/java/org/apache/sysds/utils/DoubleParser.java @@ -200,7 +200,7 @@ public static double parseFloatingPointLiteral(String str, int offset, int endIn // : is the first character after numbers. // 0 is the first number. // we use the last position, since this is not allowed to be other values than a number. - if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') + if(str.charAt(endIndex - 1) > '9' || str.charAt(endIndex - 1) < '0') return Double.parseDouble(str); final double val = parseDecFloatLiteral(str, index, offset, endIndex); diff --git a/src/main/java/org/apache/sysds/utils/SettingsChecker.java b/src/main/java/org/apache/sysds/utils/SettingsChecker.java index c5f14a7043b..10d1a42b246 100644 --- a/src/main/java/org/apache/sysds/utils/SettingsChecker.java +++ b/src/main/java/org/apache/sysds/utils/SettingsChecker.java @@ -106,25 +106,24 @@ private static long maxMemMachineOSX() { } private static long maxMemMachineWin() { - //try modern powershell, otherwise wmic, log errors as warning but avoid crashes + // try modern powershell, otherwise wmic, log errors as warning but avoid crashes long tmp = maxMemMachineWin(true); - if( tmp < 0 ) + if(tmp < 0) tmp = maxMemMachineWin(false); return tmp; } - + private static long maxMemMachineWin(boolean modern) { int startIx = modern ? 3 : 1; - String command = modern ? - "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : - "wmic memorychip get capacity"; //in bytes + String command = modern ? "powershell Get-CimInstance -ClassName Win32_PhysicalMemory | Select-Object Capacity" : "wmic memorychip get capacity"; // in + // bytes try { Runtime rt = Runtime.getRuntime(); Process pr = rt.exec(command); String[] memStr = new String(pr.getInputStream().readAllBytes(), StandardCharsets.UTF_8).split("\n"); //skip header, and aggregate DIMM capacities long capacity = 0; - for( int i=startIx; i 0 ) capacity += Long.parseLong(tmp); diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index 0622e789baa..a941e90f724 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -243,10 +243,9 @@ private static void run17(String[] args) throws Exception { } /** - * Repeatedly read the same on-disk Delta frame table (written once as setup). - * Args: {@code 18 [mode] [targetFileSizeMB]} - * where mode is one of serial|parallel|both (default parallel) and an omitted - * target file size uses the adaptive default sizing. + * Repeatedly read the same on-disk Delta frame table (written once as setup). Args: + * {@code 18 [mode] [targetFileSizeMB]} where mode is one of serial|parallel|both (default parallel) + * and an omitted target file size uses the adaptive default sizing. */ private static void run18(String[] args) throws Exception { int rows = Integer.parseInt(args[1]); diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 36ea11b3e2f..4aac0685698 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -117,8 +117,8 @@ public abstract class AutomatedTestBase { public static final double GPU_TOLERANCE = 1e-9; /** - * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy - * {@code sleep()} calls. {@link FederatedWorkerUtils} enforces its own minimum floor. + * Default deadline (ms) for federated worker/monitoring readiness waits and a few legacy {@code sleep()} calls. + * {@link FederatedWorkerUtils} enforces its own minimum floor. */ public static final int FED_WORKER_WAIT = 3000; @@ -1761,9 +1761,9 @@ private static Process spawnLocalFedWorker(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections (Netty's bind has completed), or - * throws a {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor - * elapses. + *

+ * Returns once the backend's TCP port accepts connections (Netty's bind has completed), or throws a + * {@link RuntimeException} once the {@link FederatedWorkerUtils} readiness floor elapses. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1776,10 +1776,10 @@ protected Process startLocalFedMonitoring(int port, String[] addArgs) { /** * Start a new JVM for a federated monitoring backend at the port. * - *

Returns once the backend's TCP port accepts connections, or throws a - * {@link RuntimeException} after {@code timeoutMs} elapses. The monitoring server opens the - * port after Netty's {@code bind().sync()} returns; a successful TCP connect therefore signals - * that the HTTP listener is ready to accept requests. + *

+ * Returns once the backend's TCP port accepts connections, or throws a {@link RuntimeException} after + * {@code timeoutMs} elapses. The monitoring server opens the port after Netty's {@code bind().sync()} returns; a + * successful TCP connect therefore signals that the HTTP listener is ready to accept requests. * * @param port Port to use for the JVM * @param addArgs Extra CLI args to append, or null @@ -1798,8 +1798,9 @@ private static Process spawnLocalFedMonitoring(int port, String[] addArgs) { String separator = System.getProperty("file.separator"); String classpath = System.getProperty("java.class.path"); String path = System.getProperty("java.home") + separator + "bin" + separator + "java"; - String[] args = ArrayUtils.addAll(new String[] {path, "-cp", classpath, DMLScript.class.getName(), - "-fedMonitoring", Integer.toString(port)}, addArgs); + String[] args = ArrayUtils.addAll( + new String[] {path, "-cp", classpath, DMLScript.class.getName(), "-fedMonitoring", Integer.toString(port)}, + addArgs); try { return new ProcessBuilder(args).start(); } diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index f14a614b583..0c7cd2046e5 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -2941,10 +2941,9 @@ public static void writeTestScalar(String file, double value) { } } - /** * Write scalar to file - * + * * @param file File to write to * @param value Value to write */ @@ -3519,9 +3518,9 @@ public static void shutdownThread(Thread t) { // Bounded join: workers are daemon threads, so even if one ignores the interrupt // we must not block cleanup (and the JVM) indefinitely waiting for it. t.join(THREAD_SHUTDOWN_JOIN_MS); - if( t.isAlive() ) - LOG.warn("Federated worker thread " + t.getName() - + " did not stop within " + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); + if(t.isAlive()) + LOG.warn("Federated worker thread " + t.getName() + " did not stop within " + + THREAD_SHUTDOWN_JOIN_MS + "ms; leaving it as a daemon."); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java index 07ec9752928..d76c741d870 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compile/CompilerTestBase.java @@ -67,10 +67,10 @@ public void setUp() { /** * Compile a DML script string into a runtime {@link Program} without executing it. * - * @param dmlScript the DML source - * @param args named command-line arguments ($name -> value), may be null - * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) - * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions + * @param dmlScript the DML source + * @param args named command-line arguments ($name -> value), may be null + * @param mode the global execution mode (e.g. {@link ExecMode#HYBRID}) + * @param localMaxMem the local memory budget in bytes used for memory-based exec-type decisions * @return the compiled runtime program */ protected Program compile(String dmlScript, Map args, ExecMode mode, long localMaxMem) { @@ -145,8 +145,7 @@ else if(pb instanceof FunctionProgramBlock) { /** All instructions whose opcode equals {@code opcode} (exact match). */ protected List getByOpcode(Program prog, String opcode) { - return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())) - .collect(Collectors.toList()); + return getInstructions(prog).stream().filter(i -> opcode.equals(i.getOpcode())).collect(Collectors.toList()); } protected static boolean isSpark(Instruction inst) { @@ -169,13 +168,13 @@ protected void assertCP(Program prog, String opcode) { private void assertExecType(Program prog, String opcode, boolean expectSpark) { List matches = getByOpcode(prog, opcode); - Assert.assertFalse("Expected at least one '" + opcode + "' instruction but found none.\n" - + Explain.explain(prog), matches.isEmpty()); + Assert.assertFalse( + "Expected at least one '" + opcode + "' instruction but found none.\n" + Explain.explain(prog), + matches.isEmpty()); for(Instruction inst : matches) { boolean spark = isSpark(inst); - Assert.assertEquals("Instruction '" + opcode + "' expected exec type " - + (expectSpark ? "SPARK" : "CP") + " but was " + (spark ? "SPARK" : "CP") + ".\n" - + Explain.explain(prog), expectSpark, spark); + Assert.assertEquals("Instruction '" + opcode + "' expected exec type " + (expectSpark ? "SPARK" : "CP") + + " but was " + (spark ? "SPARK" : "CP") + ".\n" + Explain.explain(prog), expectSpark, spark); } } diff --git a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java index 0b3889db908..7b45120d8f1 100644 --- a/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java +++ b/src/test/java/org/apache/sysds/test/component/compile/SparkTransitiveExecTypeCompileTest.java @@ -33,14 +33,13 @@ */ public class SparkTransitiveExecTypeCompileTest extends CompilerTestBase { - private static final String DML_HEADER = - "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and colSums run on Spark - "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) + private static final String DML_HEADER = "X = rand(rows=20000000, cols=8, seed=1);\n" + // ~1.2GB -> rand and + // colSums run on Spark + "v = colSums(X);\n"; // 1x8 Spark-resident vector (opcode uack+) @Test public void singleConsumerUnaryPulledIntoSpark() { - String dml = DML_HEADER + - "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark + String dml = DML_HEADER + "r = round(v);\n" + // sole consumer of the Spark-resident vector -> pulled into Spark "print(sum(r));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); @@ -50,62 +49,58 @@ public void singleConsumerUnaryPulledIntoSpark() { @Test public void multiConsumerUnaryStaysCP() { - String dml = DML_HEADER + - "a = round(v);\n" + // v now has two consumers (round + abs) ... - "b = abs(v);\n" + - "print(sum(a) + sum(b));\n"; + String dml = DML_HEADER + "a = round(v);\n" + // v now has two consumers (round + abs) ... + "b = abs(v);\n" + "print(sum(a) + sum(b));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uack+"); // input still has a Spark output ... - assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP + assertCP(prog, "round"); // ... but the multi-parent guard keeps both unaries in CP assertCP(prog, "abs"); } // A tall, Spark-resident column vector that is still small enough (40 KB) to be CP by memory // estimate: rowSums over a very wide matrix runs on Spark, but its 1-column result fits in CP. - private static final String TALL_VECTOR_HEADER = - "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and rowSums run on Spark - "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) + private static final String TALL_VECTOR_HEADER = "X = rand(rows=5000, cols=200000, seed=1);\n" + // ~8GB -> rand and + // rowSums run + // on Spark + "c = rowSums(X);\n"; // 5000x1 Spark-resident vector (opcode uark+) @Test public void cumulativeUnaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by estimate ... + String dml = TALL_VECTOR_HEADER + "r = cumsum(c);\n" + // sole consumer of the Spark-resident vector, CP by + // estimate ... "print(as.scalar(r[2500,1]));\n"; // ... consume via indexing (avoids the sum(cumsum) rewrite) Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); - assertSpark(prog, "uark+"); // input genuinely has a Spark output + assertSpark(prog, "uark+"); // input genuinely has a Spark output assertCP(prog, "ucumk+"); // ... but cumulative ops are excluded from the transitive pull } @Test public void singleConsumerBinaryPulledIntoSpark() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole consumer -> pulled into Spark + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // matrix-scalar on the Spark-resident vector, sole + // consumer -> pulled into Spark "print(as.scalar(r[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input genuinely has a Spark output (multi-block column vector) - assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) + assertSpark(prog, "+"); // matrix-scalar binary pulled into Spark (CP by estimate, single consumer) } @Test public void multiConsumerBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... - "b = c * 3.0;\n" + - "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; + String dml = TALL_VECTOR_HEADER + "a = c + 2.0;\n" + // c now has two consumers (+ and *) ... + "b = c * 3.0;\n" + "print(as.scalar(a[2500,1]) + as.scalar(b[2500,1]));\n"; Program prog = compile(dml, null, ExecMode.HYBRID, SMALL_MEM_BUDGET); assertSpark(prog, "uark+"); // input still has a Spark output ... - assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP + assertCP(prog, "+"); // ... but the multi-parent guard keeps both binaries in CP assertCP(prog, "*"); } @Test public void transitiveDisabledUnaryStaysCP() { - String dml = DML_HEADER + - "r = round(v);\n" + // pullable unary, but flag is off + String dml = DML_HEADER + "r = round(v);\n" + // pullable unary, but flag is off "print(sum(r));\n"; Program prog = compileWithTransitive(dml, false); @@ -115,8 +110,7 @@ public void transitiveDisabledUnaryStaysCP() { @Test public void transitiveDisabledBinaryStaysCP() { - String dml = TALL_VECTOR_HEADER + - "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off + String dml = TALL_VECTOR_HEADER + "r = c + 2.0;\n" + // pullable matrix-scalar, but flag is off "print(as.scalar(r[2500,1]));\n"; Program prog = compileWithTransitive(dml, false); diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java index 083a29f965b..22f867819c3 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedSortTest.java @@ -43,7 +43,8 @@ /** * Tests the {@code order} (sort) reorg operation on compressed matrices. A single column held in a single column group * is sorted ascending while staying compressed (via {@link org.apache.sysds.runtime.compress.lib.CLALibSort}); every - * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed reference. + * other configuration falls back to a decompressed reorg. In all cases the result must match the uncompressed + * reference. */ public class CompressedSortTest { @@ -263,8 +264,7 @@ private void runCompressed(MatrixBlock mb, CompressionType ct) { assertEquals("Expected a single column group", 1, cmb.getColGroups().size()); MatrixBlock actual = cmb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); - assertTrue("Expected the sorted result to stay compressed for " + ct, - actual instanceof CompressedMatrixBlock); + assertTrue("Expected the sorted result to stay compressed for " + ct, actual instanceof CompressedMatrixBlock); MatrixBlock expected = mb.reorgOperations(ASC, new MatrixBlock(), 0, 0, 0); TestUtils.compareMatrices(expected, CompressedMatrixBlock.getUncompressed(actual, "sort"), 0.0, "sort " + ct); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java index 833128ad9f0..49cbbdd248d 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CLALibMMChainTest.java @@ -62,8 +62,8 @@ public static void setup() { } /** - * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees a - * single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. + * Build a compressed matrix backed by a single DDC column group spanning all {@code nCol} columns. This guarantees + * a single (non-uncompressed) column group, which is what triggers the mm-chain fast path for wide enough matrices. */ private static CompressedMatrixBlock singleDDC(int nRow, int nCol, int nVal, int seed) { Random r = new Random(seed); diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java index 549010a78cb..c6afd5a3543 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CompressedBinaryMatrixMatrixSolveTest.java @@ -43,8 +43,8 @@ /** * Drive the solve opcode through {@link BinaryMatrixMatrixCPInstruction} with compressed inputs to cover the - * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The - * script-level solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. + * commons-math matrix-matrix branch that decompresses compressed left/right operands before solving. The script-level + * solve tests only ever see uncompressed inputs, so this branch is otherwise unreached. */ public class CompressedBinaryMatrixMatrixSolveTest { @@ -72,8 +72,8 @@ public void solveCompressedLeftCompressedRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); CompressedMatrixBlock bC = CompressedMatrixBlockFactory.createConstant(n, 2, 1.0); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), CompressedMatrixBlock.getUncompressed(bC), SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), + CompressedMatrixBlock.getUncompressed(bC), SOLVE); MatrixBlock actual = runSolve(aC, bC); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -94,8 +94,8 @@ public void solveCompressedLeftDenseRight() { assertTrue("A must compress to exercise the compressed-left path", aC instanceof CompressedMatrixBlock); MatrixBlock b = TestUtils.round(TestUtils.generateTestMatrixBlock(n, 1, -5, 5, 1.0, 7)); - MatrixBlock expected = LibCommonsMath.matrixMatrixOperations( - CompressedMatrixBlock.getUncompressed(aC), b, SOLVE); + MatrixBlock expected = LibCommonsMath.matrixMatrixOperations(CompressedMatrixBlock.getUncompressed(aC), b, + SOLVE); MatrixBlock actual = runSolve(aC, b); TestUtils.compareMatricesBitAvgDistance(expected, actual, 0, 0, SOLVE); @@ -119,7 +119,8 @@ private static BinaryMatrixMatrixCPInstruction solveInstruction() { } private static MatrixObject matrixObject(String name, MatrixBlock mb) { - MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, mb.getNonZeros()); + MatrixCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), 1000, + mb.getNonZeros()); MatrixObject mo = new MatrixObject(ValueType.FP64, "/dev/null/" + name, new MetaDataFormat(mc, FileFormat.BINARY), mb); return mo; diff --git a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java index 8907c82f1d1..0d2f37e319c 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/offset/OffsetClassInitConcurrencyTest.java @@ -50,7 +50,9 @@ public class OffsetClassInitConcurrencyTest { private static final String[] INIT_TARGETS = {PKG + "AOffset", PKG + "OffsetEmpty", PKG + "OffsetChar", PKG + "OffsetByte", PKG + "OffsetSingle", PKG + "OffsetTwo"}; - /** Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. */ + /** + * Whether a class-init cycle deadlocks depends on thread timing, so repeat to make a regression reliable to catch. + */ private static final int ROUNDS = 20; /** A real init deadlock never resolves; a healthy round finishes in milliseconds, so this bound is generous. */ diff --git a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java index 3493da7d2b1..f7dffa9021c 100644 --- a/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java +++ b/src/test/java/org/apache/sysds/test/component/context/SparkContextReferenceCountTest.java @@ -35,12 +35,10 @@ public class SparkContextReferenceCountTest { /** - * Two DML executions sharing the JVM-wide singleton spark context (as happens - * with surefire parallel tests, threadCount>1). When the first execution - * finishes and calls close(), the shared context must stay alive because the - * second execution still has in-flight work. Before reference counting, - * close() stopped the context unconditionally, which cancelled the second - * execution's spark job and wedged it until the test watchdog. + * Two DML executions sharing the JVM-wide singleton spark context (as happens with surefire parallel tests, + * threadCount>1). When the first execution finishes and calls close(), the shared context must stay alive + * because the second execution still has in-flight work. Before reference counting, close() stopped the context + * unconditionally, which cancelled the second execution's spark job and wedged it until the test watchdog. */ @Test public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { @@ -64,16 +62,13 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { // context that B still uses SparkExecutionContext.exitSparkExecution(); ecA.close(); - assertFalse("shared context must stay alive while another execution is active", - sc.sc().isStopped()); - assertEquals("B's job must still run on the live context", - 10L, rdd.reduce(Integer::sum).longValue()); + assertFalse("shared context must stay alive while another execution is active", sc.sc().isStopped()); + assertEquals("B's job must still run on the live context", 10L, rdd.reduce(Integer::sum).longValue()); // B finishes last: releasing the final registration lets close() stop it SparkExecutionContext.exitSparkExecution(); ecB.close(); - assertTrue("shared context must be stopped once the last execution closes", - sc.sc().isStopped()); + assertTrue("shared context must be stopped once the last execution closes", sc.sc().isStopped()); } finally { // drain any remaining registrations and stop the context so a failed @@ -87,10 +82,9 @@ public void closeKeepsContextAliveWhileAnotherExecutionIsActive() { } /** - * An unpaired close() (a caller that borrows the shared context but never - * registered via enterSparkExecution()) must not stop a context another - * execution still uses. This fails on the old unconditional-stop code, which - * tore the context down out from under the active execution. + * An unpaired close() (a caller that borrows the shared context but never registered via enterSparkExecution()) + * must not stop a context another execution still uses. This fails on the old unconditional-stop code, which tore + * the context down out from under the active execution. */ @Test public void unpairedCloseDoesNotStopAContextStillInUse() { @@ -106,14 +100,12 @@ public void unpairedCloseDoesNotStopAContextStillInUse() { // borrows the shared context): close() must not stop a context in use unregistered = ExecutionContextFactory.createSparkExecutionContext(); unregistered.close(); - assertFalse("unpaired close() must not stop a context still in use", - sc.sc().isStopped()); + assertFalse("unpaired close() must not stop a context still in use", sc.sc().isStopped()); // the registered execution finishing stops the context as the last user SparkExecutionContext.exitSparkExecution(); active.close(); - assertTrue("context must stop once the last registered execution closes", - sc.sc().isStopped()); + assertTrue("context must stop once the last registered execution closes", sc.sc().isStopped()); } finally { SparkExecutionContext.exitSparkExecution(); diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java index 2c854b4a81b..459936fa24c 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerBase.java @@ -78,17 +78,18 @@ public MatrixBlock getMatrixBlock(long id) { } /** - * Poll the federated worker until the matrix at {@code id} is observed as a - * {@link CompressedMatrixBlock}, or {@link #COMPRESS_TIMEOUT_MS} elapses. + * Poll the federated worker until the matrix at {@code id} is observed as a {@link CompressedMatrixBlock}, or + * {@link #COMPRESS_TIMEOUT_MS} elapses. * - *

Federated workers compress asynchronously after a PUT/READ_VAR (see - * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right - * after the operation can race against the in-flight compression and return the uncompressed - * block. Tests that need to observe the compressed form should poll instead of sleeping a fixed - * amount. + *

+ * Federated workers compress asynchronously after a PUT/READ_VAR (see + * {@code CompressedMatrixBlockFactory.compressAsync}), so a {@code getMatrixBlock} fired right after the operation + * can race against the in-flight compression and return the uncompressed block. Tests that need to observe the + * compressed form should poll instead of sleeping a fixed amount. * - *

On timeout this returns the most recent (uncompressed) read so the caller can produce a - * meaningful assertion failure naming the variable. + *

+ * On timeout this returns the most recent (uncompressed) read so the caller can produce a meaningful assertion + * failure naming the variable. * * @param id federated variable id * @return the matrix block, compressed if compression finished in time, otherwise the latest read diff --git a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java index 2b5ff327ef3..d3e4485bdf4 100644 --- a/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java +++ b/src/test/java/org/apache/sysds/test/component/federated/FedWorkerMatrixCompress.java @@ -68,13 +68,11 @@ public void verifySameOrAlsoCompressedAsLocalCompress() { // federated. Compression on the worker is async; poll only when we expect compression to // match the local result, otherwise a single read is enough. final long id = putMatrixBlock(mb); - final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) - ? awaitCompressed(id) - : getMatrixBlock(id); + final MatrixBlock mbr = (mbcLocal instanceof CompressedMatrixBlock) ? awaitCompressed(id) : getMatrixBlock(id); if(mbcLocal instanceof CompressedMatrixBlock && !(mbr instanceof CompressedMatrixBlock)) - fail("Invalid result, the federated site did not compress the matrix block within " - + COMPRESS_TIMEOUT_MS + "ms"); + fail("Invalid result, the federated site did not compress the matrix block within " + COMPRESS_TIMEOUT_MS + + "ms"); TestUtils.compareMatricesBitAvgDistance(mbcLocal, mbr, 0, 0, "Not equivalent matrix block returned from federated site"); diff --git a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java index 60587bf51a2..0de3b6db640 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/FrameToStringTest.java @@ -42,7 +42,7 @@ public void test100x100() { @Test public void testDecimalClampsFractionDigits() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(1); f.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -53,10 +53,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - f.set(1, 0, 5.244058388023880); // rounded at the last requested digit + f.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + f.set(1, 0, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(f, false, " ", "\n", 2, 1, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000\n")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441\n")); @@ -64,10 +64,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - FrameBlock f = new FrameBlock(new ValueType[]{ValueType.FP64}, new String[]{"C1"}); + FrameBlock f = new FrameBlock(new ValueType[] {ValueType.FP64}, new String[] {"C1"}); f.ensureAllocatedColumns(2); - f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - f.set(1, 0, 5.244058388023880); // default cap of three fraction digits + f.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + f.set(1, 0, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(f, false, " ", "\n", 2, 1, -1); assertTrue("expected unpadded 22: " + out, out.contains("22\n")); diff --git a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java index 43a53879f17..7d67c04983b 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/MatrixFromFrameSafeCastTest.java @@ -142,8 +142,7 @@ public void safeCastWarnsOnlyOnce() { // the fallback warning must be logged exactly once across both conversions final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(1, warnings); } @@ -153,8 +152,7 @@ public void strictThrowsWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 1)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -164,8 +162,7 @@ public void strictThrowsParallelWhenWarnCastDisabled() { setWarnCast(false); FrameBlock fb = mixedFrame(); - Exception e = assertThrows(DMLRuntimeException.class, - () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); + Exception e = assertThrows(DMLRuntimeException.class, () -> MatrixBlockFromFrame.convertToMatrixBlock(fb, 4)); assertTrue(e.getMessage().contains("Failed to convert FrameBlock to MatrixBlock")); } @@ -185,8 +182,7 @@ public void warnCastValidFrameConvertsWithoutFallback() { final List log = LoggingUtils.reinsert(appender); long warnings = log.stream() - .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")) - .count(); + .filter(l -> l.getMessage().toString().contains("falling back to NaN on incompatible cells")).count(); assertEquals(0, warnings); } diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java index ccba674707b..982941bb190 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/DecoderCompositeTest.java @@ -74,8 +74,8 @@ private void runDecode(String spec, int nCol, int nCat) { try { FrameBlock data = categoricalFrame(ROWS, nCol, nCat, 17); - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); MatrixBlock encoded = encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); @@ -122,8 +122,8 @@ public void singleThreadEqualsParallelManyCategories() { public void decoderIsComposite() { FrameBlock data = categoricalFrame(100, 2, 3, 1); String spec = "{recode:[C1], dummycode:[C2]}"; - MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), - data.getNumColumns(), null); + MultiColumnEncoder encoder = EncoderFactory.createEncoder(spec, data.getColumnNames(), data.getNumColumns(), + null); encoder.encode(data, 1); Decoder decoder = buildDecoder(data, spec, encoder); if(!(decoder instanceof DecoderComposite)) diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java index d9c540f54c5..412686c1e83 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/GetCategoricalMaskInstructionTest.java @@ -47,9 +47,9 @@ import org.junit.Test; /** - * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code - * paths (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and - * the unsupported opcode guard) that the script-level transform tests cannot reach. + * Unit tests that drive the get_categorical_mask instruction directly to exercise the defensive code paths + * (distinct-count prefix in the metadata frame, default column metadata, non id-based specs and the unsupported opcode + * guard) that the script-level transform tests cannot reach. */ public class GetCategoricalMaskInstructionTest { protected static final Log LOG = LogFactory.getLog(GetCategoricalMaskInstructionTest.class.getName()); @@ -210,7 +210,8 @@ public void imputeAndOmitAreAccepted() { // impute and omit do not change the output column count or categorical flag, so a spec that // only adds them on top of a recoded column must still succeed and mark that column categorical FrameBlock meta = new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}); - MatrixBlock res = run(meta, "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); + MatrixBlock res = run(meta, + "{\"ids\": true, \"recode\": [1], \"impute\": [{\"id\": 1, \"method\": \"global_mode\"}], \"omit\": [1]}"); assertEquals(1, res.getNumRows()); assertEquals(1, res.getNumColumns()); @@ -230,8 +231,7 @@ public void unsupportedOpcodeThrows() { // any frame-scalar binary opcode other than get_categorical_mask must be rejected ExecutionContext ec = ExecutionContextFactory.createContext(); ec.setAutoCreateVars(true); - ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, - new String[][] {{"a"}}))); + ec.setVariable("F", frameObject(new FrameBlock(new ValueType[] {ValueType.STRING}, new String[][] {{"a"}}))); assertThrowsMessage("Unsupported operation", () -> maskInstruction("+").processInstruction(ec)); } @@ -263,9 +263,9 @@ private static void assertMask(MatrixBlock res, double[] expected) { } /** - * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to - * that column's metadata as the recode distinct count (the path real transformencode uses), while - * a zero leaves the column with default metadata (a continuous / non-dummycoded column). + * Build a single-row metadata frame of nCol string columns. A positive distinct[i] is written to that column's + * metadata as the recode distinct count (the path real transformencode uses), while a zero leaves the column with + * default metadata (a continuous / non-dummycoded column). */ private static FrameBlock metaWithDistinct(int nCol, int[] distinct) { ValueType[] schema = new ValueType[nCol]; @@ -290,7 +290,8 @@ private static MatrixBlock run(FrameBlock meta, String spec) { private static BinaryFrameScalarCPInstruction maskInstruction(String opcode) { String in1 = InstructionUtils.concatOperandParts("F", DataType.FRAME.name(), ValueType.STRING.name(), "false"); - String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), "true"); + String in2 = InstructionUtils.concatOperandParts("spec", DataType.SCALAR.name(), ValueType.STRING.name(), + "true"); String out = InstructionUtils.concatOperandParts("out", DataType.MATRIX.name(), ValueType.FP64.name(), "false"); String str = InstructionUtils.concatOperands("CP", opcode, in1, in2, out); return (BinaryFrameScalarCPInstruction) BinaryCPInstruction.parseInstruction(str); diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java index b2d31f43b83..645c4dd09bd 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeRoundTripTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -44,10 +44,10 @@ import org.junit.Test; /** - * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so a - * decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact reconstruction - * for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search and the parallel - * block split are validated against ground truth rather than only against each other. + * Exact inverse correctness tests for the transform decoders. Recode and dummycode are lossless category encodings, so + * a decode of the encoded matrix must reconstruct the original categorical frame. These tests assert exact + * reconstruction for the dense path, the sparse path, and the parallel path so that the dummycode sparse binary search + * and the parallel block split are validated against ground truth rather than only against each other. */ public class TransformDecodeRoundTripTest { protected static final Log LOG = LogFactory.getLog(TransformDecodeRoundTripTest.class.getName()); @@ -59,9 +59,9 @@ public void setUp() { } private static FrameBlock categoricalFrame() { - final String[] values = new String[] { - "apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", "date", "banana", "elderberry", - "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", "banana"}; + final String[] values = new String[] {"apple", "banana", "apple", "cherry", "banana", "date", "apple", "cherry", + "date", "banana", "elderberry", "apple", "fig", "banana", "cherry", "apple", "date", "fig", "elderberry", + "banana"}; final FrameBlock f = new FrameBlock(new ValueType[] {ValueType.STRING}); f.ensureAllocatedColumns(values.length); for(int i = 0; i < values.length; i++) @@ -117,8 +117,8 @@ public void binWithDummycodeOnOtherColumnConsistency() { /** * Dummycode on an earlier column (1) shifts the bin column (2) to the right in the encoded matrix. The bin decoder - * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the - * non-magic offset branch of the bin source-column mapping. + * must walk the dummycode domain sizes to recover the bin column's true source position. This drives the non-magic + * offset branch of the bin source-column mapping. */ @Test public void binAfterDummycodeOnEarlierColumnConsistency() { @@ -128,9 +128,9 @@ public void binAfterDummycodeOnEarlierColumnConsistency() { } /** - * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain - * size K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead - * of numDistinct) to compute the offset. + * Same right-shift as above, but the earlier column is feature-hashed before being dummycoded. The hash domain size + * K is stored as a plain integer in the single meta cell, so the bin source-column mapping reads it (instead of + * numDistinct) to compute the offset. */ @Test public void binAfterHashDummycodeOnEarlierColumnConsistency() { @@ -211,10 +211,10 @@ public void binDecodeZeroCodeUsesFirstBinBoundary() { } /** - * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the - * decoder must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly - * deserialized decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode - * (the latter exercises the serialized _srcCols/_dcCols source-column mapping). + * Spark broadcasts the decoder to executors via Java serialization without re-running initMetaData, so the decoder + * must round-trip all of its decode state through writeExternal/readExternal. Decode with a freshly deserialized + * decoder and assert it matches the in-memory decode. Covers plain bin and bin-with-dummycode (the latter exercises + * the serialized _srcCols/_dcCols source-column mapping). */ @Test public void binDecoderSurvivesSerialization() { @@ -481,8 +481,7 @@ public void parallelDecodeWrapsWorkerException() { fail("expected the parallel decode wrapper to propagate the worker failure"); } catch(DMLRuntimeException expected) { - assertNotNull("parallel decode wrapper must retain the worker exception as cause", - expected.getCause()); + assertNotNull("parallel decode wrapper must retain the worker exception as cause", expected.getCause()); } } catch(Exception e) { diff --git a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java index 54bd1679716..5745a35d506 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java +++ b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformDecodeTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java index 8d7ad14539a..64012c06bbf 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixCoverageTest.java @@ -57,17 +57,15 @@ import io.delta.kernel.types.TimestampType; /** - * Targeted tests for the error/defensive branches of the native Delta matrix - * read/write code that the round-trip and interop tests do not reach: malformed - * per-file statistics, unsupported column types, unsupported stream operations, + * Targeted tests for the error/defensive branches of the native Delta matrix read/write code that the round-trip and + * interop tests do not reach: malformed per-file statistics, unsupported column types, unsupported stream operations, * bad table paths, and the non-dense writer input path. * - *

A few of these branches guard against inputs that the SystemDS writer and - * the Delta Kernel scan API never produce in a normal round trip (e.g. a - * statistics JSON without {@code numRecords}, or a column type code outside the - * supported set). They are exercised here by mocking the Delta Kernel data - * objects and invoking the (package-private) helpers reflectively, rather than - * widening their production visibility purely for testing. + *

+ * A few of these branches guard against inputs that the SystemDS writer and the Delta Kernel scan API never produce in + * a normal round trip (e.g. a statistics JSON without {@code numRecords}, or a column type code outside the supported + * set). They are exercised here by mocking the Delta Kernel data objects and invoking the (package-private) helpers + * reflectively, rather than widening their production visibility purely for testing. */ public class DeltaMatrixCoverageTest { @@ -89,8 +87,8 @@ public void qualifyRejectsUnknownFilesystemScheme() { @Test public void typeCodeReturnsNegativeForUnsupportedTypes() { - //non-numeric / unsupported Delta types must map to the sentinel -1 so the - //reader can reject them with a clear message rather than mis-decoding. + // non-numeric / unsupported Delta types must map to the sentinel -1 so the + // reader can reject them with a clear message rather than mis-decoding. assertEquals(-1, DeltaKernelUtils.typeCode(DateType.DATE)); assertEquals(-1, DeltaKernelUtils.typeCode(TimestampType.TIMESTAMP)); assertEquals(-1, DeltaKernelUtils.typeCode(BinaryType.BINARY)); @@ -112,17 +110,17 @@ public void writerRejectsStreamWrite() throws Exception { @Test public void numRecordsHandlesAbsentNullAndMalformedStats() throws Exception { - //no "stats" field at all -> -1 + // no "stats" field at all -> -1 assertEquals(-1, numRecords(addFileRow(new StructType().add("path", StringType.STRING), false, null))); - //stats column present but null-at -> -1 + // stats column present but null-at -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), true, null))); - //stats string explicitly null -> -1 + // stats string explicitly null -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, null))); - //malformed JSON -> JsonProcessingException -> -1 + // malformed JSON -> JsonProcessingException -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{not valid json"))); - //valid JSON but no numRecords field -> -1 + // valid JSON but no numRecords field -> -1 assertEquals(-1, numRecords(addFileRow(statsSchema(), false, "{\"minValues\":{}}"))); - //well-formed stats -> the parsed count + // well-formed stats -> the parsed count assertEquals(1234L, numRecords(addFileRow(statsSchema(), false, "{\"numRecords\":1234}"))); } @@ -131,8 +129,8 @@ public void getDoubleValueRejectsUnknownTypeCode() throws Exception { Method m = ReaderDelta.class.getDeclaredMethod("getDoubleValue", ColumnVector.class, int.class, int.class); m.setAccessible(true); try { - //type code outside the supported T_* set; the switch default must throw - //before touching the (null) vector. + // type code outside the supported T_* set; the switch default must throw + // before touching the (null) vector. m.invoke(null, (ColumnVector) null, 0, 999); fail("expected a DMLRuntimeException for an unsupported type code"); } @@ -156,9 +154,9 @@ public void numericTypeCodeRejectsNonNumericType() throws Exception { @Test public void parallelReadWrapsFileFailure() throws Exception { - //a per-file decode failure in the parallel reader must surface as a single - //clear IOException (the awaitFileTasks catch), not a raw executor error. - //Provoke it by deleting one data file after the table (and its log) exist. + // a per-file decode failure in the parallel reader must surface as a single + // clear IOException (the awaitFileTasks catch), not a raw executor error. + // Provoke it by deleting one data file after the table (and its log) exist. MatrixBlock in = TestUtils.generateTestMatrixBlock(100_000, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); DMLConfig conf = new DMLConfig(); @@ -167,15 +165,14 @@ public void parallelReadWrapsFileFailure() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_fail_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //delete one parquet data file; the transaction log still references it, - //so the scan enumerates it but the decode task fails. + // delete one parquet data file; the transaction log still references it, + // so the scan enumerates it but the decode task fails. File victim; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { - victim = s.filter(p -> p.toString().endsWith(".parquet")) - .findFirst().map(Path::toFile).orElse(null); + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { + victim = s.filter(p -> p.toString().endsWith(".parquet")).findFirst().map(Path::toFile).orElse(null); } assertTrue("expected at least one data file to delete", victim != null && victim.delete()); @@ -200,8 +197,8 @@ public void parallelReadWrapsFileFailure() throws Exception { @Test public void sparseFormatMatrixRoundTrips() throws Exception { - //a sparse-backed MatrixBlock takes the writer's non-contiguous path (no - //direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. + // a sparse-backed MatrixBlock takes the writer's non-contiguous path (no + // direct double[] view), exercising MatrixColumnVector.get via MatrixBlock. MatrixBlock in = TestUtils.generateTestMatrixBlock(2000, 7, -5, 5, 0.05, 13); in.recomputeNonZeros(); in.examSparsity(); @@ -210,8 +207,8 @@ public void sparseFormatMatrixRoundTrips() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_sparse_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", in.getNumRows(), out.getNumRows()); assertEquals("cols", in.getNumColumns(), out.getNumColumns()); @@ -224,15 +221,15 @@ public void sparseFormatMatrixRoundTrips() throws Exception { @Test public void fillDenseHandlesNonContiguousBlock() throws Exception { - //the dense fill normally hits the contiguous fast path; force a multi-block - //(non-contiguous) dense block so the row-by-row fallback is exercised. Such - //blocks only arise for matrices beyond a single contiguous array, so we - //shrink the per-block allocation cap to provoke it on a tiny matrix. + // the dense fill normally hits the contiguous fast path; force a multi-block + // (non-contiguous) dense block so the row-by-row fallback is exercised. Such + // blocks only arise for matrices beyond a single contiguous array, so we + // shrink the per-block allocation cap to provoke it on a tiny matrix. int rows = 5, cols = 4; int savedMaxAlloc = DenseBlockLDRB.MAX_ALLOC; DenseBlock db; try { - DenseBlockLDRB.MAX_ALLOC = 2 * cols; //~2 rows per block -> multiple blocks + DenseBlockLDRB.MAX_ALLOC = 2 * cols; // ~2 rows per block -> multiple blocks db = new DenseBlockLFP64(new int[] {rows, cols}); } finally { @@ -240,14 +237,14 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { } assertTrue("expected a non-contiguous (multi-block) dense block", !db.isContiguous()); - //two row-major batches (3 rows + 2 rows) covering all 5 rows + // two row-major batches (3 rows + 2 rows) covering all 5 rows double[] b0 = new double[3 * cols]; double[] b1 = new double[2 * cols]; - for( int r = 0; r < 3; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 3; r++) + for(int c = 0; c < cols; c++) b0[r * cols + c] = cell(r, c); - for( int r = 0; r < 2; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < 2; r++) + for(int c = 0; c < cols; c++) b1[r * cols + c] = cell(3 + r, c); java.util.ArrayList batches = new java.util.ArrayList<>(); batches.add(b0); @@ -258,8 +255,8 @@ public void fillDenseHandlesNonContiguousBlock() throws Exception { m.setAccessible(true); m.invoke(null, ret, batches); - for( int r = 0; r < rows; r++ ) - for( int c = 0; c < cols; c++ ) + for(int r = 0; r < rows; r++) + for(int c = 0; c < cols; c++) assertEquals("r" + r + " c" + c, cell(r, c), ret.getDenseBlock().get(r, c), 0.0); } @@ -276,8 +273,8 @@ private static StructType statsSchema() { } /** - * Build a mocked scan-file row whose AddFile child has the given schema, null - * flag and (when not null) stats string, matching what {@code numRecords} reads. + * Build a mocked scan-file row whose AddFile child has the given schema, null flag and (when not null) stats + * string, matching what {@code numRecords} reads. */ private static Row addFileRow(StructType addSchema, boolean statsNull, String statsValue) { Row outer = mock(Row.class); @@ -285,12 +282,12 @@ private static Row addFileRow(StructType addSchema, boolean statsNull, String st when(outer.getStruct(InternalScanFileUtils.ADD_FILE_ORDINAL)).thenReturn(add); when(add.getSchema()).thenReturn(addSchema); int statsOrd = addSchema.fieldNames().indexOf("stats"); - if( statsOrd >= 0 ) { + if(statsOrd >= 0) { when(add.isNullAt(statsOrd)).thenReturn(statsNull); - if( !statsNull ) + if(!statsNull) when(add.getString(statsOrd)).thenReturn(statsValue); } - return outer; //the scan-file row numRecords consumes (its AddFile child is 'add') + return outer; // the scan-file row numRecords consumes (its AddFile child is 'add') } private static long numRecords(Row scanFileRow) throws Exception { diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java index 54a3bcf6334..65c4ec21c0f 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixReadWriteTest.java @@ -63,20 +63,19 @@ import io.delta.kernel.utils.CloseableIterator; /** - * Direct (no DML) round-trip tests for the native Delta Kernel based matrix - * reader/writer. Each test writes a MatrixBlock to a fresh local Delta table - * directory and reads it back, asserting dimensions and values match. + * Direct (no DML) round-trip tests for the native Delta Kernel based matrix reader/writer. Each test writes a + * MatrixBlock to a fresh local Delta table directory and reads it back, asserting dimensions and values match. */ public class DeltaMatrixReadWriteTest { - //small writer target file size (bytes) used to force a multi-file table - //layout cheaply, instead of brute-forcing huge row counts. + // small writer target file size (bytes) used to force a multi-file table + // layout cheaply, instead of brute-forcing huge row counts. private static final long SMALL_TARGET_FILE_SIZE = 256L * 1024; private static final int ROWS_MULTI_FILE = 100_000; private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); - //WriterDelta creates the table at the given (empty) directory + // WriterDelta creates the table at the given (empty) directory String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { WriterDelta writer = new WriterDelta(); @@ -92,9 +91,9 @@ private static MatrixBlock writeThenRead(MatrixBlock in) throws Exception { @Test public void parallelReadMatchesSerialMultiFile() throws Exception { - //force a multi-file table cheaply via a small writer target file size - //(rather than a huge row count), so the parallel per-file path is - //actually exercised rather than falling back to serial. + // force a multi-file table cheaply via a small writer target file size + // (rather than a huge row count), so the parallel per-file path is + // actually exercised rather than falling back to serial. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 13); in.recomputeNonZeros(); @@ -104,21 +103,18 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_par_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); - //sanity: confirm the table really is split across multiple files + // sanity: confirm the table really is split across multiple files long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } - assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, - files > 1); + assertTrue("expected a multi-file Delta table to exercise the parallel path, got " + files, files > 1); - MatrixBlock serial = new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - MatrixBlock parallel = new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); + MatrixBlock parallel = new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), parallel.getNumRows()); assertEquals("cols", serial.getNumColumns(), parallel.getNumColumns()); @@ -135,8 +131,8 @@ public void parallelReadMatchesSerialMultiFile() throws Exception { public void roundTripDenseSmall() throws Exception { MatrixBlock in = new MatrixBlock(3, 4, false); double v = 1.0; - for( int i=0; i<3; i++ ) - for( int j=0; j<4; j++ ) + for(int i = 0; i < 3; i++) + for(int j = 0; j < 4; j++) in.set(i, j, v++); in.recomputeNonZeros(); @@ -157,7 +153,7 @@ public void roundTripDenseRandom() throws Exception { @Test public void roundTripSparseRandom() throws Exception { - //values written are dense parquet, but exercise a sparse-ish input + // values written are dense parquet, but exercise a sparse-ish input MatrixBlock in = TestUtils.generateTestMatrixBlock(1200, 9, -5, 5, 0.1, 13); in.recomputeNonZeros(); MatrixBlock out = writeThenRead(in); @@ -168,7 +164,7 @@ public void roundTripSparseRandom() throws Exception { @Test public void roundTripMultiBatch() throws Exception { - //more rows than the writer batch size (4096) to exercise chunking + // more rows than the writer batch size (4096) to exercise chunking MatrixBlock in = TestUtils.generateTestMatrixBlock(10000, 5, 0, 100, 1.0, 1); MatrixBlock out = writeThenRead(in); assertEquals("rows", 10000, out.getNumRows()); @@ -182,8 +178,9 @@ public void readDiscoversUnknownDimensions() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); - //pass -1 dimensions: the reader must discover them from the table + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); + // pass -1 dimensions: the reader must discover them from the table MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 123, out.getNumRows()); assertEquals("cols", 6, out.getNumColumns()); @@ -212,22 +209,21 @@ public void emptyMatrixRoundTrip() throws Exception { @Test public void readNonDoubleNumericColumns() throws Exception { - //tables produced by external tools (or the frame writer) can carry - //long/int/boolean columns; the matrix reader must coerce them to double + // tables produced by external tools (or the frame writer) can carry + // long/int/boolean columns; the matrix reader must coerce them to double Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] longVals = {1, -2, 1_000_000_000L, 0}; - double[] intVals = {7, -8, 123456, 0}; + double[] intVals = {7, -8, 123456, 0}; double[] boolVals = {1, 0, 1, 0}; - writeTypedColumns(tablePath, - new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, + writeTypedColumns(tablePath, new DataType[] {LongType.LONG, IntegerType.INTEGER, BooleanType.BOOLEAN}, new double[][] {longVals, intVals, boolVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 3, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("long col r" + r, longVals[r], out.get(r, 0), 0.0); assertEquals("int col r" + r, intVals[r], out.get(r, 1), 0.0); assertEquals("bool col r" + r, boolVals[r], out.get(r, 2), 0.0); @@ -240,14 +236,14 @@ public void readNonDoubleNumericColumns() throws Exception { @Test public void rewriteSamePathReplacesData() throws Exception { - //writing to a path that already holds a Delta table must fully replace it + // writing to a path that already holds a Delta table must fully replace it Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { MatrixBlock first = TestUtils.generateTestMatrixBlock(50, 8, 0, 100, 1.0, 1); new WriterDelta().writeMatrixToHDFS(first, tablePath, 50, 8, -1, first.getNonZeros()); - //second write has different dimensions and values + // second write has different dimensions and values MatrixBlock second = TestUtils.generateTestMatrixBlock(20, 3, -5, 5, 1.0, 2); new WriterDelta().writeMatrixToHDFS(second, tablePath, 20, 3, -1, second.getNonZeros()); @@ -263,10 +259,10 @@ public void rewriteSamePathReplacesData() throws Exception { @Test public void parallelBufferedPathMatchesSerial() throws Exception { - //the direct fast path is always taken for SystemDS-written tables (exact - //row stats, no deletion vectors); force the buffered fallback to exercise - //its per-file decode + serial concatenation and assert it matches serial. - //force a multi-file table cheaply via a small writer target file size. + // the direct fast path is always taken for SystemDS-written tables (exact + // row stats, no deletion vectors); force the buffered fallback to exercise + // its per-file decode + serial concatenation and assert it matches serial. + // force a multi-file table cheaply via a small writer target file size. MatrixBlock in = TestUtils.generateTestMatrixBlock(ROWS_MULTI_FILE, 8, -10, 10, 1.0, 23); in.recomputeNonZeros(); @@ -276,19 +272,22 @@ public void parallelBufferedPathMatchesSerial() throws Exception { Path dir = Files.createTempDirectory("sysds_delta_buf_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected a multi-file Delta table, got " + files, files > 1); MatrixBlock serial = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); - //subclass that always declines the direct path -> readBuffered() + // subclass that always declines the direct path -> readBuffered() MatrixBlock buffered = new ReaderDeltaParallel() { - @Override protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { return false; } + @Override + protected boolean useDirectPath(DeltaKernelUtils.ScanHandle h) { + return false; + } }.readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", serial.getNumRows(), buffered.getNumRows()); @@ -304,30 +303,30 @@ public void parallelBufferedPathMatchesSerial() throws Exception { @Test public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { - //a smaller configured target file size must make the writer roll more - //data files for the same matrix (the lever the parallel reader relies on). + // a smaller configured target file size must make the writer roll more + // data files for the same matrix (the lever the parallel reader relies on). MatrixBlock in = TestUtils.generateTestMatrixBlock(400_000, 16, -10, 10, 1.0, 7); in.recomputeNonZeros(); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(1L * 1024 * 1024)); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_cfg_"); try { - assertEquals("config getter reflects the override", - 1L * 1024 * 1024, ConfigurationManager.getDeltaWriterTargetFileSize()); + assertEquals("config getter reflects the override", 1L * 1024 * 1024, + ConfigurationManager.getDeltaWriterTargetFileSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); long files; - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { files = s.filter(p -> p.toString().endsWith(".parquet")).count(); } assertTrue("expected >1 data file with a 1MB target, got " + files, files > 1); - //data still round-trips correctly with the custom layout + // data still round-trips correctly with the custom layout MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-target-roundtrip"); } @@ -339,21 +338,20 @@ public void writerTargetFileSizeConfigProducesMoreFiles() throws Exception { @Test public void readerBatchSizeConfigRoundTrips() throws Exception { - //a non-default reader batch size must not change the result (more, smaller - //batches exercise the per-batch extract/concatenate loop more often). + // a non-default reader batch size must not change the result (more, smaller + // batches exercise the per-batch extract/concatenate loop more often). MatrixBlock in = TestUtils.generateTestMatrixBlock(5000, 7, -10, 10, 1.0, 11); - //isolate the override in a fresh thread-local config (restored in finally) + // isolate the override in a fresh thread-local config (restored in finally) DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_READER_BATCH_SIZE, "128"); ConfigurationManager.setLocalConfig(conf); Path dir = Files.createTempDirectory("sysds_delta_bs_"); try { - assertEquals("config getter reflects the override", - 128, ConfigurationManager.getDeltaReaderBatchSize()); + assertEquals("config getter reflects the override", 128, ConfigurationManager.getDeltaReaderBatchSize()); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); - new WriterDelta().writeMatrixToHDFS(in, tablePath, - in.getNumRows(), in.getNumColumns(), -1, in.getNonZeros()); + new WriterDelta().writeMatrixToHDFS(in, tablePath, in.getNumRows(), in.getNumColumns(), -1, + in.getNonZeros()); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); TestUtils.compareMatrices(in, out, 1e-12, "small-batch-roundtrip"); } @@ -365,14 +363,13 @@ public void readerBatchSizeConfigRoundTrips() throws Exception { @Test public void factoryRoutesDeltaToParallelWhenEnabled() { - //the factory must pick the parallel reader iff parallel CP read is enabled + // the factory must pick the parallel reader iff parallel CP read is enabled CompilerConfig cc = ConfigurationManager.getCompilerConfig(); try { cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, true); ConfigurationManager.setLocalConfig(cc); MatrixReader par = MatrixReaderFactory.createMatrixReader(FileFormat.DELTA); - assertTrue("expected ReaderDeltaParallel when parallel read enabled", - par instanceof ReaderDeltaParallel); + assertTrue("expected ReaderDeltaParallel when parallel read enabled", par instanceof ReaderDeltaParallel); cc.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); ConfigurationManager.setLocalConfig(cc); @@ -387,20 +384,18 @@ public void factoryRoutesDeltaToParallelWhenEnabled() { @Test public void readFloatColumnsCoercedToDouble() throws Exception { - //float columns must be widened to double on read (exact-representable values) + // float columns must be widened to double on read (exact-representable values) Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] f0 = {1.5, -2.25, 0.0, 1024.5}; double[] f1 = {-0.5, 3.75, 100.125, -7.0}; - writeTypedColumns(tablePath, - new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, - new double[][] {f0, f1}); + writeTypedColumns(tablePath, new DataType[] {FloatType.FLOAT, FloatType.FLOAT}, new double[][] {f0, f1}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("f0 r" + r, f0[r], out.get(r, 0), 0.0); assertEquals("f1 r" + r, f1[r], out.get(r, 1), 0.0); } @@ -412,21 +407,20 @@ public void readFloatColumnsCoercedToDouble() throws Exception { @Test public void readShortByteColumnsCoercedToDouble() throws Exception { - //short/byte columns must be coerced to double on read, exercising the - //T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. + // short/byte columns must be coerced to double on read, exercising the + // T_SHORT / T_BYTE branches of ReaderDelta.getDoubleValue. Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { double[] shortVals = {1, -2, 30000, 0}; - double[] byteVals = {7, -8, 120, 0}; - writeTypedColumns(tablePath, - new DataType[] {ShortType.SHORT, ByteType.BYTE}, + double[] byteVals = {7, -8, 120, 0}; + writeTypedColumns(tablePath, new DataType[] {ShortType.SHORT, ByteType.BYTE}, new double[][] {shortVals, byteVals}); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 2, out.getNumColumns()); - for( int r=0; r<4; r++ ) { + for(int r = 0; r < 4; r++) { assertEquals("short col r" + r, shortVals[r], out.get(r, 0), 0.0); assertEquals("byte col r" + r, byteVals[r], out.get(r, 1), 0.0); } @@ -438,8 +432,8 @@ public void readShortByteColumnsCoercedToDouble() throws Exception { @Test public void writerRejectsDimensionMismatch() throws Exception { - //WriterDelta validates that the passed rlen/clen match the MatrixBlock - //and rejects a mismatch with an IOException. + // WriterDelta validates that the passed rlen/clen match the MatrixBlock + // and rejects a mismatch with an IOException. MatrixBlock in = TestUtils.generateTestMatrixBlock(10, 4, -1, 1, 1.0, 5); in.recomputeNonZeros(); Path dir = Files.createTempDirectory("sysds_delta_"); @@ -459,18 +453,18 @@ public void writerRejectsDimensionMismatch() throws Exception { @Test public void readNullCellsBecomeZero() throws Exception { - //nullable numeric columns with null cells must read back as 0.0 + // nullable numeric columns with null cells must read back as 0.0 Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - double[] vals = {3.0, 7.0, 9.0, 11.0}; - boolean[] nulls = {false, true, false, true}; + double[] vals = {3.0, 7.0, 9.0, 11.0}; + boolean[] nulls = {false, true, false, true}; writeNullableDoubleColumn(tablePath, vals, nulls); MatrixBlock out = new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1); assertEquals("rows", 4, out.getNumRows()); assertEquals("cols", 1, out.getNumColumns()); - for( int r=0; r<4; r++ ) + for(int r = 0; r < 4; r++) assertEquals("r" + r, nulls[r] ? 0.0 : vals[r], out.get(r, 0), 0.0); } finally { @@ -480,7 +474,7 @@ public void readNullCellsBecomeZero() throws Exception { @Test public void readStringColumnRejected() throws Exception { - //string columns cannot back an all-double matrix -> reader must reject them + // string columns cannot back an all-double matrix -> reader must reject them Path dir = Files.createTempDirectory("sysds_delta_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -503,9 +497,10 @@ public void readStringColumnRejected() throws Exception { private static void writeTypedColumns(String tablePath, DataType[] types, double[][] vals) throws Exception { Engine engine = DeltaKernelUtils.createEngine(); StructType schema = new StructType(); - for( int c=0; c singleton(FilteredColumnarBatch fcb) { return new CloseableIterator() { private boolean _done = false; - @Override public boolean hasNext() { return !_done; } - @Override public FilteredColumnarBatch next() { - if( _done ) throw new NoSuchElementException(); + + @Override + public boolean hasNext() { + return !_done; + } + + @Override + public FilteredColumnarBatch next() { + if(_done) + throw new NoSuchElementException(); _done = true; return fcb; } - @Override public void close() {} + + @Override + public void close() { + } }; } - /** Minimal in-memory columnar batch backed by per-column double[] values, with - * an optional per-column null mask ({@code nulls==null} => no nulls). */ + /** + * Minimal in-memory columnar batch backed by per-column double[] values, with an optional per-column null mask + * ({@code nulls==null} => no nulls). + */ private static class TypedBatch implements ColumnarBatch { private final StructType _schema; private final DataType[] _types; private final double[][] _vals; private final boolean[][] _nulls; + TypedBatch(StructType schema, DataType[] types, double[][] vals, boolean[][] nulls) { - _schema = schema; _types = types; _vals = vals; _nulls = nulls; + _schema = schema; + _types = types; + _vals = vals; + _nulls = nulls; } - @Override public StructType getSchema() { return _schema; } - @Override public int getSize() { return _vals[0].length; } - @Override public ColumnVector getColumnVector(int ordinal) { - return new TypedVector(_types[ordinal], _vals[ordinal], - _nulls == null ? null : _nulls[ordinal]); + + @Override + public StructType getSchema() { + return _schema; + } + + @Override + public int getSize() { + return _vals[0].length; + } + + @Override + public ColumnVector getColumnVector(int ordinal) { + return new TypedVector(_types[ordinal], _vals[ordinal], _nulls == null ? null : _nulls[ordinal]); } } @@ -568,28 +599,98 @@ private static class TypedVector implements ColumnVector { private final DataType _type; private final double[] _vals; private final boolean[] _nulls; - TypedVector(DataType type, double[] vals, boolean[] nulls) { _type = type; _vals = vals; _nulls = nulls; } - @Override public DataType getDataType() { return _type; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _nulls != null && _nulls[rowId]; } - @Override public double getDouble(int rowId) { return _vals[rowId]; } - @Override public float getFloat(int rowId) { return (float) _vals[rowId]; } - @Override public long getLong(int rowId) { return (long) _vals[rowId]; } - @Override public int getInt(int rowId) { return (int) _vals[rowId]; } - @Override public short getShort(int rowId) { return (short) _vals[rowId]; } - @Override public byte getByte(int rowId) { return (byte) _vals[rowId]; } - @Override public boolean getBoolean(int rowId) { return _vals[rowId] != 0; } - @Override public void close() {} + + TypedVector(DataType type, double[] vals, boolean[] nulls) { + _type = type; + _vals = vals; + _nulls = nulls; + } + + @Override + public DataType getDataType() { + return _type; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _nulls != null && _nulls[rowId]; + } + + @Override + public double getDouble(int rowId) { + return _vals[rowId]; + } + + @Override + public float getFloat(int rowId) { + return (float) _vals[rowId]; + } + + @Override + public long getLong(int rowId) { + return (long) _vals[rowId]; + } + + @Override + public int getInt(int rowId) { + return (int) _vals[rowId]; + } + + @Override + public short getShort(int rowId) { + return (short) _vals[rowId]; + } + + @Override + public byte getByte(int rowId) { + return (byte) _vals[rowId]; + } + + @Override + public boolean getBoolean(int rowId) { + return _vals[rowId] != 0; + } + + @Override + public void close() { + } } /** Column view exposing a String[] as a Delta string column. */ private static class StringVector implements ColumnVector { private final String[] _vals; - StringVector(String[] vals) { _vals = vals; } - @Override public DataType getDataType() { return StringType.STRING; } - @Override public int getSize() { return _vals.length; } - @Override public boolean isNullAt(int rowId) { return _vals[rowId] == null; } - @Override public String getString(int rowId) { return _vals[rowId]; } - @Override public void close() {} + + StringVector(String[] vals) { + _vals = vals; + } + + @Override + public DataType getDataType() { + return StringType.STRING; + } + + @Override + public int getSize() { + return _vals.length; + } + + @Override + public boolean isNullAt(int rowId) { + return _vals[rowId] == null; + } + + @Override + public String getString(int rowId) { + return _vals[rowId]; + } + + @Override + public void close() { + } } } diff --git a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java index 2d79b79f2dd..45194d12b5a 100644 --- a/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java +++ b/src/test/java/org/apache/sysds/test/component/io/DeltaMatrixSparkInteropTest.java @@ -49,24 +49,22 @@ import org.junit.Test; /** - * Cross-engine interoperability tests for the native (Delta Kernel based) matrix - * reader/writer against the reference Delta implementation (Delta's Spark - * connector, {@code delta-spark}, pulled in test-only). + * Cross-engine interoperability tests for the native (Delta Kernel based) matrix reader/writer against the reference + * Delta implementation (Delta's Spark connector, {@code delta-spark}, pulled in test-only). * - *

The other Delta matrix tests round-trip exclusively through SystemDS' own - * Kernel-based read/write paths, so they cannot catch a table that SystemDS - * writes in a way other Delta engines reject (or vice versa). These tests close - * that gap by routing data through two independent engines: + *

+ * The other Delta matrix tests round-trip exclusively through SystemDS' own Kernel-based read/write paths, so they + * cannot catch a table that SystemDS writes in a way other Delta engines reject (or vice versa). These tests close that + * gap by routing data through two independent engines: *

    - *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • - *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a - * table with deletion vectors / a second commit that the SystemDS writer - * never produces itself.
    • + *
  • SystemDS writes -> Spark/Delta reads (our output is spec-compliant), and
    • + *
  • Spark/Delta writes -> SystemDS reads, including a multi-file layout and a table with deletion vectors / a + * second commit that the SystemDS writer never produces itself.
    • *
* - *

Row order is never assumed: every table carries a unique id in column 0 and - * comparisons are keyed by that id, since neither engine guarantees row order - * across files. + *

+ * Row order is never assumed: every table carries a unique id in column 0 and comparisons are keyed by that id, since + * neither engine guarantees row order across files. */ @net.jcip.annotations.NotThreadSafe public class DeltaMatrixSparkInteropTest { @@ -75,23 +73,19 @@ public class DeltaMatrixSparkInteropTest { @BeforeClass public static void startSpark() { - //each test class runs in its own fork (surefire reuseForks=false), so this - //is the only SparkSession in the JVM and gets the Delta extensions injected. + // each test class runs in its own fork (surefire reuseForks=false), so this + // is the only SparkSession in the JVM and gets the Delta extensions injected. SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); - spark = SparkSession.builder() - .appName("sysds-delta-interop") - .master("local[2]") - .config("spark.ui.enabled", "false") - .config("spark.sql.shuffle.partitions", "2") + spark = SparkSession.builder().appName("sysds-delta-interop").master("local[2]") + .config("spark.ui.enabled", "false").config("spark.sql.shuffle.partitions", "2") .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") - .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - .getOrCreate(); + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog").getOrCreate(); } @AfterClass public static void stopSpark() { - if( spark != null ) + if(spark != null) spark.stop(); SparkSession.clearActiveSession(); SparkSession.clearDefaultSession(); @@ -100,13 +94,13 @@ public static void stopSpark() { @Test public void systemdsWriteSparkReadMultiFile() throws Exception { - //SystemDS writes a (forced) multi-file Delta table; the reference Delta - //engine (Spark) must read every data file back with matching values. + // SystemDS writes a (forced) multi-file Delta table; the reference Delta + // engine (Spark) must read every data file back with matching values. int rows = 500, cols = 5; MatrixBlock in = indexedMatrix(rows, cols); - //small target file size -> multiple parquet data files (exercise that an - //external reader stitches all of our data files, not just the first). + // small target file size -> multiple parquet data files (exercise that an + // external reader stitches all of our data files, not just the first). DMLConfig conf = new DMLConfig(); conf.setTextValue(DMLConfig.DELTA_WRITER_TARGET_FILE_SIZE, String.valueOf(16L * 1024)); ConfigurationManager.setLocalConfig(conf); @@ -122,10 +116,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { List read = df.collectAsList(); assertEquals(rows, read.size()); - for( Row r : read ) { + for(Row r : read) { int id = (int) Math.round(r.getDouble(0)); assertTrue("id in range: " + id, id >= 0 && id < rows); - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals("r" + id + " c" + c, in.get(id, c), r.getDouble(c), 1e-9); } } @@ -137,10 +131,10 @@ public void systemdsWriteSparkReadMultiFile() throws Exception { @Test public void sparkWriteSystemdsReadMultiFile() throws Exception { - //the reference Delta engine writes a multi-file table; both the serial and - //parallel SystemDS readers must reconstruct it (coercing long ids to double). + // the reference Delta engine writes a multi-file table; both the serial and + // parallel SystemDS readers must reconstruct it (coercing long ids to double). int rows = 600, cols = 4; - Dataset df = indexedDataFrame(rows, cols).repartition(3); //-> multiple data files + Dataset df = indexedDataFrame(rows, cols).repartition(3); // -> multiple data files Path dir = Files.createTempDirectory("sysds_delta_p2s_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { @@ -148,10 +142,10 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { assertTrue("spark should have written a multi-file table", countParquet(tablePath) > 1); Map expected = expectedById(rows, cols); - assertMatchesById(new ReaderDelta() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "serial"); - assertMatchesById(new ReaderDeltaParallel() - .readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, "parallel"); + assertMatchesById(new ReaderDelta().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "serial"); + assertMatchesById(new ReaderDeltaParallel().readMatrixFromHDFS(tablePath, -1, -1, -1, -1), expected, cols, + "parallel"); } finally { FileUtils.deleteQuietly(dir.toFile()); @@ -160,15 +154,15 @@ public void sparkWriteSystemdsReadMultiFile() throws Exception { @Test public void sparkDeletionVectorsSystemdsRead() throws Exception { - //a Delta table with deletion vectors + a second commit (the DELETE) is a - //layout the SystemDS writer never emits; the readers must honor the DV and - //return only the surviving rows. This exercises the hasDeletionVector path. + // a Delta table with deletion vectors + a second commit (the DELETE) is a + // layout the SystemDS writer never emits; the readers must honor the DV and + // return only the surviving rows. This exercises the hasDeletionVector path. int rows = 400, cols = 3, deleteBelow = 50; Path dir = Files.createTempDirectory("sysds_delta_dv_"); String tablePath = new File(dir.toFile(), "table").getAbsolutePath(); try { - //enable deletion vectors for tables created in this block, then delete a - //row range so Delta records a DV rather than rewriting the data files. + // enable deletion vectors for tables created in this block, then delete a + // row range so Delta records a DV rather than rewriting the data files. spark.conf().set(DV_DEFAULT, "true"); indexedDataFrame(rows, cols).write().format("delta").save(tablePath); spark.sql("DELETE FROM delta.`" + tablePath + "` WHERE c0 < " + deleteBelow); @@ -185,21 +179,20 @@ public void sparkDeletionVectorsSystemdsRead() throws Exception { assertMatchesById(parallel, expected, cols, "parallel-dv"); } finally { - //fresh fork per test class, so simply clearing the override is enough + // fresh fork per test class, so simply clearing the override is enough spark.conf().unset(DV_DEFAULT); FileUtils.deleteQuietly(dir.toFile()); } } - private static final String DV_DEFAULT = - "spark.databricks.delta.properties.defaults.enableDeletionVectors"; + private static final String DV_DEFAULT = "spark.databricks.delta.properties.defaults.enableDeletionVectors"; /** Matrix whose column 0 is the row index and remaining columns are exact doubles. */ private static MatrixBlock indexedMatrix(int rows, int cols) { MatrixBlock mb = new MatrixBlock(rows, cols, false); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { mb.set(r, 0, r); - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) mb.set(r, c, value(r, c)); } mb.recomputeNonZeros(); @@ -209,15 +202,15 @@ private static MatrixBlock indexedMatrix(int rows, int cols) { /** Spark DataFrame mirroring {@link #indexedMatrix} with columns c0..c(cols-1) as doubles. */ private static Dataset indexedDataFrame(int rows, int cols) { StructField[] fields = new StructField[cols]; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) fields[c] = DataTypes.createStructField("c" + c, DataTypes.DoubleType, false); StructType schema = DataTypes.createStructType(fields); List data = new ArrayList<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { Object[] vals = new Object[cols]; vals[0] = (double) r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) vals[c] = value(r, c); data.add(RowFactory.create(vals)); } @@ -231,10 +224,10 @@ private static double value(int row, int col) { private static Map expectedById(int rows, int cols) { Map exp = new HashMap<>(rows); - for( int r = 0; r < rows; r++ ) { + for(int r = 0; r < rows; r++) { double[] row = new double[cols]; row[0] = r; - for( int c = 1; c < cols; c++ ) + for(int c = 1; c < cols; c++) row[c] = value(r, c); exp.put(r, row); } @@ -246,25 +239,25 @@ private static void assertMatchesById(MatrixBlock out, Map ex assertEquals(tag + " rows", expected.size(), out.getNumRows()); assertEquals(tag + " cols", cols, out.getNumColumns()); boolean[] seen = new boolean[expected.size() == 0 ? 0 : maxId(expected) + 1]; - for( int r = 0; r < out.getNumRows(); r++ ) { + for(int r = 0; r < out.getNumRows(); r++) { int id = (int) Math.round(out.get(r, 0)); double[] exp = expected.get(id); assertTrue(tag + ": unexpected/duplicate id " + id, exp != null && id < seen.length && !seen[id]); seen[id] = true; - for( int c = 0; c < cols; c++ ) + for(int c = 0; c < cols; c++) assertEquals(tag + " id" + id + " c" + c, exp[c], out.get(r, c), 1e-9); } } private static int maxId(Map expected) { int m = 0; - for( int id : expected.keySet() ) + for(int id : expected.keySet()) m = Math.max(m, id); return m; } private static long countParquet(String tablePath) throws Exception { - try( java.util.stream.Stream s = Files.walk(new File(tablePath).toPath()) ) { + try(java.util.stream.Stream s = Files.walk(new File(tablePath).toPath())) { return s.filter(p -> p.toString().endsWith(".parquet")).count(); } } diff --git a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java index 472b61d8cd6..ae19b291882 100644 --- a/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java +++ b/src/test/java/org/apache/sysds/test/component/matrix/QuantilePickTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -26,11 +26,10 @@ /** * Tests the single-column (unweighted) branch of {@link MatrixBlock#pickValue(double, boolean)} and - * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used - * by the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted - * branch (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent - * (value, weight) representation. The two-column (weighted) branch is exercised separately through the compressed - * sort tests. + * {@link MatrixBlock#median()}. The values are assumed to be sorted in ascending order, mirroring the contract used by + * the quantile pick instructions. The unweighted branch uses the same ceil-based rank as the two-column weighted branch + * (with an implicit weight of 1 per value), so a single column yields the same quantile as the equivalent (value, + * weight) representation. The two-column (weighted) branch is exercised separately through the compressed sort tests. */ public class QuantilePickTest { diff --git a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java index 5c9ed821e78..05aca41ebc7 100644 --- a/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java +++ b/src/test/java/org/apache/sysds/test/component/tensor/TensorToStringTest.java @@ -30,7 +30,7 @@ public class TensorToStringTest { @Test public void testDecimalClampsFractionDigits() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 1}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 1}); tb.allocateBlock(); tb.set(0, 0, 5.244058388023880); // decimal=2 must print exactly two fraction digits, not DecimalFormat's default max of 3 @@ -41,10 +41,10 @@ public void testDecimalClampsFractionDigits() { @Test public void testDecimalPadsAndRounds() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits - tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit + tb.set(0, 0, 22.0); // integer-valued: padded up to the requested digits + tb.set(0, 1, 5.244058388023880); // rounded at the last requested digit String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, 4); assertTrue("expected 22.0000 padded: " + out, out.contains("22.0000")); assertTrue("expected 5.2441 rounded: " + out, out.contains("5.2441")); @@ -52,10 +52,10 @@ public void testDecimalPadsAndRounds() { @Test public void testNegativeDecimalUsesDefaultFormatting() { - TensorBlock tb = new TensorBlock(ValueType.FP64, new int[]{1, 2}); + TensorBlock tb = new TensorBlock(ValueType.FP64, new int[] {1, 2}); tb.allocateBlock(); - tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained - tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits + tb.set(0, 0, 22.0); // integer-valued: no fraction digits when unconstrained + tb.set(0, 1, 5.244058388023880); // default cap of three fraction digits // decimal < 0 leaves DecimalFormat unconstrained (no min/max fraction digits set) String out = DataConverter.toString(tb, false, " ", "\n", "[", "]", 1, 2, -1); assertTrue("expected unpadded 22: " + out, out.contains("22")); diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 810e2614e7c..426de81263f 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -52,18 +52,12 @@ public class QuantileTest extends AutomatedTestBase public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); - addTestConfiguration(TEST_NAME2, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); - addTestConfiguration(TEST_NAME3, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); - addTestConfiguration(TEST_NAME4, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); - addTestConfiguration(TEST_NAME5, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); - addTestConfiguration(TEST_NAME6, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); + addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); + addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); + addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); + addTestConfiguration(TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java new file mode 100644 index 00000000000..35cfb2982fc --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part2/BuiltinSTEPGlmTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.builtin.part2; + +import org.junit.Test; +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.common.Types.ExecType; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; + +public class BuiltinSTEPGlmTest extends AutomatedTestBase { + private final static String TEST_NAME = "stepGLM"; + private final static String TEST_DIR = "functions/builtin/"; + private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinSTEPGlmTest.class.getSimpleName() + "/"; + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {})); + } + + @Test + public void testLmMatrixDenseCPlm() { + runSTEPGlmTest(ExecType.CP); + } + + @Test + public void testLmMatrixSparseSPlm() { + runSTEPGlmTest(ExecType.SPARK); + } + + private void runSTEPGlmTest(ExecType instType) { + ExecMode platformOld = setExecMode(instType); + + try { + loadTestConfiguration(getTestConfiguration(TEST_NAME)); + + String HOME = SCRIPT_DIR + TEST_DIR; + + // Pointing to the generated validation DML script + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[] {}; + + // runTest executes the script; fails if the DML script invokes stop() + runTest(true, false, null, -1); + } + finally { + rtplatform = platformOld; + } + } +} diff --git a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java index 5de429a3c53..d8cfc4d9fd7 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/monitoring/FederatedBackendPerformanceTest.java @@ -91,10 +91,12 @@ public void testBackendPerformance() throws InterruptedException { taskFutures.forEach(res -> { try { Assert.assertEquals("Stats parsed correctly", res.get().statusCode(), 200); - } catch (InterruptedException e) { + } + catch(InterruptedException e) { Thread.currentThread().interrupt(); Assert.fail("Interrupted while fetching statistics: " + e.getMessage()); - } catch (ExecutionException e) { + } + catch(ExecutionException e) { Assert.fail("Failed to fetch statistics: " + e.getMessage()); } }); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java index f8acdd07930..1cab99ee1ab 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part4/FederatedLogicalTest.java @@ -375,9 +375,8 @@ public void federatedLogicalTest(String testname, Type op_type, ExecMode execMod int port2 = single_fed_worker ? 0 : getRandomAvailablePort(); int port3 = single_fed_worker ? 0 : getRandomAvailablePort(); int port4 = single_fed_worker ? 0 : getRandomAvailablePort(); - Process[] workers = startLocalFedWorkers(single_fed_worker - ? new int[] {port1} - : new int[] {port1, port2, port3, port4}); + Process[] workers = startLocalFedWorkers( + single_fed_worker ? new int[] {port1} : new int[] {port1, port2, port3, port4}); try { if(!isAlive(workers)) diff --git a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java index dbbf199f8fa..9d2bb583a4f 100644 --- a/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/indexing/LeftIndexingTest.java @@ -72,27 +72,26 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod } if(et == ExecType.SPARK) { - rtplatform = ExecMode.SPARK; + rtplatform = ExecMode.SPARK; } else { // rtplatform = (et==ExecType.MR)? ExecMode.HADOOP : ExecMode.SINGLE_NODE; - rtplatform = ExecMode.HYBRID; + rtplatform = ExecMode.HYBRID; } if( rtplatform == ExecMode.SPARK ) DMLScript.USE_LOCAL_SPARK_CONFIG = true; config.addVariable("rows", rows); - config.addVariable("cols", cols); + config.addVariable("cols", cols); - long rowstart=816, rowend=1229, colstart=967, colend=1009; - // long rowstart=2, rowend=4, colstart=9, colend=10; + long rowstart = 816, rowend = 1229, colstart = 967, colend = 1009; + // long rowstart=2, rowend=4, colstart=9, colend=10; /* - Random rand=new Random(System.currentTimeMillis()); - rowstart=(long)(rand.nextDouble()*((double)rows))+1; - rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; - colstart=(long)(rand.nextDouble()*((double)cols))+1; - colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; - */ + * Random rand=new Random(System.currentTimeMillis()); rowstart=(long)(rand.nextDouble()*((double)rows))+1; + * rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart; + * colstart=(long)(rand.nextDouble()*((double)cols))+1; + * colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart; + */ config.addVariable("rowstart", rowstart); config.addVariable("rowend", rowend); config.addVariable("colstart", colstart); @@ -119,17 +118,20 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod double sparsity=1.0;//rand.nextDouble(); double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis()); writeInputMatrix("A", A, true); - - sparsity=0.1;//rand.nextDouble(); - double[][] B = getRandomMatrix((int)(rowend-rowstart+1), (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.1;// rand.nextDouble(); + double[][] B = getRandomMatrix((int) (rowend - rowstart + 1), (int) (colend - colstart + 1), min, max, + sparsity, System.currentTimeMillis()); writeInputMatrix("B", B, true); - - sparsity=0.5;//rand.nextDouble(); - double[][] C = getRandomMatrix((int)(rowend), (int)(cols-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.5;// rand.nextDouble(); + double[][] C = getRandomMatrix((int) (rowend), (int) (cols - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("C", C, true); - - sparsity=0.01;//rand.nextDouble(); - double[][] D = getRandomMatrix(rows, (int)(colend-colstart+1), min, max, sparsity, System.currentTimeMillis()); + + sparsity = 0.01;// rand.nextDouble(); + double[][] D = getRandomMatrix(rows, (int) (colend - colstart + 1), min, max, sparsity, + System.currentTimeMillis()); writeInputMatrix("D", D, true); /* @@ -138,9 +140,9 @@ private void runTestLeftIndexing(ExecType et, LeftIndexingOp.LeftIndexingMethod * While loop iteration - 10 jobs * Final output write - 1 job */ - //boolean exceptionExpected = false; - //int expectedNumberOfJobs = 12; - //runTest(exceptionExpected, null, expectedNumberOfJobs); + // boolean exceptionExpected = false; + // int expectedNumberOfJobs = 12; + // runTest(exceptionExpected, null, expectedNumberOfJobs); boolean exceptionExpected = false; int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, null, expectedNumberOfJobs); diff --git a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java index a1b51ee9d81..fad1cc123c8 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/ScalarIOTest.java @@ -81,8 +81,9 @@ public void testDoubleScalarWrite() { fullDMLScriptName = HOME + "ScalarComputeWrite.dml"; runTest(true, false, null, -1); - double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1,1)).doubleValue(); - Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); + double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).doubleValue(); + Assert.assertEquals("Computation test for Doubles failed: Values not equal: " + double_scalar + + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar); } @Test @@ -122,8 +123,7 @@ public void testIntScalarRead() { programArgs = new String[]{"-args", String.valueOf(int_scalar), output("a.scalar")}; runTest(true, false, null, -1); - int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).intValue(); + int int_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)).intValue(); Assert.assertEquals("Values not equal: " + int_scalar + Opcodes.NOTEQUAL.toString() + int_out_scalar, int_scalar, int_out_scalar); @@ -143,8 +143,8 @@ public void testDoubleScalarRead() { programArgs = new String[]{ "-args", String.valueOf(double_scalar), output("a.scalar") }; runTest(true, false, null, -1); - double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)) - .get(new CellIndex(1,1)).doubleValue(); + double double_out_scalar = TestUtils.readDMLScalarFromHDFS(output(OUT_FILE)).get(new CellIndex(1, 1)) + .doubleValue(); Assert.assertEquals("Values not equal: " + double_scalar + Opcodes.NOTEQUAL.toString() + double_out_scalar, double_scalar, double_out_scalar, 0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java index a4013c3672d..20ae7aa63ea 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/delta/DeltaReadWriteTest.java @@ -35,14 +35,14 @@ /** * End-to-end DML test of the native Delta read/write path. * - *

The write and the read are run as two separate SystemDS executions - * on purpose. If they shared a single script/process, SystemDS would reuse the - * still-materialized in-memory matrix for the subsequent read and never invoke - * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache - * reports 0 HDFS hits in that case). Splitting the executions forces a genuine - * read from disk, and we additionally assert via {@link CacheStatistics} that - * the read run actually performed HDFS reads (the Delta table + the text - * reference) rather than serving the matrix from cache.

+ *

+ * The write and the read are run as two separate SystemDS executions on purpose. If they shared a single + * script/process, SystemDS would reuse the still-materialized in-memory matrix for the subsequent read and never invoke + * {@link org.apache.sysds.runtime.io.ReaderDelta} at all (verified: the cache reports 0 HDFS hits in that case). + * Splitting the executions forces a genuine read from disk, and we additionally assert via {@link CacheStatistics} that + * the read run actually performed HDFS reads (the Delta table + the text reference) rather than serving the matrix from + * cache. + *

*/ public class DeltaReadWriteTest extends AutomatedTestBase { @@ -54,10 +54,8 @@ public class DeltaReadWriteTest extends AutomatedTestBase { @Override public void setUp() { TestUtils.clearAssertionInformation(); - addTestConfiguration(WRITE_NAME, - new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] { "ref" })); - addTestConfiguration(READ_NAME, - new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] { "R" })); + addTestConfiguration(WRITE_NAME, new TestConfiguration(TEST_CLASS_DIR, WRITE_NAME, new String[] {"ref"})); + addTestConfiguration(READ_NAME, new TestConfiguration(TEST_CLASS_DIR, READ_NAME, new String[] {"R"})); } @Test @@ -84,17 +82,16 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { String deltaPath = output("deltaTable"); String refPath = output("ref"); fullDMLScriptName = HOME + WRITE_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - String.valueOf(rows), String.valueOf(cols), String.valueOf(sparsity), - deltaPath, refPath }; + programArgs = new String[] {"-stats", "-args", String.valueOf(rows), String.valueOf(cols), + String.valueOf(sparsity), deltaPath, refPath}; runTest(true, false, null, -1); // the write run must have materialized two matrices to disk (the Delta // table under test + the text reference); WriterDelta genuinely hitting // HDFS is what produces these write-side cache statistics. long hdfsWrites = CacheStatistics.getHDFSWrites(); - assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " - + hdfsWrites, hdfsWrites >= 2); + assertTrue("expected >= 2 HDFS writes in the write run (delta + reference), got " + hdfsWrites, + hdfsWrites >= 2); // and a real Delta table (transaction log) must have been created assertTrue("missing Delta transaction log under " + deltaPath, new File(deltaPath, "_delta_log").isDirectory()); @@ -102,19 +99,18 @@ private void runDeltaRoundTrip(int rows, int cols, double sparsity) { // ---- phase 2: fresh execution reads the Delta table and compares ---- getAndLoadTestConfiguration(READ_NAME); fullDMLScriptName = HOME + READ_NAME + ".dml"; - programArgs = new String[] { "-stats", "-args", - deltaPath, refPath, output("R") }; + programArgs = new String[] {"-stats", "-args", deltaPath, refPath, output("R")}; runTest(true, false, null, -1); // the read run must have materialized two matrices from disk (the Delta // table under test + the text reference); a cached/short-circuited read // would report fewer HDFS hits and fail here. long hdfsReads = CacheStatistics.getHDFSHits(); - assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " - + hdfsReads, hdfsReads >= 2); + assertTrue("expected >= 2 HDFS reads in the read run (delta + reference), got " + hdfsReads, + hdfsReads >= 2); HashMap R = readDMLMatrixFromOutputDir("R"); - //text-cell output omits exact zeros, so a missing cell means 0.0 + // text-cell output omits exact zeros, so a missing cell means 0.0 double diff = R.getOrDefault(new CellIndex(1, 1), 0.0); double nrow = R.getOrDefault(new CellIndex(1, 2), 0.0); double ncol = R.getOrDefault(new CellIndex(1, 3), 0.0); diff --git a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java index cc1412b1606..a844321c249 100644 --- a/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java +++ b/src/test/java/org/apache/sysds/test/functions/io/parquet/FrameParquetSchemaTest.java @@ -49,10 +49,9 @@ public class FrameParquetSchemaTest extends AutomatedTestBase { @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"Rout"})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Rout"})); } - /** * Test for sequential writer and reader * diff --git a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java index dfb3d8a19de..6e6e4665f5e 100644 --- a/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/jmlc/JMLConnectionTest.java @@ -42,12 +42,8 @@ */ @net.jcip.annotations.NotThreadSafe public class JMLConnectionTest extends AutomatedTestBase { - public static final String META = "{\"data_type\": \"matrix\",\n" + - " \"value_type\": \"double\", \n" + - " \"rows\": 1,\n" + - " \"cols\": 1,\n" + - " \"nnz\": 1,\n" + - " \"format\": \"csv\"}"; + public static final String META = "{\"data_type\": \"matrix\",\n" + " \"value_type\": \"double\", \n" + + " \"rows\": 1,\n" + " \"cols\": 1,\n" + " \"nnz\": 1,\n" + " \"format\": \"csv\"}"; private final static String TEST_NAME = "JMLConnectionTest"; private final static String TEST_DIR = "functions/jmlc/"; @@ -99,12 +95,14 @@ public void testConnectionInvalidInName() throws DMLException { conn.gatherMemStats(false); Assert.assertFalse(DMLScript.STATISTICS); - try (conn) { - conn.prepareScript("printx('hello')", new String[]{"$inScalar1", null}, new String[]{null}); + try(conn) { + conn.prepareScript("printx('hello')", new String[] {"$inScalar1", null}, new String[] {null}); throw new AssertionError("Test should have thrown a LanguageException"); - } catch (LanguageException e) { + } + catch(LanguageException e) { Assert.assertTrue(e.getMessage().startsWith("Invalid variable names")); - } finally { + } + finally { DMLScript.STATISTICS = oldStat; DMLScript.JMLC_MEM_STATISTICS = oldJMLCStat; } @@ -112,21 +110,24 @@ public void testConnectionInvalidInName() throws DMLException { @Test public void testConnectionParseLanguageException() { - try (Connection conn = new Connection()) { - conn.prepareScript("printx('hello')", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("printx('hello')", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a DMLException"); - } catch (DMLException e) { + } + catch(DMLException e) { Throwable cause = e.getCause(); - Assert.assertTrue(cause.getMessage().startsWith("ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); + Assert.assertTrue(cause.getMessage().startsWith( + "ERROR: [line 1:0] -> printx('hello') -- function printx is undefined in namespace .builtinNS")); } } @Test public void testConnectionParseException() { - try (Connection conn = new Connection()) { - conn.prepareScript("print('hello'", new String[]{}, new String[]{}); + try(Connection conn = new Connection()) { + conn.prepareScript("print('hello'", new String[] {}, new String[] {}); throw new AssertionError("Test should have thrown a ParseException"); - } catch (Exception e) { + } + catch(Exception e) { Assert.assertEquals("ParseException", e.getClass().getSimpleName()); } } @@ -144,10 +145,11 @@ public void testConnectionClose() { @Test public void testReadScriptHDFS() { - try (Connection conn = new Connection()) { + try(Connection conn = new Connection()) { conn.readScript("hdfs://localhost:9000/Test"); - } catch (IOException e) { - Assert.assertEquals("ConnectException",e.getClass().getSimpleName()); + } + catch(IOException e) { + Assert.assertEquals("ConnectException", e.getClass().getSimpleName()); } } diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java index 4852220861e..2178884ef5b 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedFullReuseTest.java @@ -111,17 +111,16 @@ public void federatedReuse(String test) { // Run reference dml script with normal matrix. Reuse of ba+*. fullDMLScriptName = HOME + test + "Reference.dml"; - programArgs = new String[] {"-stats", "-lineage", "reuse_full", - "-nvargs", "X1=" + input("X1"), "X2=" + input("X2"), "Y1=" + input("Y1"), - "Y2=" + input("Y2"), "Z=" + expected("Z")}; + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", "X1=" + input("X1"), + "X2=" + input("X2"), "Y1=" + input("Y1"), "Y2=" + input("Y2"), "Z=" + expected("Z")}; runTest(true, false, null, -1); long mmCount = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); // Run actual dml script with federated matrix // The fed workers reuse ba+* fullDMLScriptName = HOME + test + ".dml"; - programArgs = new String[] {"-stats","-lineage", "reuse_full", - "-nvargs", "X1=" + TestUtils.federatedAddress(port1, input("X1")), + programArgs = new String[] {"-stats", "-lineage", "reuse_full", "-nvargs", + "X1=" + TestUtils.federatedAddress(port1, input("X1")), "X2=" + TestUtils.federatedAddress(port2, input("X2")), "Y1=" + TestUtils.federatedAddress(port1, input("Y1")), "Y2=" + TestUtils.federatedAddress(port2, input("Y2")), "r=" + rows, "c=" + cols, "Z=" + output("Z")}; @@ -129,12 +128,12 @@ public void federatedReuse(String test) { long mmCount_fed = Statistics.getCPHeavyHitterCount(Opcodes.MMULT.toString()); long fedMMCount = Statistics.getCPHeavyHitterCount("fed_ba+*"); - // compare results + // compare results compareResults(1e-9); // compare matrix multiplication count - // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) - Assert.assertTrue("Violated reuse count: "+mmCount_fed+" == "+mmCount*2, - mmCount_fed == mmCount * 2); // #threads = 2 + // #federated execution of ba+* = #threads times #non-federated execution of ba+* (after reuse) + Assert.assertTrue("Violated reuse count: " + mmCount_fed + " == " + mmCount * 2, + mmCount_fed == mmCount * 2); // #threads = 2 switch(test) { case TEST_NAME1: // If the o/p is federated, fed_ba+* will be called everytime diff --git a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java index eca3628a89b..c86eb0f4941 100644 --- a/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/lineage/FedUDFReuseTest.java @@ -121,9 +121,8 @@ private void runTriUDFReuse(ExecMode execMode) { // Run reference dml script with normal matrix fullDMLScriptName = HOME + TEST_NAME + "Reference.dml"; - programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", - input("X1"), input("X2"), input("X3"), input("X4"), - Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; + programArgs = new String[] {"-lineage", "reuse_full", "-stats", "100", "-args", input("X1"), input("X2"), + input("X3"), input("X4"), Boolean.toString(rowPartitioned).toUpperCase(), expected("S")}; runTest(null); // Run actual dml script with federated matrix diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java index 18ca2fbc454..3ffdfe1d30b 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ToStringTest.java @@ -272,82 +272,79 @@ protected void toStringTestHelper(ExecMode platform, String testName, String exp } @Test - public void testPrintWithDecimal(){ + public void testPrintWithDecimal() { String testName = "ToString12"; String decimalPoints = "2"; String value = "22"; String expectedOutput = "22.00\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal2(){ + public void testPrintWithDecimal2() { String testName = "ToString12"; String decimalPoints = "2"; String value = "5.244058388023880"; String expectedOutput = "5.24\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal3(){ + public void testPrintWithDecimal3() { String testName = "ToString12"; String decimalPoints = "10"; String value = "5.244058388023880"; String expectedOutput = "5.2440583880\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal4(){ + public void testPrintWithDecimal4() { String testName = "ToString12"; String decimalPoints = "4"; String value = "5.244058388023880"; String expectedOutput = "5.2441\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - @Test - public void testPrintWithDecimal5(){ + public void testPrintWithDecimal5() { String testName = "ToString12"; String decimalPoints = "10"; String value = "0.000000008023880"; String expectedOutput = "0.0000000080\n"; - + addTestConfiguration(testName, new TestConfiguration(TEST_CLASS_DIR, testName)); toStringTestHelper2(ExecMode.SINGLE_NODE, testName, expectedOutput, decimalPoints, value); } - protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, String value) { + protected void toStringTestHelper2(ExecMode platform, String testName, String expectedOutput, String decimalPoints, + String value) { ExecMode platformOld = rtplatform; - + rtplatform = platform; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if (rtplatform == ExecMode.SPARK) + if(rtplatform == ExecMode.SPARK) DMLScript.USE_LOCAL_SPARK_CONFIG = true; try { // Create and load test configuration getAndLoadTestConfiguration(testName); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-args", output(OUTPUT_NAME), value, decimalPoints}; + programArgs = new String[] {"-args", output(OUTPUT_NAME), value, decimalPoints}; // Run DML and R scripts runTest(true, false, null, -1); diff --git a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java index 770c5b7c5bf..26143dc16ee 100644 --- a/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/ooc/ReshapeTest.java @@ -76,10 +76,9 @@ public ReshapeTest(int rlen, int clen, int rows, int cols, boolean rowWise) { @Parameterized.Parameters(name = "{0}x{1} {2}x{3} rowWise {4}") public static Iterable getParams() { - int[][][] dims = { - {{1000, 1000}, {1, 1000000}}, // single row/col - {{3000, 4000}, {1500, 8000}}, // partialBlocks - {{2400, 1400}, {800, 4200}} // fullBlocks + int[][][] dims = {{{1000, 1000}, {1, 1000000}}, // single row/col + {{3000, 4000}, {1500, 8000}}, // partialBlocks + {{2400, 1400}, {800, 4200}} // fullBlocks }; ArrayList params = new ArrayList<>(); @@ -117,7 +116,8 @@ public void runTestMatrixReshapeOOC() { double[][] X = getRandomMatrix(rlen, clen, 0, 1, 1, 7); MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(Types.FileFormat.BINARY); - writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, rlen * clen); + writer.writeMatrixToHDFS(DataConverter.convertToMatrixBlock(X), input(INPUT_NAME), rlen, clen, 1000, + rlen * clen); HDFSTool.writeMetaDataFile(input(INPUT_NAME + ".mtd"), Types.ValueType.FP64, new MatrixCharacteristics(rlen, clen, blen, rlen * clen), Types.FileFormat.BINARY); @@ -143,8 +143,8 @@ public void runTestMatrixReshapeOOC() { runTest(true, false, null, -1); // compare results - MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), - Types.FileFormat.BINARY, rows, cols, blen); + MatrixBlock actual = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME), Types.FileFormat.BINARY, rows, + cols, blen); MatrixBlock expected = DataConverter.readMatrixFromHDFS(output(OUTPUT_NAME + "_target"), Types.FileFormat.BINARY, rows, cols, blen); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java index 5f42db7d733..49a52587cde 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/MatrixReshapeTest.java @@ -335,9 +335,9 @@ private void runTestMatrixReshape( ReshapeType type, boolean rowwise, boolean sp String.valueOf(trows), String.valueOf(tcols), output("Y") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + trows + " " + tcols + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + trows + " " + tcols + " " + + expectedDir(); + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); writeInputMatrix("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java index dcdafddcd47..69d6958f8a6 100644 --- a/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/reorg/VectorReshapeTest.java @@ -94,9 +94,9 @@ private void runVectorReshape(boolean sparse, ExecType et) String.valueOf(rows2), String.valueOf(cols2), output("R") }; fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + - inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); - + rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + rows2 + " " + cols2 + " " + + expectedDir(); + double sparsity = sparse ? sparsitySparse : sparsityDense; double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); writeInputMatrixWithMTD("X", X, true); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index 60b491b8141..b16554045e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -151,8 +151,8 @@ private void runTestMatrixChainDP(String testName) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index bf9acd9e52a..96c479e206d 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -123,8 +123,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + - getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail( + "Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); @@ -132,8 +132,7 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", - "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java index e8e885f905f..15b80e49618 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteQuantizationFusedCompressionTest.java @@ -74,7 +74,7 @@ public void testRewriteQuantizationFusedCompressionNoRewrite() { /** * Unified method to test both scalar and matrix scale factors. - * + * * @param testname Test name * @param rewrites Whether to enable fusion rewrites * @param isScalar Whether the scale factor is a scalar or a matrix diff --git a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java index 30681f373e4..39266f5f3d3 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/GetCategoricalMaskTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -106,7 +106,8 @@ public void testHash2() throws Exception { @Test public void testHash3() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8}, 32); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8}, 32); MatrixBlock expected = new MatrixBlock(1, 7, new double[] {1, 1, 1, 0, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3], \"hash\": [1,3], \"K\": 3}"; @@ -114,11 +115,11 @@ public void testHash3() throws Exception { } - @Test public void testHybrid1() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.INT64,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.INT64, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 9, new double[] {1, 1, 1, 0, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -127,8 +128,9 @@ public void testHybrid1() throws Exception { @Test public void testHybrid2() throws Exception { - FrameBlock fb = TestUtils.generateRandomFrameBlock(100, new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN,ValueType.UINT8, ValueType.BOOLEAN}, 32); - MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1,1, 1, 1, 1,1,1}); + FrameBlock fb = TestUtils.generateRandomFrameBlock(100, + new ValueType[] {ValueType.UINT8, ValueType.BOOLEAN, ValueType.UINT8, ValueType.BOOLEAN}, 32); + MatrixBlock expected = new MatrixBlock(1, 10, new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); String spec = "{\"ids\": true, \"dummycode\": [1,2,3,4], \"hash\": [1,3], \"K\": 3}"; runTransformTest(fb, spec, expected); @@ -139,7 +141,7 @@ private void runTransformTest(FrameBlock fb, String spec, MatrixBlock expected) try { getAndLoadTestConfiguration(TEST_NAME1); - + String inF = input("F-In"); String inS = input("spec"); diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java index 8c4ba6ae8ad..cd28649dc42 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformFrameEncodeBagOfWords.java @@ -283,7 +283,8 @@ private String[][] readTwoColumnStringCSV(String s) { out[1][i] = in.getString(i, 1); } return out; - } catch (IOException e) { + } + catch(IOException e) { throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java index d3d71d820d6..ae13cbd510f 100644 --- a/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/vect/LeftIndexingChainUpdateTest.java @@ -92,7 +92,7 @@ private void runVectorizationTest( String testName, boolean rewrites ) runTest(true, false, null, -1); runRScript(true); - //compare results + // compare results HashMap dmlfile = readDMLMatrixFromOutputDir("R"); HashMap rfile = readRMatrixFromExpectedDir("R"); TestUtils.compareMatrices(dmlfile, rfile, 1e-14, "DML", "R"); diff --git a/src/test/scripts/functions/builtin/stepGLM.dml b/src/test/scripts/functions/builtin/stepGLM.dml new file mode 100644 index 00000000000..392c6b6d245 --- /dev/null +++ b/src/test/scripts/functions/builtin/stepGLM.dml @@ -0,0 +1,58 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +source("scripts/builtin/stepGLM.dml") as stepGLM; + +N = 1000; +P = 10; +X = rand(rows=N, cols=P, min=-1.0, max=1.0, pdf="uniform", seed=123); + +beta_true = matrix(0, rows=P, cols=1); +beta_true[2,1] = 3.5; +beta_true[5,1] = -2.0; +beta_true[8,1] = 1.5; + +Z = X %*% beta_true; +P_y = 1.0 / (1.0 + exp(-Z)); +Y = (rand(rows=N, cols=1, min=0.0, max=1.0, seed=456) < P_y) * 1.0; + +[AIC, B, S] = stepGLM::m_stepGLM(X=X, Y=Y, link=2, yneg=0.0, icpt=0, tol=1e-6, disp=0.0, moi=200, mii=0, thr=0.01); + + +beta_est = matrix(0, rows=P, cols=1); +for (i in 1:nrow(B)) { + idx = as.scalar(S[1, i]); + beta_est[idx, 1] = B[i, 1]; +} + +# if beta_est and beta_true have the same sparsity +if (nrow(B) != 3 | sum(beta_est != 0 & beta_true == 0) > 0 | sum(beta_est == 0 & beta_true != 0) > 0) { + stop("Test failed: Unexpected non-zero element in beta_est"); +} + +# if maximal element divergence remains below epsilon bound +epsilon = 0.5; +if (max(abs(beta_est - beta_true)) > epsilon) { + stop("Test failed: Element divergence exceeds epsilon=" + epsilon +" tolerance."); +} + +